From a18807f89a873a5b1802ec90aa93092a3b94b3fd Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 21 Jun 2023 21:32:49 +0200 Subject: [PATCH 01/71] LLVM IR code generator refactoring and updates * Migrate to opaque pointers (LLVM 15+) which will be the only ones supported from LLVM 16 onwards * Change code generation model. Now the "client" classes don't need to concern themselves with **how** the LLVM IR code looks and how it is formatted. Instead, they build a model of the code they want to output and let the generator do the rest. * Handle many more tasks automatically: * LLVM IR string management and registration * Buffer management for structures which have pointers to buffers * References to local variables and strings * Local temporary counting (unnamed labels and function parameters) --- ...teCompressedAssembliesNativeSourceFiles.cs | 14 +- .../Tasks/GenerateJniRemappingNativeCode.cs | 8 +- .../Tasks/GeneratePackageManagerJava.cs | 30 +- ...pplicationConfigNativeAssemblyGenerator.cs | 168 +- ...ressedAssembliesNativeAssemblyGenerator.cs | 81 +- .../JniRemappingAssemblyGenerator.cs | 162 +- .../LlvmIrGenerator/Arm32LlvmIrGenerator.cs | 43 - .../LlvmIrGenerator/Arm64LlvmIrGenerator.cs | 47 - .../LlvmIrGenerator/FunctionAttributes.cs | 599 ++++- .../LlvmIrGenerator/IStructureInfo.cs | 15 - .../LlvmFunctionAttributeSet.cs | 50 - .../LlvmIrGenerator/LlvmIrBufferManager.cs | 67 + .../LlvmIrGenerator/LlvmIrComposer.cs | 79 +- .../LlvmIrGenerator/LlvmIrDataLayout.cs | 353 +++ .../LlvmIrGenerator/LlvmIrFunction.cs | 534 ++++- .../LlvmIrFunctionAttributeSet.cs | 106 + .../LlvmIrGenerator/LlvmIrFunctionBody.cs | 262 +++ .../LlvmIrGenerator/LlvmIrGenerator.Code.cs | 552 ----- .../LlvmIrGenerator.Constants.cs | 50 + .../LlvmIrGenerator/LlvmIrGenerator.cs | 2086 +++++++---------- .../LlvmIrGenerator/LlvmIrInstructions.cs | 479 ++++ .../LlvmIrGenerator/LlvmIrKnownMetadata.cs | 7 + .../LlvmIrGenerator/LlvmIrMetadataManager.cs | 57 +- .../Utilities/LlvmIrGenerator/LlvmIrModule.cs | 561 +++++ .../LlvmIrGenerator/LlvmIrModuleAArch64.cs | 52 + .../LlvmIrGenerator/LlvmIrModuleArmV7a.cs | 63 + .../LlvmIrGenerator/LlvmIrModuleTarget.cs | 73 + .../LlvmIrGenerator/LlvmIrModuleX64.cs | 76 + .../LlvmIrGenerator/LlvmIrModuleX86.cs | 68 + .../LlvmIrGenerator/LlvmIrStringGroup.cs | 16 + .../LlvmIrGenerator/LlvmIrStringManager.cs | 71 + .../LlvmIrGenerator/LlvmIrVariable.cs | 234 +- .../LlvmIrGenerator/LlvmIrVariableOptions.cs | 4 +- .../LlvmIrVariableReference.cs | 50 - .../LlvmNativeFunctionSignature.cs | 34 - .../LlvmIrGenerator/MemberInfoUtilities.cs | 2 +- .../LlvmIrGenerator/StructureInfo.cs | 75 +- .../LlvmIrGenerator/StructureInstance.cs | 67 +- .../LlvmIrGenerator/StructureMemberInfo.cs | 23 +- .../LlvmIrGenerator/StructureStringData.cs | 14 - .../LlvmIrGenerator/TypeUtilities.cs | 114 +- .../LlvmIrGenerator/X64LlvmIrGenerator.cs | 52 - .../LlvmIrGenerator/X86LlvmIrGenerator.cs | 45 - .../MarshalMethodsNativeAssemblyGenerator.cs | 705 ++++-- .../Utilities/TypeMapGenerator.cs | 51 +- ...TypeMappingDebugNativeAssemblyGenerator.cs | 36 +- ...peMappingReleaseNativeAssemblyGenerator.cs | 373 +-- 47 files changed, 5591 insertions(+), 3117 deletions(-) delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/Arm32LlvmIrGenerator.cs delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/Arm64LlvmIrGenerator.cs delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/IStructureInfo.cs delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmFunctionAttributeSet.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrBufferManager.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrDataLayout.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunctionAttributeSet.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunctionBody.cs delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.Code.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.Constants.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrInstructions.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrKnownMetadata.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModule.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModuleAArch64.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModuleArmV7a.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModuleTarget.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModuleX64.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModuleX86.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrStringGroup.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrStringManager.cs delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariableReference.cs delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmNativeFunctionSignature.cs delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/StructureStringData.cs delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/X64LlvmIrGenerator.cs delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/X86LlvmIrGenerator.cs diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs index 8367c21b744..48f849596e0 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs @@ -74,16 +74,22 @@ void GenerateCompressedAssemblySources () void Generate (IDictionary dict) { - var llvmAsmgen = new CompressedAssembliesNativeAssemblyGenerator (dict); - llvmAsmgen.Init (); + var composer = new CompressedAssembliesNativeAssemblyGenerator (dict); + LLVMIR.LlvmIrModule compressedAssemblies = composer.Construct (); foreach (string abi in SupportedAbis) { string baseAsmFilePath = Path.Combine (EnvironmentOutputDirectory, $"compressed_assemblies.{abi.ToLowerInvariant ()}"); string llvmIrFilePath = $"{baseAsmFilePath}.ll"; using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) { - llvmAsmgen.Write (GeneratePackageManagerJava.GetAndroidTargetArchForAbi (abi), sw, llvmIrFilePath); - sw.Flush (); + try { + composer.Generate (compressedAssemblies, GeneratePackageManagerJava.GetAndroidTargetArchForAbi (abi), sw, llvmIrFilePath); + } catch { + throw; + } finally { + sw.Flush (); + } + if (Files.CopyIfStreamChanged (sw.BaseStream, llvmIrFilePath)) { Log.LogDebugMessage ($"File {llvmIrFilePath} was regenerated"); } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJniRemappingNativeCode.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJniRemappingNativeCode.cs index 7bd5824012a..b89ce87d26d 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJniRemappingNativeCode.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJniRemappingNativeCode.cs @@ -77,16 +77,16 @@ void Generate () Generate (new JniRemappingAssemblyGenerator (typeReplacements, methodReplacements), typeReplacements.Count); } - void Generate (JniRemappingAssemblyGenerator jniRemappingGenerator, int typeReplacementsCount) + void Generate (JniRemappingAssemblyGenerator jniRemappingComposer, int typeReplacementsCount) { - jniRemappingGenerator.Init (); + LLVMIR.LlvmIrModule module = jniRemappingComposer.Construct (); foreach (string abi in SupportedAbis) { string baseAsmFilePath = Path.Combine (OutputDirectory, $"jni_remap.{abi.ToLowerInvariant ()}"); string llFilePath = $"{baseAsmFilePath}.ll"; using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) { - jniRemappingGenerator.Write (GeneratePackageManagerJava.GetAndroidTargetArchForAbi (abi), sw, llFilePath); + jniRemappingComposer.Generate (module, GeneratePackageManagerJava.GetAndroidTargetArchForAbi (abi), sw, llFilePath); sw.Flush (); Files.CopyIfStreamChanged (sw.BaseStream, llFilePath); } @@ -94,7 +94,7 @@ void Generate (JniRemappingAssemblyGenerator jniRemappingGenerator, int typeRepl BuildEngine4.RegisterTaskObjectAssemblyLocal ( ProjectSpecificTaskObjectKey (JniRemappingNativeCodeInfoKey), - new JniRemappingNativeCodeInfo (typeReplacementsCount, jniRemappingGenerator.ReplacementMethodIndexEntryCount), + new JniRemappingNativeCodeInfo (typeReplacementsCount, jniRemappingComposer.ReplacementMethodIndexEntryCount), RegisteredTaskObjectLifetime.Build ); } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index 623b20976de..670fcfe566e 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -390,7 +390,7 @@ void AddEnvironment () // and up to 4 other for arch-specific assemblies. Only **one** arch-specific store is ever loaded on the app // runtime, thus the number 2 here. All architecture specific stores contain assemblies with the same names // and in the same order. - MonoComponents = monoComponents, + MonoComponents = (MonoComponent)monoComponents, NativeLibraries = uniqueNativeLibraries, HaveAssemblyStore = UseAssemblyStore, AndroidRuntimeJNIEnvToken = android_runtime_jnienv_class_token, @@ -400,7 +400,7 @@ void AddEnvironment () JniRemappingReplacementMethodIndexEntryCount = jniRemappingNativeCodeInfo == null ? 0 : jniRemappingNativeCodeInfo.ReplacementMethodIndexEntryCount, MarshalMethodsEnabled = EnableMarshalMethods, }; - appConfigAsmGen.Init (); + LLVMIR.LlvmIrModule appConfigModule = appConfigAsmGen.Construct (); var marshalMethodsState = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (GenerateJavaStubs.MarshalMethodsRegisterTaskKey), RegisteredTaskObjectLifetime.Build); MarshalMethodsNativeAssemblyGenerator marshalMethodsAsmGen; @@ -415,7 +415,7 @@ void AddEnvironment () } else { marshalMethodsAsmGen = new MarshalMethodsNativeAssemblyGenerator (assemblyCount, uniqueAssemblyNames); } - marshalMethodsAsmGen.Init (); + LLVMIR.LlvmIrModule marshalMethodsModule = marshalMethodsAsmGen.Construct (); foreach (string abi in SupportedAbis) { string targetAbi = abi.ToLowerInvariant (); @@ -423,18 +423,28 @@ void AddEnvironment () string marshalMethodsBaseAsmFilePath = Path.Combine (EnvironmentOutputDirectory, $"marshal_methods.{targetAbi}"); string environmentLlFilePath = $"{environmentBaseAsmFilePath}.ll"; string marshalMethodsLlFilePath = $"{marshalMethodsBaseAsmFilePath}.ll"; - AndroidTargetArch targetArch = GetAndroidTargetArchForAbi (abi); + using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) { - appConfigAsmGen.Write (targetArch, sw, environmentLlFilePath); - sw.Flush (); - Files.CopyIfStreamChanged (sw.BaseStream, environmentLlFilePath); + try { + appConfigAsmGen.Generate (appConfigModule, targetArch, sw, environmentLlFilePath); + } catch { + throw; + } finally { + sw.Flush (); + Files.CopyIfStreamChanged (sw.BaseStream, environmentLlFilePath); + } } using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) { - marshalMethodsAsmGen.Write (targetArch, sw, marshalMethodsLlFilePath); - sw.Flush (); - Files.CopyIfStreamChanged (sw.BaseStream, marshalMethodsLlFilePath); + try { + marshalMethodsAsmGen.Generate (marshalMethodsModule, targetArch, sw, marshalMethodsLlFilePath); + } catch { + throw; + } finally { + sw.Flush (); + Files.CopyIfStreamChanged (sw.BaseStream, marshalMethodsLlFilePath); + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs index c240e5aefa2..2dcbac7b0e4 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs @@ -6,7 +6,6 @@ using Java.Interop.Tools.TypeNameMappings; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; - using Xamarin.Android.Tasks.LLVMIR; namespace Xamarin.Android.Tasks @@ -27,17 +26,13 @@ sealed class DSOCacheEntryContextDataProvider : NativeAssemblerStructContextData { public override string GetComment (object data, string fieldName) { - var dso_entry = data as DSOCacheEntry; - if (dso_entry == null) { - throw new InvalidOperationException ("Invalid data type, expected an instance of DSOCacheEntry"); - } - + var dso_entry = EnsureType (data); if (String.Compare ("hash", fieldName, StringComparison.Ordinal) == 0) { - return $"hash 0x{dso_entry.hash:x}, from name: {dso_entry.HashedName}"; + return $" hash 0x{dso_entry.hash:x}, from name: {dso_entry.HashedName}"; } if (String.Compare ("name", fieldName, StringComparison.Ordinal) == 0) { - return $"name: {dso_entry.name}"; + return $" name: {dso_entry.name}"; } return String.Empty; @@ -57,6 +52,7 @@ sealed class DSOCacheEntry public ulong hash; public bool ignore; + [NativeAssembler (UsesDataProvider = true)] public string name; public IntPtr handle = IntPtr.Zero; } @@ -131,24 +127,24 @@ sealed class XamarinAndroidBundledAssembly public uint name_length; [NativeAssembler (UsesDataProvider = true), NativePointer (PointsToPreAllocatedBuffer = true)] - public char name; + public string name; } // Keep in sync with FORMAT_TAG in src/monodroid/jni/xamarin-app.hh const ulong FORMAT_TAG = 0x015E6972616D58; - SortedDictionary environmentVariables; - SortedDictionary systemProperties; + SortedDictionary ? environmentVariables; + SortedDictionary ? systemProperties; TaskLoggingHelper log; - StructureInstance? application_config; + StructureInstance? application_config; List>? dsoCache; List>? xamarinAndroidBundledAssemblies; - StructureInfo? applicationConfigStructureInfo; - StructureInfo? dsoCacheEntryStructureInfo; - StructureInfo? xamarinAndroidBundledAssemblyStructureInfo; - StructureInfo assemblyStoreSingleAssemblyRuntimeDataStructureinfo; - StructureInfo assemblyStoreRuntimeDataStructureInfo; + StructureInfo? applicationConfigStructureInfo; + StructureInfo? dsoCacheEntryStructureInfo; + StructureInfo? xamarinAndroidBundledAssemblyStructureInfo; + StructureInfo? assemblyStoreSingleAssemblyRuntimeDataStructureinfo; + StructureInfo? assemblyStoreRuntimeDataStructureInfo; public bool UsesMonoAOT { get; set; } public bool UsesMonoLLVM { get; set; } @@ -188,8 +184,23 @@ public ApplicationConfigNativeAssemblyGenerator (IDictionary env this.log = log; } - public override void Init () + protected override void Construct (LlvmIrModule module) { + MapStructures (module); + + module.AddGlobalVariable ("format_tag", FORMAT_TAG, comment: $" 0x{FORMAT_TAG:x}"); + module.AddGlobalVariable ("mono_aot_mode_name", MonoAOTMode); + + var envVars = new LlvmIrGlobalVariable (environmentVariables, "app_environment_variables") { + Comment = " Application environment variables array, name:value", + }; + module.Add (envVars, stringGroupName: "env", stringGroupComment: " Application environment variables name:value pairs"); + + var sysProps = new LlvmIrGlobalVariable (systemProperties, "app_system_properties") { + Comment = " System properties defined by the application", + }; + module.Add (sysProps, stringGroupName: "sysprop", stringGroupComment: " System properties name:value pairs"); + dsoCache = InitDSOCache (); var app_cfg = new ApplicationConfig { uses_mono_llvm = UsesMonoLLVM, @@ -218,7 +229,14 @@ public override void Init () mono_components_mask = (uint)MonoComponents, android_package_name = AndroidPackageName, }; - application_config = new StructureInstance (app_cfg); + application_config = new StructureInstance (applicationConfigStructureInfo, app_cfg); + module.AddGlobalVariable ("application_config", application_config); + + var dso_cache = new LlvmIrGlobalVariable (dsoCache, "dso_cache", LlvmIrVariableOptions.GlobalWritable) { + Comment = " DSO cache entries", + BeforeWriteCallback = HashAndSortDSOCache, + }; + module.Add (dso_cache); if (!HaveAssemblyStore) { xamarinAndroidBundledAssemblies = new List> (NumberOfAssembliesInApk); @@ -229,13 +247,63 @@ public override void Init () data_size = 0, data = 0, name_length = (uint)BundledAssemblyNameWidth, - name = '\0', + name = null, }; for (int i = 0; i < NumberOfAssembliesInApk; i++) { - xamarinAndroidBundledAssemblies.Add (new StructureInstance (emptyBundledAssemblyData)); + xamarinAndroidBundledAssemblies.Add (new StructureInstance (xamarinAndroidBundledAssemblyStructureInfo, emptyBundledAssemblyData)); + } + } + + string bundledBuffersSize = xamarinAndroidBundledAssemblies == null ? "empty (unused when assembly stores are enabled)" : $"{BundledAssemblyNameWidth} bytes long"; + var bundled_assemblies = new LlvmIrGlobalVariable (typeof(List>), "bundled_assemblies", LlvmIrVariableOptions.GlobalWritable) { + Value = xamarinAndroidBundledAssemblies, + Comment = $" Bundled assembly name buffers, all {bundledBuffersSize}", + }; + module.Add (bundled_assemblies); + + AddAssemblyStores (module); + } + + void AddAssemblyStores (LlvmIrModule module) + { + ulong itemCount = (ulong)(HaveAssemblyStore ? NumberOfAssembliesInApk : 0); + var assembly_store_bundled_assemblies = new LlvmIrGlobalVariable (typeof(List>), "assembly_store_bundled_assemblies", LlvmIrVariableOptions.GlobalWritable) { + ZeroInitializeArray = true, + ArrayItemCount = itemCount, + }; + module.Add (assembly_store_bundled_assemblies); + + itemCount = (ulong)(HaveAssemblyStore ? NumberOfAssemblyStoresInApks : 0); + var assembly_stores = new LlvmIrGlobalVariable (typeof(List>), "assembly_stores", LlvmIrVariableOptions.GlobalWritable) { + ZeroInitializeArray = true, + ArrayItemCount = itemCount, + }; + module.Add (assembly_stores); + } + + void HashAndSortDSOCache (LlvmIrVariable variable, LlvmIrModuleTarget target, object? state) + { + var cache = variable.Value as List>; + if (cache == null) { + throw new InvalidOperationException ($"Internal error: DSO cache must no be empty"); + } + + bool is64Bit = target.Is64Bit; + foreach (StructureInstance instance in cache) { + if (instance.Obj == null) { + throw new InvalidOperationException ("Internal error: DSO cache must not contain null entries"); } + + var entry = instance.Obj as DSOCacheEntry; + if (entry == null) { + throw new InvalidOperationException ($"Internal error: DSO cache entry has unexpected type {instance.Obj.GetType ()}"); + } + + entry.hash = GetXxHash (entry.HashedName, is64Bit); } + + cache.Sort ((StructureInstance a, StructureInstance b) => a.Instance.hash.CompareTo (b.Instance.hash)); } List> InitDSOCache () @@ -273,7 +341,7 @@ List> InitDSOCache () name = name, }; - dsoCache.Add (new StructureInstance (entry)); + dsoCache.Add (new StructureInstance (dsoCacheEntryStructureInfo, entry)); } } @@ -300,56 +368,14 @@ void AddNameMutations (string name) } } - protected override void MapStructures (LlvmIrGenerator generator) - { - applicationConfigStructureInfo = generator.MapStructure (); - generator.MapStructure (); - assemblyStoreSingleAssemblyRuntimeDataStructureinfo = generator.MapStructure (); - assemblyStoreRuntimeDataStructureInfo = generator.MapStructure (); - xamarinAndroidBundledAssemblyStructureInfo = generator.MapStructure (); - dsoCacheEntryStructureInfo = generator.MapStructure (); - } - - protected override void Write (LlvmIrGenerator generator) - { - generator.WriteVariable ("format_tag", FORMAT_TAG); - generator.WriteString ("mono_aot_mode_name", MonoAOTMode); - - generator.WriteNameValueArray ("app_environment_variables", environmentVariables); - generator.WriteNameValueArray ("app_system_properties", systemProperties); - - generator.WriteStructure (applicationConfigStructureInfo, application_config, LlvmIrVariableOptions.GlobalConstant, "application_config"); - - WriteDSOCache (generator); - WriteBundledAssemblies (generator); - WriteAssemblyStoreAssemblies (generator); - } - - void WriteAssemblyStoreAssemblies (LlvmIrGenerator generator) - { - ulong count = (ulong)(HaveAssemblyStore ? NumberOfAssembliesInApk : 0); - generator.WriteStructureArray (assemblyStoreSingleAssemblyRuntimeDataStructureinfo, count, "assembly_store_bundled_assemblies", initialComment: "Assembly store individual assembly data"); - - count = (ulong)(HaveAssemblyStore ? NumberOfAssemblyStoresInApks : 0); - generator.WriteStructureArray (assemblyStoreRuntimeDataStructureInfo, count, "assembly_stores", initialComment: "Assembly store data"); - } - - void WriteBundledAssemblies (LlvmIrGenerator generator) + void MapStructures (LlvmIrModule module) { - generator.WriteStructureArray (xamarinAndroidBundledAssemblyStructureInfo, xamarinAndroidBundledAssemblies, "bundled_assemblies", initialComment: $"Bundled assembly name buffers, all {BundledAssemblyNameWidth} bytes long"); - } - - void WriteDSOCache (LlvmIrGenerator generator) - { - bool is64Bit = generator.Is64Bit; - - // We need to hash here, because the hash is architecture-specific - foreach (StructureInstance entry in dsoCache) { - entry.Obj.hash = HashName (entry.Obj.HashedName, is64Bit); - } - dsoCache.Sort ((StructureInstance a, StructureInstance b) => a.Obj.hash.CompareTo (b.Obj.hash)); - - generator.WriteStructureArray (dsoCacheEntryStructureInfo, dsoCache, "dso_cache"); + applicationConfigStructureInfo = module.MapStructure (); + module.MapStructure (); + assemblyStoreSingleAssemblyRuntimeDataStructureinfo = module.MapStructure (); + assemblyStoreRuntimeDataStructureInfo = module.MapStructure (); + xamarinAndroidBundledAssemblyStructureInfo = module.MapStructure (); + dsoCacheEntryStructureInfo = module.MapStructure (); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssembliesNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssembliesNativeAssemblyGenerator.cs index 39ce70a4d73..e0b3740a453 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssembliesNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssembliesNativeAssemblyGenerator.cs @@ -5,21 +5,21 @@ namespace Xamarin.Android.Tasks { - class CompressedAssembliesNativeAssemblyGenerator : LlvmIrComposer + partial class CompressedAssembliesNativeAssemblyGenerator : LlvmIrComposer { const string DescriptorsArraySymbolName = "compressed_assembly_descriptors"; const string CompressedAssembliesSymbolName = "compressed_assemblies"; sealed class CompressedAssemblyDescriptorContextDataProvider : NativeAssemblerStructContextDataProvider { - public override ulong GetBufferSize (object data, string fieldName) + public override string? GetPointedToSymbolName (object data, string fieldName) { if (String.Compare ("data", fieldName, StringComparison.Ordinal) != 0) { - return 0; + return null; } var descriptor = EnsureType (data); - return descriptor.uncompressed_file_size; + return descriptor.BufferSymbolName; } } @@ -28,10 +28,13 @@ public override ulong GetBufferSize (object data, string fieldName) [NativeAssemblerStructContextDataProvider (typeof (CompressedAssemblyDescriptorContextDataProvider))] sealed class CompressedAssemblyDescriptor { + [NativeAssembler (Ignore = true)] + public string BufferSymbolName; + public uint uncompressed_file_size; public bool loaded; - [NativeAssembler (UsesDataProvider = true), NativePointer (PointsToPreAllocatedBuffer = true)] + [NativeAssembler (UsesDataProvider = true), NativePointer (PointsToSymbol = "")] public byte data; }; @@ -60,60 +63,84 @@ sealed class CompressedAssemblies }; IDictionary assemblies; - StructureInfo compressedAssemblyDescriptorStructureInfo; - StructureInfo compressedAssembliesStructureInfo; - List>? compressedAssemblyDescriptors; - StructureInstance compressedAssemblies; + StructureInfo compressedAssemblyDescriptorStructureInfo; + StructureInfo compressedAssembliesStructureInfo; public CompressedAssembliesNativeAssemblyGenerator (IDictionary assemblies) { this.assemblies = assemblies; } - public override void Init () + void InitCompressedAssemblies (out List>? compressedAssemblyDescriptors, + out StructureInstance? compressedAssemblies, + out List? buffers) { if (assemblies == null || assemblies.Count == 0) { + compressedAssemblyDescriptors = null; + compressedAssemblies = null; + buffers = null; return; } + ulong counter = 0; compressedAssemblyDescriptors = new List> (assemblies.Count); + buffers = new List (assemblies.Count); foreach (var kvp in assemblies) { string assemblyName = kvp.Key; CompressedAssemblyInfo info = kvp.Value; + string bufferName = $"__compressedAssemblyData_{counter++}"; var descriptor = new CompressedAssemblyDescriptor { + BufferSymbolName = bufferName, uncompressed_file_size = info.FileSize, loaded = false, data = 0 }; - compressedAssemblyDescriptors.Add (new StructureInstance (descriptor)); + var bufferVar = new LlvmIrGlobalVariable (typeof(List), bufferName, LlvmIrVariableOptions.LocalWritable) { + ZeroInitializeArray = true, + ArrayItemCount = descriptor.uncompressed_file_size, + }; + buffers.Add (bufferVar); + + compressedAssemblyDescriptors.Add (new StructureInstance (compressedAssemblyDescriptorStructureInfo, descriptor)); } - compressedAssemblies = new StructureInstance (new CompressedAssemblies { count = (uint)assemblies.Count }); + compressedAssemblies = new StructureInstance (compressedAssembliesStructureInfo, new CompressedAssemblies { count = (uint)assemblies.Count }); } - protected override void MapStructures (LlvmIrGenerator generator) + protected override void Construct (LlvmIrModule module) { - compressedAssemblyDescriptorStructureInfo = generator.MapStructure (); - compressedAssembliesStructureInfo = generator.MapStructure (); - } + MapStructures (module); + + List>? compressedAssemblyDescriptors; + StructureInstance? compressedAssemblies; + List? buffers; + + InitCompressedAssemblies (out compressedAssemblyDescriptors, out compressedAssemblies, out buffers); - protected override void Write (LlvmIrGenerator generator) - { if (compressedAssemblyDescriptors == null) { - generator.WriteStructure (compressedAssembliesStructureInfo, null, CompressedAssembliesSymbolName); + module.AddGlobalVariable ( + typeof(StructureInstance), + CompressedAssembliesSymbolName, + new StructureInstance (compressedAssembliesStructureInfo, new CompressedAssemblies ()) { IsZeroInitialized = true }, + LlvmIrVariableOptions.GlobalWritable + ); return; } - generator.WriteStructureArray ( - compressedAssemblyDescriptorStructureInfo, - compressedAssemblyDescriptors, - LlvmIrVariableOptions.LocalWritable, - DescriptorsArraySymbolName, - initialComment: "Compressed assembly data storage" - ); - generator.WriteStructure (compressedAssembliesStructureInfo, compressedAssemblies, CompressedAssembliesSymbolName); + module.AddGlobalVariable (CompressedAssembliesSymbolName, compressedAssemblies, LlvmIrVariableOptions.GlobalWritable); + module.AddGlobalVariable (DescriptorsArraySymbolName, compressedAssemblyDescriptors, LlvmIrVariableOptions.LocalWritable); + + module.Add (new LlvmIrGroupDelimiterVariable ()); + module.Add (buffers); + module.Add (new LlvmIrGroupDelimiterVariable ()); + } + + void MapStructures (LlvmIrModule module) + { + compressedAssemblyDescriptorStructureInfo = module.MapStructure (); + compressedAssembliesStructureInfo = module.MapStructure (); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/JniRemappingAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/JniRemappingAssemblyGenerator.cs index 486fd2f4c7e..b7cbd4b49fd 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/JniRemappingAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/JniRemappingAssemblyGenerator.cs @@ -44,6 +44,9 @@ public JniRemappingMethodReplacement (string sourceType, string sourceMethod, st class JniRemappingAssemblyGenerator : LlvmIrComposer { + const string TypeReplacementsVariableName = "jni_remapping_type_replacements"; + const string MethodReplacementIndexVariableName = "jni_remapping_method_replacement_index"; + sealed class JniRemappingTypeReplacementEntryContextDataProvider : NativeAssemblerStructContextDataProvider { public override string GetComment (object data, string fieldName) @@ -51,11 +54,11 @@ public override string GetComment (object data, string fieldName) var entry = EnsureType(data); if (String.Compare ("name", fieldName, StringComparison.Ordinal) == 0) { - return $"name: {entry.name.str}"; + return $" name: {entry.name.str}"; } if (String.Compare ("replacement", fieldName, StringComparison.Ordinal) == 0) { - return $"replacement: {entry.replacement}"; + return $" replacement: {entry.replacement}"; } return String.Empty; @@ -69,7 +72,7 @@ public override string GetComment (object data, string fieldName) var entry = EnsureType (data); if (String.Compare ("name", fieldName, StringComparison.Ordinal) == 0) { - return $"name: {entry.name.str}"; + return $" name: {entry.name.str}"; } return String.Empty; @@ -104,11 +107,11 @@ public override string GetComment (object data, string fieldName) var entry = EnsureType (data); if (String.Compare ("name", fieldName, StringComparison.Ordinal) == 0) { - return $"name: {entry.name.str}"; + return $" name: {entry.name.str}"; } if (String.Compare ("replacement", fieldName, StringComparison.Ordinal) == 0) { - return $"replacement: {entry.replacement.target_type}.{entry.replacement.target_name}"; + return $" replacement: {entry.replacement.target_type}.{entry.replacement.target_name}"; } if (String.Compare ("signature", fieldName, StringComparison.Ordinal) == 0) { @@ -179,14 +182,11 @@ sealed class JniRemappingTypeReplacementEntry List typeReplacementsInput; List methodReplacementsInput; - StructureInfo jniRemappingStringStructureInfo; - StructureInfo jniRemappingReplacementMethodStructureInfo; - StructureInfo jniRemappingIndexMethodEntryStructureInfo; - StructureInfo jniRemappingIndexTypeEntryStructureInfo; - StructureInfo jniRemappingTypeReplacementEntryStructureInfo; - - List> typeReplacements; - List> methodIndexTypes; + StructureInfo jniRemappingStringStructureInfo; + StructureInfo jniRemappingReplacementMethodStructureInfo; + StructureInfo jniRemappingIndexMethodEntryStructureInfo; + StructureInfo jniRemappingIndexTypeEntryStructureInfo; + StructureInfo jniRemappingTypeReplacementEntryStructureInfo; public int ReplacementMethodIndexEntryCount { get; private set; } = 0; @@ -199,24 +199,25 @@ public JniRemappingAssemblyGenerator (List typeRepl this.methodReplacementsInput = methodReplacements ?? throw new ArgumentNullException (nameof (methodReplacements)); } - public override void Init () + (List>? typeReplacements, List>? methodIndexTypes) Init () { if (typeReplacementsInput == null) { - return; + return (null, null); } - typeReplacements = new List> (); + var typeReplacements = new List> (); + Console.WriteLine ($"Type replacement input count: {typeReplacementsInput.Count}"); foreach (JniRemappingTypeReplacement mtr in typeReplacementsInput) { var entry = new JniRemappingTypeReplacementEntry { name = MakeJniRemappingString (mtr.From), replacement = mtr.To, }; - typeReplacements.Add (new StructureInstance (entry)); + typeReplacements.Add (new StructureInstance (jniRemappingTypeReplacementEntryStructureInfo, entry)); } - typeReplacements.Sort ((StructureInstance l, StructureInstance r) => l.Obj.name.str.CompareTo (r.Obj.name.str)); + typeReplacements.Sort ((StructureInstance l, StructureInstance r) => l.Instance.name.str.CompareTo (r.Instance.name.str)); - methodIndexTypes = new List> (); + var methodIndexTypes = new List> (); var types = new Dictionary> (StringComparer.Ordinal); foreach (JniRemappingMethodReplacement mmr in methodReplacementsInput) { @@ -227,7 +228,7 @@ public override void Init () TypeMethods = new List> (), }; - typeEntry = new StructureInstance (entry); + typeEntry = new StructureInstance (jniRemappingIndexTypeEntryStructureInfo, entry); methodIndexTypes.Add (typeEntry); types.Add (mmr.SourceType, typeEntry); } @@ -242,17 +243,19 @@ public override void Init () }, }; - typeEntry.Obj.TypeMethods.Add (new StructureInstance (method)); + typeEntry.Instance.TypeMethods.Add (new StructureInstance (jniRemappingIndexMethodEntryStructureInfo, method)); } foreach (var kvp in types) { - kvp.Value.Obj.method_count = (uint)kvp.Value.Obj.TypeMethods.Count; - kvp.Value.Obj.TypeMethods.Sort ((StructureInstance l, StructureInstance r) => l.Obj.name.str.CompareTo (r.Obj.name.str)); + kvp.Value.Instance.method_count = (uint)kvp.Value.Instance.TypeMethods.Count; + kvp.Value.Instance.TypeMethods.Sort ((StructureInstance l, StructureInstance r) => l.Instance.name.str.CompareTo (r.Instance.name.str)); } - methodIndexTypes.Sort ((StructureInstance l, StructureInstance r) => l.Obj.name.str.CompareTo (r.Obj.name.str)); + methodIndexTypes.Sort ((StructureInstance l, StructureInstance r) => l.Instance.name.str.CompareTo (r.Instance.name.str)); ReplacementMethodIndexEntryCount = methodIndexTypes.Count; + return (typeReplacements, methodIndexTypes); + string MakeMethodsArrayName (string typeName) { return $"mm_{typeName.Replace ('/', '_')}"; @@ -265,101 +268,58 @@ JniRemappingString MakeJniRemappingString (string str) str = str, }; } - } - - uint GetLength (string str) - { - if (String.IsNullOrEmpty (str)) { - return 0; - } - - return (uint)Encoding.UTF8.GetBytes (str).Length; - } - - protected override void MapStructures (LlvmIrGenerator generator) - { - jniRemappingStringStructureInfo = generator.MapStructure (); - jniRemappingReplacementMethodStructureInfo = generator.MapStructure (); - jniRemappingIndexMethodEntryStructureInfo = generator.MapStructure (); - jniRemappingIndexTypeEntryStructureInfo = generator.MapStructure (); - jniRemappingTypeReplacementEntryStructureInfo = generator.MapStructure (); - } - - void WriteNestedStructure (LlvmIrGenerator generator, LlvmIrGenerator.StructureBodyWriterOptions bodyWriterOptions, Type structureType, object fieldInstance) - { - if (fieldInstance == null) { - return; - } - if (structureType == typeof (JniRemappingString)) { - generator.WriteNestedStructure (jniRemappingStringStructureInfo, new StructureInstance ((JniRemappingString)fieldInstance), bodyWriterOptions); - return; - } - - if (structureType == typeof (JniRemappingReplacementMethod)) { - generator.WriteNestedStructure (jniRemappingReplacementMethodStructureInfo, new StructureInstance ((JniRemappingReplacementMethod)fieldInstance), bodyWriterOptions); - return; - } - - if (structureType == typeof (JniRemappingIndexTypeEntry)) { - generator.WriteNestedStructure (jniRemappingIndexTypeEntryStructureInfo, new StructureInstance ((JniRemappingIndexTypeEntry)fieldInstance), bodyWriterOptions); - } + uint GetLength (string str) + { + if (String.IsNullOrEmpty (str)) { + return 0; + } - if (structureType == typeof (JniRemappingIndexMethodEntry)) { - generator.WriteNestedStructure (jniRemappingIndexMethodEntryStructureInfo, new StructureInstance ((JniRemappingIndexMethodEntry)fieldInstance), bodyWriterOptions); + return (uint)Encoding.UTF8.GetBytes (str).Length; } - - throw new InvalidOperationException ($"Unsupported nested structure type {structureType}"); } - protected override void Write (LlvmIrGenerator generator) + protected override void Construct (LlvmIrModule module) { - generator.WriteEOL (); - generator.WriteEOL ("JNI remapping data"); + MapStructures (module); + List>? typeReplacements; + List>? methodIndexTypes; + + (typeReplacements, methodIndexTypes) = Init (); if (typeReplacements == null) { - generator.WriteStructureArray ( - jniRemappingTypeReplacementEntryStructureInfo, - 0, - LlvmIrVariableOptions.GlobalConstant, - "jni_remapping_type_replacements" + module.AddGlobalVariable ( + typeof(StructureInstance), + TypeReplacementsVariableName, + new StructureInstance (jniRemappingTypeReplacementEntryStructureInfo, new JniRemappingTypeReplacementEntry ()) { IsZeroInitialized = true }, + LlvmIrVariableOptions.GlobalConstant ); - generator.WriteStructureArray ( - jniRemappingIndexTypeEntryStructureInfo, - 0, - LlvmIrVariableOptions.GlobalConstant, - "jni_remapping_method_replacement_index" + module.AddGlobalVariable ( + typeof(StructureInstance), + MethodReplacementIndexVariableName, + new StructureInstance (jniRemappingIndexTypeEntryStructureInfo, new JniRemappingIndexTypeEntry ()) { IsZeroInitialized = true }, + LlvmIrVariableOptions.GlobalConstant ); - return; } - generator.WriteStructureArray ( - jniRemappingTypeReplacementEntryStructureInfo, - typeReplacements, - LlvmIrVariableOptions.GlobalConstant, - "jni_remapping_type_replacements", - nestedStructureWriter: WriteNestedStructure - ); + module.AddGlobalVariable (TypeReplacementsVariableName, typeReplacements, LlvmIrVariableOptions.GlobalConstant); foreach (StructureInstance entry in methodIndexTypes) { - generator.WriteStructureArray ( - jniRemappingIndexMethodEntryStructureInfo, - entry.Obj.TypeMethods, - LlvmIrVariableOptions.LocalConstant, - entry.Obj.MethodsArraySymbolName, - nestedStructureWriter: WriteNestedStructure - ); + module.AddGlobalVariable (entry.Instance.MethodsArraySymbolName, entry.Instance.TypeMethods, LlvmIrVariableOptions.LocalConstant); } - generator.WriteStructureArray ( - jniRemappingIndexTypeEntryStructureInfo, - methodIndexTypes, - LlvmIrVariableOptions.GlobalConstant, - "jni_remapping_method_replacement_index", - nestedStructureWriter: WriteNestedStructure - ); + module.AddGlobalVariable (MethodReplacementIndexVariableName, methodIndexTypes, LlvmIrVariableOptions.GlobalConstant); + } + + void MapStructures (LlvmIrModule module) + { + jniRemappingStringStructureInfo = module.MapStructure (); + jniRemappingReplacementMethodStructureInfo = module.MapStructure (); + jniRemappingIndexMethodEntryStructureInfo = module.MapStructure (); + jniRemappingIndexTypeEntryStructureInfo = module.MapStructure (); + jniRemappingTypeReplacementEntryStructureInfo = module.MapStructure (); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/Arm32LlvmIrGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/Arm32LlvmIrGenerator.cs deleted file mode 100644 index 5f60214ea33..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/Arm32LlvmIrGenerator.cs +++ /dev/null @@ -1,43 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; - -using Xamarin.Android.Tools; - -namespace Xamarin.Android.Tasks.LLVMIR -{ - class Arm32LlvmIrGenerator : LlvmIrGenerator - { - // See https://llvm.org/docs/LangRef.html#data-layout - // - // Value as used by Android NDK's clang++ - // - protected override string DataLayout => "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"; - public override int PointerSize => 4; - protected override string Triple => "armv7-unknown-linux-android"; // NDK appends API level, we don't need that - - static readonly LlvmFunctionAttributeSet commonAttributes = new LlvmFunctionAttributeSet { - new FramePointerFunctionAttribute ("all"), - new TargetCpuFunctionAttribute ("generic"), - new TargetFeaturesFunctionAttribute ("+armv7-a,+d32,+dsp,+fp64,+neon,+thumb-mode,+vfp2,+vfp2sp,+vfp3,+vfp3d16,+vfp3d16sp,+vfp3sp,-aes,-fp-armv8,-fp-armv8d16,-fp-armv8d16sp,-fp-armv8sp,-fp16,-fp16fml,-fullfp16,-sha2,-vfp4,-vfp4d16,-vfp4d16sp,-vfp4sp"), - }; - - public Arm32LlvmIrGenerator (AndroidTargetArch arch, StreamWriter output, string fileName) - : base (arch, output, fileName) - {} - - protected override void AddModuleFlagsMetadata (List flagsFields) - { - base.AddModuleFlagsMetadata (flagsFields); - flagsFields.Add (MetadataManager.AddNumbered (LlvmIrModuleMergeBehavior.Error, "min_enum_size", 4)); - } - - protected override void InitFunctionAttributes () - { - base.InitFunctionAttributes (); - - FunctionAttributes[FunctionAttributesXamarinAppInit].Add (commonAttributes); - FunctionAttributes[FunctionAttributesJniMethods].Add (commonAttributes); - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/Arm64LlvmIrGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/Arm64LlvmIrGenerator.cs deleted file mode 100644 index 68ca5fd19e8..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/Arm64LlvmIrGenerator.cs +++ /dev/null @@ -1,47 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; - -using Xamarin.Android.Tools; - -namespace Xamarin.Android.Tasks.LLVMIR -{ - class Arm64LlvmIrGenerator : LlvmIrGenerator - { - // See https://llvm.org/docs/LangRef.html#data-layout - // - // Value as used by Android NDK's clang++ - // - protected override string DataLayout => "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"; - public override int PointerSize => 8; - protected override string Triple => "aarch64-unknown-linux-android"; // NDK appends API level, we don't need that - - static readonly LlvmFunctionAttributeSet commonAttributes = new LlvmFunctionAttributeSet { - new FramePointerFunctionAttribute ("non-leaf"), - new TargetCpuFunctionAttribute ("generic"), - new TargetFeaturesFunctionAttribute ("+neon,+outline-atomics"), - }; - - public Arm64LlvmIrGenerator (AndroidTargetArch arch, StreamWriter output, string fileName) - : base (arch, output, fileName) - {} - - protected override void AddModuleFlagsMetadata (List flagsFields) - { - base.AddModuleFlagsMetadata (flagsFields); - - flagsFields.Add (MetadataManager.AddNumbered (LlvmIrModuleMergeBehavior.Error, "branch-target-enforcement", 0)); - flagsFields.Add (MetadataManager.AddNumbered (LlvmIrModuleMergeBehavior.Error, "sign-return-address", 0)); - flagsFields.Add (MetadataManager.AddNumbered (LlvmIrModuleMergeBehavior.Error, "sign-return-address-all", 0)); - flagsFields.Add (MetadataManager.AddNumbered (LlvmIrModuleMergeBehavior.Error, "sign-return-address-with-bkey", 0)); - } - - protected override void InitFunctionAttributes () - { - base.InitFunctionAttributes (); - - FunctionAttributes[FunctionAttributesXamarinAppInit].Add (commonAttributes); - FunctionAttributes[FunctionAttributesJniMethods].Add (commonAttributes); - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/FunctionAttributes.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/FunctionAttributes.cs index d0bbc9cc7cc..a64f6194655 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/FunctionAttributes.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/FunctionAttributes.cs @@ -6,7 +6,7 @@ namespace Xamarin.Android.Tasks.LLVMIR { // Not all attributes are currently used throughout the code, but we define them call for potential future use. // Documentation can be found here: https://llvm.org/docs/LangRef.html#function-attributes - abstract class LLVMFunctionAttribute + abstract class LlvmIrFunctionAttribute : IComparable, IComparable, IEquatable { public string Name { get; } public bool Quoted { get; } @@ -14,7 +14,7 @@ abstract class LLVMFunctionAttribute public bool ParamsAreOptional { get; } public bool HasValueAsignment { get; } - protected LLVMFunctionAttribute (string name, bool quoted, bool supportsParams, bool optionalParams, bool hasValueAssignment) + protected LlvmIrFunctionAttribute (string name, bool quoted, bool supportsParams, bool optionalParams, bool hasValueAssignment) { Name = EnsureNonEmptyParameter (nameof (name), name); @@ -89,16 +89,93 @@ protected string EnsureNonEmptyParameter (string name, string value) return value; } + + public int CompareTo (object obj) + { + var attr = obj as LlvmIrFunctionAttribute; + if (obj == null) { + return 1; + } + + return CompareTo (attr); + } + + public int CompareTo (LlvmIrFunctionAttribute other) + { + return Name.CompareTo (other?.Name); + } + + public override int GetHashCode() + { + int hc = 0; + if (Name != null) { + hc ^= Name.GetHashCode (); + } + + return + hc ^ + Quoted.GetHashCode () ^ + SupportsParams.GetHashCode () ^ + ParamsAreOptional.GetHashCode () ^ + HasValueAsignment.GetHashCode (); + } + + public override bool Equals (object obj) + { + var attr = obj as LlvmIrFunctionAttribute; + if (attr == null) { + return false; + } + + return Equals (attr); + } + + public virtual bool Equals (LlvmIrFunctionAttribute other) + { + if (other == null) { + return false; + } + + if (String.Compare (Name, other.Name, StringComparison.Ordinal) != 0) { + return false; + } + + return + Quoted == other.Quoted && + SupportsParams == other.SupportsParams && + ParamsAreOptional == other.ParamsAreOptional && + HasValueAsignment == other.HasValueAsignment; + } + + public static bool operator > (LlvmIrFunctionAttribute a, LlvmIrFunctionAttribute b) + { + return a.CompareTo (b) > 0; + } + + public static bool operator < (LlvmIrFunctionAttribute a, LlvmIrFunctionAttribute b) + { + return a.CompareTo (b) < 0; + } + + public static bool operator >= (LlvmIrFunctionAttribute a, LlvmIrFunctionAttribute b) + { + return a.CompareTo (b) >= 0; + } + + public static bool operator <= (LlvmIrFunctionAttribute a, LlvmIrFunctionAttribute b) + { + return a.CompareTo (b) <= 0; + } } - abstract class LLVMFlagFunctionAttribute : LLVMFunctionAttribute + abstract class LlvmIrFlagFunctionAttribute : LlvmIrFunctionAttribute { - protected LLVMFlagFunctionAttribute (string name, bool quoted = false) + protected LlvmIrFlagFunctionAttribute (string name, bool quoted = false) : base (name, quoted, supportsParams: false, optionalParams: false, hasValueAssignment: false) {} } - class AlignstackFunctionAttribute : LLVMFunctionAttribute + class AlignstackFunctionAttribute : LlvmIrFunctionAttribute { uint alignment; @@ -116,9 +193,28 @@ protected override void RenderParams (StringBuilder sb) { sb.Append (alignment.ToString (CultureInfo.InvariantCulture)); } + + public override bool Equals (LlvmIrFunctionAttribute other) + { + if (!base.Equals (other)) { + return false; + } + + var attr = other as AlignstackFunctionAttribute; + if (attr == null) { + return false; + } + + return alignment == attr.alignment; + } + + public override int GetHashCode () + { + return base.GetHashCode () ^ alignment.GetHashCode (); + } } - class AllocFamilyFunctionAttribute : LLVMFunctionAttribute + class AllocFamilyFunctionAttribute : LlvmIrFunctionAttribute { string family; @@ -132,9 +228,28 @@ protected override void RenderAssignedValue (StringBuilder sb) { sb.Append (family); } + + public override bool Equals (LlvmIrFunctionAttribute other) + { + if (!base.Equals (other)) { + return false; + } + + var attr = other as AllocFamilyFunctionAttribute; + if (attr == null) { + return false; + } + + return String.Compare (family, attr.family, StringComparison.Ordinal) == 0; + } + + public override int GetHashCode () + { + return base.GetHashCode () ^ (family?.GetHashCode () ?? 0); + } } - class AllockindFunctionAttribute : LLVMFunctionAttribute + class AllockindFunctionAttribute : LlvmIrFunctionAttribute { string kind; @@ -150,9 +265,28 @@ protected override void RenderParams (StringBuilder sb) sb.Append (kind); sb.Append ('"'); } + + public override bool Equals (LlvmIrFunctionAttribute other) + { + if (!base.Equals (other)) { + return false; + } + + var attr = other as AllockindFunctionAttribute; + if (attr == null) { + return false; + } + + return String.Compare (kind, attr.kind, StringComparison.Ordinal) == 0; + } + + public override int GetHashCode () + { + return base.GetHashCode () ^ (kind?.GetHashCode () ?? 0); + } } - class AllocsizeFunctionAttribute : LLVMFunctionAttribute + class AllocsizeFunctionAttribute : LlvmIrFunctionAttribute { uint elementSize; uint? numberOfElements; @@ -174,65 +308,84 @@ protected override void RenderParams (StringBuilder sb) sb.Append (", "); sb.Append (numberOfElements.Value.ToString (CultureInfo.InvariantCulture)); } + + public override bool Equals (LlvmIrFunctionAttribute other) + { + if (!base.Equals (other)) { + return false; + } + + var attr = other as AllocsizeFunctionAttribute; + if (attr == null) { + return false; + } + + return elementSize == attr.elementSize && numberOfElements == attr.numberOfElements; + } + + public override int GetHashCode () + { + return base.GetHashCode () ^ elementSize.GetHashCode () ^ (numberOfElements?.GetHashCode () ?? 0); + } } - class AlwaysinlineFunctionAttribute : LLVMFlagFunctionAttribute + class AlwaysinlineFunctionAttribute : LlvmIrFlagFunctionAttribute { public AlwaysinlineFunctionAttribute () : base ("alwaysinline") {} } - class ArgmemonlyFunctionAttribute : LLVMFlagFunctionAttribute + class ArgmemonlyFunctionAttribute : LlvmIrFlagFunctionAttribute { public ArgmemonlyFunctionAttribute () : base ("argmemonly") {} } - class BuiltinFunctionAttribute : LLVMFlagFunctionAttribute + class BuiltinFunctionAttribute : LlvmIrFlagFunctionAttribute { public BuiltinFunctionAttribute () : base ("builtin") {} } - class ColdFunctionAttribute : LLVMFlagFunctionAttribute + class ColdFunctionAttribute : LlvmIrFlagFunctionAttribute { public ColdFunctionAttribute () : base ("cold") {} } - class ConvergentFunctionAttribute : LLVMFlagFunctionAttribute + class ConvergentFunctionAttribute : LlvmIrFlagFunctionAttribute { public ConvergentFunctionAttribute () : base ("convergent") {} } - class DisableSanitizerInstrumentationFunctionAttribute : LLVMFlagFunctionAttribute + class DisableSanitizerInstrumentationFunctionAttribute : LlvmIrFlagFunctionAttribute { public DisableSanitizerInstrumentationFunctionAttribute () : base ("disable_sanitizer_instrumentation") {} } - class DontcallErrorFunctionAttribute : LLVMFlagFunctionAttribute + class DontcallErrorFunctionAttribute : LlvmIrFlagFunctionAttribute { public DontcallErrorFunctionAttribute () : base ("dontcall-error", quoted: true) {} } - class DontcallWarnFunctionAttribute : LLVMFlagFunctionAttribute + class DontcallWarnFunctionAttribute : LlvmIrFlagFunctionAttribute { public DontcallWarnFunctionAttribute () : base ("dontcall-warn", quoted: true) {} } - class FramePointerFunctionAttribute : LLVMFunctionAttribute + class FramePointerFunctionAttribute : LlvmIrFunctionAttribute { string fpMode; @@ -252,387 +405,494 @@ public FramePointerFunctionAttribute (string fpMode = "none") } protected override void RenderAssignedValue (StringBuilder sb) => sb.Append (fpMode); + + public override bool Equals (LlvmIrFunctionAttribute other) + { + if (!base.Equals (other)) { + return false; + } + + var attr = other as FramePointerFunctionAttribute; + if (attr == null) { + return false; + } + + return String.Compare (fpMode, attr.fpMode, StringComparison.Ordinal) == 0; + } + + public override int GetHashCode () + { + return base.GetHashCode () ^ (fpMode?.GetHashCode () ?? 0); + } } - class HotFunctionAttribute : LLVMFlagFunctionAttribute + class HotFunctionAttribute : LlvmIrFlagFunctionAttribute { public HotFunctionAttribute () : base ("hot") {} } - class InaccessiblememonlyFunctionAttribute : LLVMFlagFunctionAttribute + class InaccessiblememonlyFunctionAttribute : LlvmIrFlagFunctionAttribute { public InaccessiblememonlyFunctionAttribute () : base ("inaccessiblememonly") {} } - class InaccessiblememOrArgmemonlyFunctionAttribute : LLVMFlagFunctionAttribute + class InaccessiblememOrArgmemonlyFunctionAttribute : LlvmIrFlagFunctionAttribute { public InaccessiblememOrArgmemonlyFunctionAttribute () : base ("inaccessiblemem_or_argmemonly") {} } - class InlinehintFunctionAttribute : LLVMFlagFunctionAttribute + class InlinehintFunctionAttribute : LlvmIrFlagFunctionAttribute { public InlinehintFunctionAttribute () : base ("inlinehint") {} } - class JumptableFunctionAttribute : LLVMFlagFunctionAttribute + class JumptableFunctionAttribute : LlvmIrFlagFunctionAttribute { public JumptableFunctionAttribute () : base ("jumptable") {} } - class MinsizeFunctionAttribute : LLVMFlagFunctionAttribute + enum MemoryAttributeAccessKind + { + None, + Read, + Write, + ReadWrite, + } + + class MemoryFunctionAttribute : LlvmIrFunctionAttribute + { + public MemoryAttributeAccessKind? Default { get; set; } + public MemoryAttributeAccessKind? Argmem { get; set; } + public MemoryAttributeAccessKind? InaccessibleMem { get; set; } + + public MemoryFunctionAttribute () + : base ("memory", quoted: false, supportsParams: true, optionalParams: true, hasValueAssignment: false) + {} + + protected override bool HasOptionalParams () + { + // All of them are optional, but at least one of them must be specified + bool ret = Default.HasValue || Argmem.HasValue || InaccessibleMem.HasValue; + if (!ret) { + throw new InvalidOperationException ("Internal error: at least one access kind must be specified"); + } + + return ret; + } + + protected override void RenderParams (StringBuilder sb) + { + bool haveSomething = false; + + if (Default.HasValue) { + AppendParam (GetAccessKindString (Default)); + } + + if (Argmem.HasValue) { + AppendParam ($"argmem: {GetAccessKindString (Argmem)}"); + } + + if (InaccessibleMem.HasValue) { + AppendParam ($"inaccessiblemem: {GetAccessKindString (InaccessibleMem)}"); + } + + void AppendParam (string text) + { + if (haveSomething) { + sb.Append (", "); + } + sb.Append (text); + haveSomething = true; + } + } + + string GetAccessKindString (MemoryAttributeAccessKind? kind) + { + return kind.Value switch { + MemoryAttributeAccessKind.None => "none", + MemoryAttributeAccessKind.Read => "read", + MemoryAttributeAccessKind.Write => "write", + MemoryAttributeAccessKind.ReadWrite => "readwrite", + _ => throw new InvalidOperationException ($"Internal error: unsupported access kind {kind}") + }; + } + + public override bool Equals (LlvmIrFunctionAttribute other) + { + if (!base.Equals (other)) { + return false; + } + + var attr = other as MemoryFunctionAttribute; + if (attr == null) { + return false; + } + + return Default == attr.Default && Argmem == attr.Argmem && InaccessibleMem == attr.InaccessibleMem; + } + } + + class MinsizeFunctionAttribute : LlvmIrFlagFunctionAttribute { public MinsizeFunctionAttribute () : base ("minsize") {} } - class NakedFunctionAttribute : LLVMFlagFunctionAttribute + class NakedFunctionAttribute : LlvmIrFlagFunctionAttribute { public NakedFunctionAttribute () : base ("naked") {} } - class NoInlineLineTablesFunctionAttribute : LLVMFlagFunctionAttribute + class NoInlineLineTablesFunctionAttribute : LlvmIrFlagFunctionAttribute { public NoInlineLineTablesFunctionAttribute () : base ("no-inline-line-tables", quoted: true) {} } - class NoJumpTablesFunctionAttribute : LLVMFlagFunctionAttribute + class NoJumpTablesFunctionAttribute : LlvmIrFlagFunctionAttribute { public NoJumpTablesFunctionAttribute () : base ("no-jump-tables") {} } - class NobuiltinFunctionAttribute : LLVMFlagFunctionAttribute + class NobuiltinFunctionAttribute : LlvmIrFlagFunctionAttribute { public NobuiltinFunctionAttribute () : base ("nobuiltin") {} } - class NoduplicateFunctionAttribute : LLVMFlagFunctionAttribute + class NocallbackFunctionAttribute : LlvmIrFlagFunctionAttribute + { + public NocallbackFunctionAttribute () + : base ("nocallback") + {} + } + + class NoduplicateFunctionAttribute : LlvmIrFlagFunctionAttribute { public NoduplicateFunctionAttribute () : base ("noduplicate") {} } - class NofreeFunctionAttribute : LLVMFlagFunctionAttribute + class NofreeFunctionAttribute : LlvmIrFlagFunctionAttribute { public NofreeFunctionAttribute () : base ("nofree") {} } - class NoimplicitfloatFunctionAttribute : LLVMFlagFunctionAttribute + class NoimplicitfloatFunctionAttribute : LlvmIrFlagFunctionAttribute { public NoimplicitfloatFunctionAttribute () : base ("noimplicitfloat") {} } - class NoinlineFunctionAttribute : LLVMFlagFunctionAttribute + class NoinlineFunctionAttribute : LlvmIrFlagFunctionAttribute { public NoinlineFunctionAttribute () : base ("noinline") {} } - class NomergeFunctionAttribute : LLVMFlagFunctionAttribute + class NomergeFunctionAttribute : LlvmIrFlagFunctionAttribute { public NomergeFunctionAttribute () : base ("nomerge") {} } - class NonlazybindFunctionAttribute : LLVMFlagFunctionAttribute + class NonlazybindFunctionAttribute : LlvmIrFlagFunctionAttribute { public NonlazybindFunctionAttribute () : base ("nonlazybind") {} } - class NoprofileFunctionAttribute : LLVMFlagFunctionAttribute + class NoprofileFunctionAttribute : LlvmIrFlagFunctionAttribute { public NoprofileFunctionAttribute () : base ("noprofile") {} } - class NoredzoneFunctionAttribute : LLVMFlagFunctionAttribute + class NoredzoneFunctionAttribute : LlvmIrFlagFunctionAttribute { public NoredzoneFunctionAttribute () : base ("noredzone") {} } - class IndirectTlsSegRefsFunctionAttribute : LLVMFlagFunctionAttribute + class IndirectTlsSegRefsFunctionAttribute : LlvmIrFlagFunctionAttribute { public IndirectTlsSegRefsFunctionAttribute () : base ("indirect-tls-seg-refs") {} } - class NoreturnFunctionAttribute : LLVMFlagFunctionAttribute + class NoreturnFunctionAttribute : LlvmIrFlagFunctionAttribute { public NoreturnFunctionAttribute () : base ("noreturn") {} } - class NorecurseFunctionAttribute : LLVMFlagFunctionAttribute + class NorecurseFunctionAttribute : LlvmIrFlagFunctionAttribute { public NorecurseFunctionAttribute () : base ("norecurse") {} } - class WillreturnFunctionAttribute : LLVMFlagFunctionAttribute + class WillreturnFunctionAttribute : LlvmIrFlagFunctionAttribute { public WillreturnFunctionAttribute () : base ("willreturn") {} } - class NosyncFunctionAttribute : LLVMFlagFunctionAttribute + class NosyncFunctionAttribute : LlvmIrFlagFunctionAttribute { public NosyncFunctionAttribute () : base ("nosync") {} } - class NounwindFunctionAttribute : LLVMFlagFunctionAttribute + class NounwindFunctionAttribute : LlvmIrFlagFunctionAttribute { public NounwindFunctionAttribute () : base ("nounwind") {} } - class NosanitizeBoundsFunctionAttribute : LLVMFlagFunctionAttribute + class NosanitizeBoundsFunctionAttribute : LlvmIrFlagFunctionAttribute { public NosanitizeBoundsFunctionAttribute () : base ("nosanitize_bounds") {} } - class NosanitizeCoverageFunctionAttribute : LLVMFlagFunctionAttribute + class NosanitizeCoverageFunctionAttribute : LlvmIrFlagFunctionAttribute { public NosanitizeCoverageFunctionAttribute () : base ("nosanitize_coverage") {} } - class NullPointerIsValidFunctionAttribute : LLVMFlagFunctionAttribute + class NullPointerIsValidFunctionAttribute : LlvmIrFlagFunctionAttribute { public NullPointerIsValidFunctionAttribute () : base ("null_pointer_is_valid") {} } - class OptforfuzzingFunctionAttribute : LLVMFlagFunctionAttribute + class OptforfuzzingFunctionAttribute : LlvmIrFlagFunctionAttribute { public OptforfuzzingFunctionAttribute () : base ("optforfuzzing") {} } - class OptnoneFunctionAttribute : LLVMFlagFunctionAttribute + class OptnoneFunctionAttribute : LlvmIrFlagFunctionAttribute { public OptnoneFunctionAttribute () : base ("optnone") {} } - class OptsizeFunctionAttribute : LLVMFlagFunctionAttribute + class OptsizeFunctionAttribute : LlvmIrFlagFunctionAttribute { public OptsizeFunctionAttribute () : base ("optsize") {} } - class PatchableFunctionFunctionAttribute : LLVMFlagFunctionAttribute + class PatchableFunctionFunctionAttribute : LlvmIrFlagFunctionAttribute { public PatchableFunctionFunctionAttribute () : base ("patchable-function", quoted: true) {} } - class ProbeStackFunctionAttribute : LLVMFlagFunctionAttribute + class ProbeStackFunctionAttribute : LlvmIrFlagFunctionAttribute { public ProbeStackFunctionAttribute () : base ("probe-stack") {} } - class ReadnoneFunctionAttribute : LLVMFlagFunctionAttribute + class ReadnoneFunctionAttribute : LlvmIrFlagFunctionAttribute { public ReadnoneFunctionAttribute () : base ("readnone") {} } - class ReadonlyFunctionAttribute : LLVMFlagFunctionAttribute + class ReadonlyFunctionAttribute : LlvmIrFlagFunctionAttribute { public ReadonlyFunctionAttribute () : base ("readonly") {} } - class StackProbeSizeFunctionAttribute : LLVMFlagFunctionAttribute + class StackProbeSizeFunctionAttribute : LlvmIrFlagFunctionAttribute { public StackProbeSizeFunctionAttribute () : base ("stack-probe-size", quoted: true) {} } - class NoStackArgProbeFunctionAttribute : LLVMFlagFunctionAttribute + class NoStackArgProbeFunctionAttribute : LlvmIrFlagFunctionAttribute { public NoStackArgProbeFunctionAttribute () : base ("no-stack-arg-probe") {} } - class WriteonlyFunctionAttribute : LLVMFlagFunctionAttribute + class WriteonlyFunctionAttribute : LlvmIrFlagFunctionAttribute { public WriteonlyFunctionAttribute () : base ("writeonly") {} } - class ReturnsTwiceFunctionAttribute : LLVMFlagFunctionAttribute + class ReturnsTwiceFunctionAttribute : LlvmIrFlagFunctionAttribute { public ReturnsTwiceFunctionAttribute () : base ("returns_twice") {} } - class SafestackFunctionAttribute : LLVMFlagFunctionAttribute + class SafestackFunctionAttribute : LlvmIrFlagFunctionAttribute { public SafestackFunctionAttribute () : base ("safestack") {} } - class SanitizeAddressFunctionAttribute : LLVMFlagFunctionAttribute + class SanitizeAddressFunctionAttribute : LlvmIrFlagFunctionAttribute { public SanitizeAddressFunctionAttribute () : base ("sanitize_address") {} } - class SanitizeMemoryFunctionAttribute : LLVMFlagFunctionAttribute + class SanitizeMemoryFunctionAttribute : LlvmIrFlagFunctionAttribute { public SanitizeMemoryFunctionAttribute () : base ("sanitize_memory") {} } - class SanitizeThreadFunctionAttribute : LLVMFlagFunctionAttribute + class SanitizeThreadFunctionAttribute : LlvmIrFlagFunctionAttribute { public SanitizeThreadFunctionAttribute () : base ("sanitize_thread") {} } - class SanitizeHwaddressFunctionAttribute : LLVMFlagFunctionAttribute + class SanitizeHwaddressFunctionAttribute : LlvmIrFlagFunctionAttribute { public SanitizeHwaddressFunctionAttribute () : base ("sanitize_hwaddress") {} } - class SanitizeMemtagFunctionAttribute : LLVMFlagFunctionAttribute + class SanitizeMemtagFunctionAttribute : LlvmIrFlagFunctionAttribute { public SanitizeMemtagFunctionAttribute () : base ("sanitize_memtag") {} } - class SpeculativeLoadHardeningFunctionAttribute : LLVMFlagFunctionAttribute + class SpeculativeLoadHardeningFunctionAttribute : LlvmIrFlagFunctionAttribute { public SpeculativeLoadHardeningFunctionAttribute () : base ("speculative_load_hardening") {} } - class SpeculatableFunctionAttribute : LLVMFlagFunctionAttribute + class SpeculatableFunctionAttribute : LlvmIrFlagFunctionAttribute { public SpeculatableFunctionAttribute () : base ("speculatable") {} } - class SspFunctionAttribute : LLVMFlagFunctionAttribute + class SspFunctionAttribute : LlvmIrFlagFunctionAttribute { public SspFunctionAttribute () : base ("ssp") {} } - class SspstrongFunctionAttribute : LLVMFlagFunctionAttribute + class SspstrongFunctionAttribute : LlvmIrFlagFunctionAttribute { public SspstrongFunctionAttribute () : base ("sspstrong") {} } - class SspreqFunctionAttribute : LLVMFlagFunctionAttribute + class SspreqFunctionAttribute : LlvmIrFlagFunctionAttribute { public SspreqFunctionAttribute () : base ("sspreq") {} } - class StrictfpFunctionAttribute : LLVMFlagFunctionAttribute + class StrictfpFunctionAttribute : LlvmIrFlagFunctionAttribute { public StrictfpFunctionAttribute () : base ("strictfp") {} } - class DenormalFpMathFunctionAttribute : LLVMFlagFunctionAttribute + class DenormalFpMathFunctionAttribute : LlvmIrFlagFunctionAttribute { public DenormalFpMathFunctionAttribute () : base ("denormal-fp-math", quoted: true) {} } - class DenormalFpMathF32FunctionAttribute : LLVMFlagFunctionAttribute + class DenormalFpMathF32FunctionAttribute : LlvmIrFlagFunctionAttribute { public DenormalFpMathF32FunctionAttribute () : base ("denormal-fp-math-f32", quoted: true) {} } - class ThunkFunctionAttribute : LLVMFlagFunctionAttribute + class ThunkFunctionAttribute : LlvmIrFlagFunctionAttribute { public ThunkFunctionAttribute () : base ("thunk", quoted: true) {} } - class TlsLoadHoistFunctionAttribute : LLVMFlagFunctionAttribute + class TlsLoadHoistFunctionAttribute : LlvmIrFlagFunctionAttribute { public TlsLoadHoistFunctionAttribute () : base ("tls-load-hoist") {} } - class UwtableFunctionAttribute : LLVMFunctionAttribute + class UwtableFunctionAttribute : LlvmIrFunctionAttribute { bool? isSync; @@ -652,30 +912,49 @@ protected override void RenderParams (StringBuilder sb) sb.Append (isSync.Value ? "sync" : "async"); } + + public override bool Equals (LlvmIrFunctionAttribute other) + { + if (!base.Equals (other)) { + return false; + } + + var attr = other as UwtableFunctionAttribute; + if (attr == null) { + return false; + } + + return isSync == attr.isSync; + } + + public override int GetHashCode () + { + return base.GetHashCode () ^ (isSync?.GetHashCode () ?? 0); + } } - class NocfCheckFunctionAttribute : LLVMFlagFunctionAttribute + class NocfCheckFunctionAttribute : LlvmIrFlagFunctionAttribute { public NocfCheckFunctionAttribute () : base ("nocf_check") {} } - class ShadowcallstackFunctionAttribute : LLVMFlagFunctionAttribute + class ShadowcallstackFunctionAttribute : LlvmIrFlagFunctionAttribute { public ShadowcallstackFunctionAttribute () : base ("shadowcallstack") {} } - class MustprogressFunctionAttribute : LLVMFlagFunctionAttribute + class MustprogressFunctionAttribute : LlvmIrFlagFunctionAttribute { public MustprogressFunctionAttribute () : base ("mustprogress") {} } - class WarnStackSizeFunctionAttribute : LLVMFunctionAttribute + class WarnStackSizeFunctionAttribute : LlvmIrFunctionAttribute { uint threshold; @@ -686,9 +965,28 @@ public WarnStackSizeFunctionAttribute (uint threshold) } protected override void RenderAssignedValue (StringBuilder sb) => sb.Append (threshold); + + public override bool Equals (LlvmIrFunctionAttribute other) + { + if (!base.Equals (other)) { + return false; + } + + var attr = other as WarnStackSizeFunctionAttribute; + if (attr == null) { + return false; + } + + return threshold == attr.threshold; + } + + public override int GetHashCode () + { + return base.GetHashCode () ^ threshold.GetHashCode (); + } } - class VscaleRangeFunctionAttribute : LLVMFunctionAttribute + class VscaleRangeFunctionAttribute : LlvmIrFunctionAttribute { uint min; uint? max; @@ -710,9 +1008,28 @@ protected override void RenderParams (StringBuilder sb) sb.Append (", "); sb.Append (max.Value.ToString (CultureInfo.InvariantCulture)); } + + public override bool Equals (LlvmIrFunctionAttribute other) + { + if (!base.Equals (other)) { + return false; + } + + var attr = other as VscaleRangeFunctionAttribute; + if (attr == null) { + return false; + } + + return min == attr.min && max == attr.max; + } + + public override int GetHashCode () + { + return base.GetHashCode () ^ min.GetHashCode () ^ (max?.GetHashCode () ?? 0); + } } - class MinLegalVectorWidthFunctionAttribute : LLVMFunctionAttribute + class MinLegalVectorWidthFunctionAttribute : LlvmIrFunctionAttribute { uint size; @@ -723,9 +1040,28 @@ public MinLegalVectorWidthFunctionAttribute (uint size) } protected override void RenderAssignedValue (StringBuilder sb) => sb.Append (size.ToString (CultureInfo.InvariantCulture)); + + public override bool Equals (LlvmIrFunctionAttribute other) + { + if (!base.Equals (other)) { + return false; + } + + var attr = other as MinLegalVectorWidthFunctionAttribute; + if (attr == null) { + return false; + } + + return size == attr.size; + } + + public override int GetHashCode () + { + return base.GetHashCode () ^ size.GetHashCode (); + } } - class StackProtectorBufferSizeFunctionAttribute : LLVMFunctionAttribute + class StackProtectorBufferSizeFunctionAttribute : LlvmIrFunctionAttribute { uint size; @@ -736,9 +1072,28 @@ public StackProtectorBufferSizeFunctionAttribute (uint size) } protected override void RenderAssignedValue (StringBuilder sb) => sb.Append (size.ToString (CultureInfo.InvariantCulture)); + + public override bool Equals (LlvmIrFunctionAttribute other) + { + if (!base.Equals (other)) { + return false; + } + + var attr = other as StackProtectorBufferSizeFunctionAttribute; + if (attr == null) { + return false; + } + + return size == attr.size; + } + + public override int GetHashCode () + { + return base.GetHashCode () ^ size.GetHashCode (); + } } - class TargetCpuFunctionAttribute : LLVMFunctionAttribute + class TargetCpuFunctionAttribute : LlvmIrFunctionAttribute { string cpu; @@ -749,9 +1104,28 @@ public TargetCpuFunctionAttribute (string cpu) } protected override void RenderAssignedValue (StringBuilder sb) => sb.Append (cpu); + + public override bool Equals (LlvmIrFunctionAttribute other) + { + if (!base.Equals (other)) { + return false; + } + + var attr = other as TargetCpuFunctionAttribute; + if (attr == null) { + return false; + } + + return String.Compare (cpu, attr.cpu, StringComparison.Ordinal) == 0; + } + + public override int GetHashCode () + { + return base.GetHashCode () ^ (cpu?.GetHashCode () ?? 0); + } } - class TuneCpuFunctionAttribute : LLVMFunctionAttribute + class TuneCpuFunctionAttribute : LlvmIrFunctionAttribute { string cpu; @@ -762,9 +1136,28 @@ public TuneCpuFunctionAttribute (string cpu) } protected override void RenderAssignedValue (StringBuilder sb) => sb.Append (cpu); + + public override bool Equals (LlvmIrFunctionAttribute other) + { + if (!base.Equals (other)) { + return false; + } + + var attr = other as TuneCpuFunctionAttribute; + if (attr == null) { + return false; + } + + return String.Compare (cpu, attr.cpu, StringComparison.Ordinal) == 0; + } + + public override int GetHashCode () + { + return base.GetHashCode () ^ (cpu?.GetHashCode () ?? 0); + } } - class TargetFeaturesFunctionAttribute : LLVMFunctionAttribute + class TargetFeaturesFunctionAttribute : LlvmIrFunctionAttribute { string features; @@ -775,9 +1168,28 @@ public TargetFeaturesFunctionAttribute (string features) } protected override void RenderAssignedValue (StringBuilder sb) => sb.Append (features); + + public override bool Equals (LlvmIrFunctionAttribute other) + { + if (!base.Equals (other)) { + return false; + } + + var attr = other as TargetFeaturesFunctionAttribute; + if (attr == null) { + return false; + } + + return String.Compare (features, attr.features, StringComparison.Ordinal) == 0; + } + + public override int GetHashCode () + { + return base.GetHashCode () ^ (features?.GetHashCode () ?? 0); + } } - class NoTrappingMathFunctionAttribute : LLVMFunctionAttribute + class NoTrappingMathFunctionAttribute : LlvmIrFunctionAttribute { bool yesno; @@ -788,9 +1200,28 @@ public NoTrappingMathFunctionAttribute (bool yesno) } protected override void RenderAssignedValue (StringBuilder sb) => sb.Append (yesno.ToString ().ToLowerInvariant ()); + + public override bool Equals (LlvmIrFunctionAttribute other) + { + if (!base.Equals (other)) { + return false; + } + + var attr = other as NoTrappingMathFunctionAttribute; + if (attr == null) { + return false; + } + + return yesno == attr.yesno; + } + + public override int GetHashCode () + { + return base.GetHashCode () ^ yesno.GetHashCode (); + } } - class StackrealignFunctionAttribute : LLVMFlagFunctionAttribute + class StackrealignFunctionAttribute : LlvmIrFlagFunctionAttribute { public StackrealignFunctionAttribute () : base ("stackrealign", quoted: true) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/IStructureInfo.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/IStructureInfo.cs deleted file mode 100644 index afd17fefda5..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/IStructureInfo.cs +++ /dev/null @@ -1,15 +0,0 @@ -using System; - -namespace Xamarin.Android.Tasks.LLVMIR -{ - interface IStructureInfo - { - Type Type { get; } - ulong Size { get; } - int MaxFieldAlignment { get; } - string Name { get; } - string NativeTypeDesignator { get; } - - void RenderDeclaration (LlvmIrGenerator generator); - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmFunctionAttributeSet.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmFunctionAttributeSet.cs deleted file mode 100644 index 4ba4ed9be75..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmFunctionAttributeSet.cs +++ /dev/null @@ -1,50 +0,0 @@ -using System; -using System.Collections; -using System.Collections.Generic; -using System.Linq; - -namespace Xamarin.Android.Tasks.LLVMIR -{ - class LlvmFunctionAttributeSet : IEnumerable - { - HashSet attributes; - - public LlvmFunctionAttributeSet () - { - attributes = new HashSet (); - } - - public void Add (LLVMFunctionAttribute attr) - { - if (attr == null) { - throw new ArgumentNullException (nameof (attr)); - } - - // TODO: implement uniqueness checks - attributes.Add (attr); - } - - public void Add (LlvmFunctionAttributeSet sourceSet) - { - if (sourceSet == null) { - throw new ArgumentNullException (nameof (sourceSet)); - } - - foreach (LLVMFunctionAttribute attr in sourceSet) { - Add (attr); - } - } - - public string Render () - { - List list = attributes.ToList (); - list.Sort ((LLVMFunctionAttribute a, LLVMFunctionAttribute b) => a.Name.CompareTo (b.Name)); - - return String.Join (" ", list.Select (a => a.Render ())); - } - - public IEnumerator GetEnumerator () => attributes.GetEnumerator (); - - IEnumerator IEnumerable.GetEnumerator () => GetEnumerator (); - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrBufferManager.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrBufferManager.cs new file mode 100644 index 00000000000..8cb8c2910e4 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrBufferManager.cs @@ -0,0 +1,67 @@ +using System; +using System.Collections.Generic; + +namespace Xamarin.Android.Tasks.LLVMIR; + +partial class LlvmIrModule +{ + sealed class LlvmIrBufferManager + { + Dictionary counters; + Dictionary> bufferVariableNames; + + public LlvmIrBufferManager () + { + counters = new Dictionary (StringComparer.Ordinal); + } + + public string Allocate (StructureInstance structure, StructureMemberInfo smi, ulong size) + { + string baseName = $"_{structure.Info.Name}_{smi.Info.Name}"; + + if (!counters.TryGetValue (baseName, out ulong count)) { + count = 0; + counters.Add (baseName, count); + } else { + count++; + counters[baseName] = count; + } + + return Register (structure, smi, $"{baseName}_{count:x}_{structure.IndexInArray:x}"); + } + + public string? GetBufferVariableName (StructureInstance structure, StructureMemberInfo smi) + { + if (bufferVariableNames == null || bufferVariableNames.Count == 0) { + return null; + } + + if (!bufferVariableNames.TryGetValue (structure.Obj, out Dictionary members)) { + return null; + } + + if (!members.TryGetValue (MakeUniqueMemberId (structure, smi), out string bufferVariableName)) { + return null; + } + + return bufferVariableName; + } + + string Register (StructureInstance structure, StructureMemberInfo smi, string bufferVariableName) + { + if (bufferVariableNames == null) { + bufferVariableNames = new Dictionary> (); + } + + if (!bufferVariableNames.TryGetValue (structure.Obj, out Dictionary members)) { + members = new Dictionary (StringComparer.Ordinal); + bufferVariableNames.Add (structure.Obj, members); + } + + members.Add (MakeUniqueMemberId (structure, smi), bufferVariableName); + return bufferVariableName; + } + + string MakeUniqueMemberId (StructureInstance structure, StructureMemberInfo smi) => $"{smi.Info.Name}_{structure.IndexInArray}"; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs index 374a432e76a..8db94269f32 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs @@ -7,74 +7,51 @@ namespace Xamarin.Android.Tasks.LLVMIR { - /// - /// Base class for all classes which "compose" LLVM IR assembly. - /// abstract class LlvmIrComposer { - protected AndroidTargetArch TargetArch { get; } + bool constructed; - protected LlvmIrComposer () - {} + protected abstract void Construct (LlvmIrModule module); - public void Write (AndroidTargetArch arch, StreamWriter output, string fileName) + public LlvmIrModule Construct () { - LlvmIrGenerator generator = LlvmIrGenerator.Create (arch, output, fileName); + var module = new LlvmIrModule (); + Construct (module); + module.AfterConstruction (); + constructed = true; - InitGenerator (generator); - MapStructures (generator); - generator.WriteFileTop (); - generator.WriteStructureDeclarations (); - Write (generator); - generator.WriteFileEnd (); + return module; } - protected static string GetAbiName (AndroidTargetArch arch) + public void Generate (LlvmIrModule module, AndroidTargetArch arch, StreamWriter output, string fileName) { - return arch switch { - AndroidTargetArch.Arm => "armeabi-v7a", - AndroidTargetArch.Arm64 => "arm64-v8a", - AndroidTargetArch.X86 => "x86", - AndroidTargetArch.X86_64 => "x86_64", - _ => throw new InvalidOperationException ($"Unsupported Android architecture: {arch}"), - }; + if (!constructed) { + throw new InvalidOperationException ($"Internal error: module not constructed yet. Was Constrict () called?"); + } + + LlvmIrGenerator generator = LlvmIrGenerator.Create (arch, fileName); + generator.Generate (output, module); + output.Flush (); } - protected ulong HashName (string name, bool is64Bit) + public static ulong GetXxHash (string str, bool is64Bit) { - byte[] nameBytes = Encoding.UTF8.GetBytes (name); + byte[] stringBytes = Encoding.UTF8.GetBytes (str); if (is64Bit) { - return XxHash64.HashToUInt64 (nameBytes); + return XxHash64.HashToUInt64 (stringBytes); } - return (ulong)XxHash32.HashToUInt32 (nameBytes); + return (ulong)XxHash32.HashToUInt32 (stringBytes); } - protected virtual void InitGenerator (LlvmIrGenerator generator) - {} - - /// - /// Initialize the composer. It needs to allocate and populate all the structures that - /// are used by the composer, before they can be mapped by the generator. The code here - /// should initialize only the architecture-independent fields of structures etc to - /// write. The composer is reused between architectures, and only the Write method is - /// aware of which architecture is targetted. - /// - public abstract void Init (); - - /// - /// Maps all the structures used to internal LLVM IR representation. Every structure MUST - /// be mapped. - /// - protected abstract void MapStructures (LlvmIrGenerator generator); + protected LlvmIrGlobalVariable EnsureGlobalVariable (LlvmIrVariable variable) + { + var gv = variable as LlvmIrGlobalVariable; + if (gv == null) { + throw new InvalidOperationException ("Internal error: global variable expected"); + } - /// - /// Generate LLVM IR code from data structures initialized by . This is - /// called once per ABI, with the appropriate for the target - /// ABI. If any ABI-specific initialization must be performed on the data structures to - /// be written, it has to be done here (applies to e.g. constructs that require to know the - /// native pointer size). - /// - protected abstract void Write (LlvmIrGenerator generator); + return gv; + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrDataLayout.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrDataLayout.cs new file mode 100644 index 00000000000..33da1be9a57 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrDataLayout.cs @@ -0,0 +1,353 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text; + +namespace Xamarin.Android.Tasks.LLVMIR; + +abstract class LlvmIrDataLayoutField +{ + public const char Separator = ':'; + + public string Id { get; } + + protected LlvmIrDataLayoutField (string id) + { + if (String.IsNullOrEmpty (id)) { + throw new ArgumentException (nameof (id), "must not be null or empty"); + } + + Id = id; + } + + public virtual void Render (StringBuilder sb) + { + sb.Append (Id); + } + + public static string ConvertToString (uint v) + { + return v.ToString (CultureInfo.InvariantCulture); + } + + protected void Append (StringBuilder sb, uint v, bool needSeparator = true) + { + if (needSeparator) { + sb.Append (Separator); + } + + sb.Append (ConvertToString (v)); + } + + protected void Append (StringBuilder sb, uint? v, bool needSeparator = true) + { + if (!v.HasValue) { + return; + } + + Append (sb, v.Value, needSeparator); + } +} + +class LlvmIrDataLayoutPointerSize : LlvmIrDataLayoutField +{ + public uint? AddressSpace { get; set; } + public uint Abi { get; } + public uint Size { get; } + public uint? Pref { get; set; } + public uint? Idx { get; set; } + + public LlvmIrDataLayoutPointerSize (uint size, uint abi) + : base ("p") + { + Size = size; + Abi = abi; + } + + public override void Render (StringBuilder sb) + { + base.Render (sb); + + if (AddressSpace.HasValue && AddressSpace.Value > 0) { + Append (sb, AddressSpace.Value, needSeparator: false); + } + Append (sb, Size); + Append (sb, Abi); + Append (sb, Pref); + Append (sb, Idx); + } +} + +abstract class LlvmIrDataLayoutTypeAlignment : LlvmIrDataLayoutField +{ + public uint Size { get; } + public uint Abi { get; } + public uint? Pref { get; set; } + + protected LlvmIrDataLayoutTypeAlignment (string id, uint size, uint abi) + : base (id) + { + Size = size; + Abi = abi; + } + + public override void Render (StringBuilder sb) + { + base.Render (sb); + + Append (sb, Size, needSeparator: false); + Append (sb, Abi); + Append (sb, Pref); + } +} + +class LlvmIrDataLayoutIntegerAlignment : LlvmIrDataLayoutTypeAlignment +{ + public LlvmIrDataLayoutIntegerAlignment (uint size, uint abi, uint? pref = null) + : base ("i", size, abi) + { + if (size == 8 && abi != 8) { + throw new ArgumentOutOfRangeException (nameof (abi), "Must equal 8 for i8"); + } + + Pref = pref; + } +} + +class LlvmIrDataLayoutVectorAlignment : LlvmIrDataLayoutTypeAlignment +{ + public LlvmIrDataLayoutVectorAlignment (uint size, uint abi, uint? pref = null) + : base ("v", size, abi) + { + Pref = pref; + } +} + +class LlvmIrDataLayoutFloatAlignment : LlvmIrDataLayoutTypeAlignment +{ + public LlvmIrDataLayoutFloatAlignment (uint size, uint abi, uint? pref = null) + : base ("f", size, abi) + { + Pref = pref; + } +} + +class LlvmIrDataLayoutAggregateObjectAlignment : LlvmIrDataLayoutField +{ + public uint Abi { get; } + public uint? Pref { get; set; } + + public LlvmIrDataLayoutAggregateObjectAlignment (uint abi, uint? pref = null) + : base ("a") + { + Abi = abi; + Pref = pref; + } + + public override void Render (StringBuilder sb) + { + base.Render (sb); + + Append (sb, Abi); + Append (sb, Pref); + } +} + +enum LlvmIrDataLayoutFunctionPointerAlignmentType +{ + Independent, + Multiple, +} + +class LlvmIrDataLayoutFunctionPointerAlignment : LlvmIrDataLayoutField +{ + public uint Abi { get; } + public LlvmIrDataLayoutFunctionPointerAlignmentType Type { get; } + + public LlvmIrDataLayoutFunctionPointerAlignment (LlvmIrDataLayoutFunctionPointerAlignmentType type, uint abi) + : base ("F") + { + Type = type; + Abi = abi; + } + + public override void Render (StringBuilder sb) + { + base.Render (sb); + + char type = Type switch { + LlvmIrDataLayoutFunctionPointerAlignmentType.Independent => 'i', + LlvmIrDataLayoutFunctionPointerAlignmentType.Multiple => 'n', + _ => throw new InvalidOperationException ($"Unsupported function pointer alignment type '{Type}'") + }; + sb.Append (type); + Append (sb, Abi, needSeparator: false); + } +} + +enum LlvmIrDataLayoutManglingOption +{ + ELF, + GOFF, + MIPS, + MachO, + WindowsX86COFF, + WindowsCOFF, + XCOFF +} + +class LlvmIrDataLayoutMangling : LlvmIrDataLayoutField +{ + public LlvmIrDataLayoutManglingOption Option { get; } + + public LlvmIrDataLayoutMangling (LlvmIrDataLayoutManglingOption option) + : base ("m") + { + Option = option; + } + + public override void Render (StringBuilder sb) + { + base.Render (sb); + + sb.Append (Separator); + + char opt = Option switch { + LlvmIrDataLayoutManglingOption.ELF => 'e', + LlvmIrDataLayoutManglingOption.GOFF => 'l', + LlvmIrDataLayoutManglingOption.MIPS => 'm', + LlvmIrDataLayoutManglingOption.MachO => 'o', + LlvmIrDataLayoutManglingOption.WindowsX86COFF => 'x', + LlvmIrDataLayoutManglingOption.WindowsCOFF => 'w', + LlvmIrDataLayoutManglingOption.XCOFF => 'a', + _ => throw new InvalidOperationException ($"Unsupported mangling option '{Option}'") + }; + + sb.Append (opt); + } +} + +// See: https://llvm.org/docs/LangRef.html#data-layout +class LlvmIrDataLayout +{ + bool bigEndian; + bool littleEndian = true; + + public bool BigEndian { + get => bigEndian; + set { + bigEndian = value; + littleEndian = !bigEndian; + } + } + + public bool LittleEndian { + get => littleEndian; + set { + littleEndian = value; + bigEndian = !littleEndian; + } + } + + public uint? AllocaAddressSpaceId { get; set; } + public uint? GlobalsAddressSpaceId { get; set; } + public LlvmIrDataLayoutMangling? Mangling { get; set; } + public uint? ProgramAddressSpaceId { get; set; } + public uint? StackAlignment { get; set; } + + public LlvmIrDataLayoutAggregateObjectAlignment? AggregateObjectAlignment { get; set; } + public List? FloatAlignment { get; set; } + public LlvmIrDataLayoutFunctionPointerAlignment? FunctionPointerAlignment { get; set; } + public List? IntegerAlignment { get; set; } + public List? VectorAlignment { get; set; } + public List? PointerSize { get; set; } + + public List? NativeIntegerWidths { get; set; } + public List? NonIntegralPointerTypeAddressSpaces { get; set; } + + public string Render () + { + var sb = new StringBuilder (); + + sb.Append ("target datalayout = \""); + + sb.Append (LittleEndian ? 'e' : 'E'); + + if (Mangling != null) { + sb.Append ('-'); + Mangling.Render (sb); + } + + AppendFieldList (PointerSize); + + if (FunctionPointerAlignment != null) { + sb.Append ('-'); + FunctionPointerAlignment.Render (sb); + } + + AppendFieldList (IntegerAlignment); + AppendFieldList (FloatAlignment); + AppendFieldList (VectorAlignment); + + Append ('P', ProgramAddressSpaceId); + Append ('G', GlobalsAddressSpaceId); + Append ('A', AllocaAddressSpaceId); + + if (AggregateObjectAlignment != null) { + sb.Append ('-'); + AggregateObjectAlignment.Render (sb); + } + + AppendList ("n", NativeIntegerWidths); + AppendList ("ni", NonIntegralPointerTypeAddressSpaces); + Append ('S', StackAlignment); + + sb.Append ('"'); + + return sb.ToString (); + + void AppendFieldList (List? list) where T: LlvmIrDataLayoutField + { + if (list == null || list.Count == 0) { + return; + } + + foreach (LlvmIrDataLayoutField field in list) { + sb.Append ('-'); + field.Render (sb); + } + } + + void AppendList (string id, List? list) + { + if (list == null || list.Count == 0) { + return; + } + + sb.Append ('-'); + sb.Append (id); + + bool first = true; + foreach (uint v in list) { + if (first) { + first = false; + } else { + sb.Append (LlvmIrDataLayoutField.Separator); + } + + sb.Append (LlvmIrDataLayoutField.ConvertToString (v)); + } + } + + void Append (char id, uint? v) + { + if (!v.HasValue) { + return; + } + + sb.Append ('-'); + sb.Append (id); + sb.Append (LlvmIrDataLayoutField.ConvertToString (v.Value)); + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunction.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunction.cs index f2f9ba989ec..d65a8fef5a6 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunction.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunction.cs @@ -1,188 +1,500 @@ using System; using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using System.Text; namespace Xamarin.Android.Tasks.LLVMIR { - class LlvmIrFunctionLocalVariable : LlvmIrVariable + interface ILlvmIrSavedFunctionParameterState {} + + class LlvmIrFunctionParameter : LlvmIrLocalVariable { - public LlvmIrFunctionLocalVariable (Type type, string? name = null, bool isNativePointer = false) - : base (type, name, signature: null, isNativePointer: isNativePointer) - {} + sealed class SavedParameterState : ILlvmIrSavedFunctionParameterState + { + public readonly LlvmIrFunctionParameter Owner; + + public uint? Align; + public bool? AllocPtr; + public uint? Dereferenceable; + public bool? ImmArg; + public bool? NoCapture; + public bool? NonNull; + public bool? NoUndef; + public bool? ReadNone; + public bool? SignExt; + public bool? ZeroExt; + public bool? IsCplusPlusReference; + + public SavedParameterState (LlvmIrFunctionParameter owner, SavedParameterState? previousState = null) + { + Owner = owner; + if (previousState == null) { + return; + } + + Align = previousState.Align; + AllocPtr = previousState.AllocPtr; + Dereferenceable = previousState.Dereferenceable; + ImmArg = previousState.ImmArg; + NoCapture = previousState.NoCapture; + NonNull = previousState.NonNull; + ReadNone = previousState.ReadNone; + SignExt = previousState.SignExt; + ZeroExt = previousState.ZeroExt; + } + } + + SavedParameterState state; - public LlvmIrFunctionLocalVariable (LlvmNativeFunctionSignature nativeFunction, string? name = null, bool isNativePointer = false) - : base (typeof(LlvmNativeFunctionSignature), name, nativeFunction, isNativePointer: isNativePointer) + // To save on time, we declare only attributes that are actually used in our generated code. More will be added, as needed. + + /// + /// align(n) attribute, see . + /// As a special case for us, a value of 0 means use the natural target pointer alignment. + /// + public uint? Align { + get => state.Align; + set => state.Align = value; + } + + /// + /// allocptr attribute, see + /// + public bool? AllocPtr { + get => state.AllocPtr; + set => state.AllocPtr = value; + } + + /// + /// dereferenceable(n) attribute, see . + /// As a special case for us, a value of 0 means use the natural target pointer alignment. + /// + public uint? Dereferenceable { + get => state.Dereferenceable; + set => state.Dereferenceable = value; + } + + /// + /// immarg attribute, see + /// + public bool? ImmArg { + get => state.ImmArg; + set => state.ImmArg = value; + } + + /// + /// nocapture attribute, see + /// + public bool? NoCapture { + get => state.NoCapture; + set => state.NoCapture = value; + } + + /// + /// nonnull attribute, see + /// + public bool? NonNull { + get => state.NonNull; + set => state.NonNull = value; + } + + /// + /// noundef attribute, see + /// + public bool? NoUndef { + get => state.NoUndef; + set => state.NoUndef = value; + } + + /// + /// readnone attribute, see + /// + public bool? ReadNone { + get => state.ReadNone; + set => state.ReadNone = value; + } + + /// + /// signext attribute, see + /// + public bool? SignExt { + get => state.SignExt; + set => state.SignExt = value; + } + + /// + /// zeroext attribute, see + /// + public bool? ZeroExt { + get => state.ZeroExt; + set => state.ZeroExt = value; + } + + /// + /// This serves a purely documentational purpose, when generating comments about types. It describes a parameter that is a C++ reference, something we can't + /// reflect on the managed side. + /// + public bool? IsCplusPlusReference { + get => state.IsCplusPlusReference; + set => state.IsCplusPlusReference = value; + } + + public LlvmIrFunctionParameter (Type type, string? name = null) + : base (type, name) { - if (nativeFunction == null) { - throw new ArgumentNullException(nameof (nativeFunction)); + NameMatters = false; + state = new SavedParameterState (this); + + // TODO: check why it doesn't work as expected - can't see the flags set in the output + if (type == typeof(sbyte) || type == typeof (short)) { + SignExt = true; + } else if (type == typeof(byte) || type == typeof (ushort)) { + ZeroExt = true; } } - public LlvmIrFunctionLocalVariable (LlvmIrVariable variable, string? name = null, bool isNativePointer = false) - : base (variable, name, isNativePointer) - {} + /// + /// Save (opaque) parameter state. This is necessary because we generate code from the same model (module) for different + /// targets. At the same time, function, signature and parameter instances are shared between the different code generation + /// sessions, so we must sure the state as set by the model is properly preserved. NOTE: it does NOT make the code thread-safe! + /// Instances are **still** shared and thus different threads would step on each other's toes should they saved and restored + /// state without synchronization. + /// + public ILlvmIrSavedFunctionParameterState SaveState () + { + SavedParameterState ret = state; + state = new SavedParameterState (this, ret); + return ret; + } + + /// + /// Restore (opaque) state. for more info + /// + public void RestoreState (ILlvmIrSavedFunctionParameterState savedState) + { + var oldState = savedState as SavedParameterState; + if (oldState == null) { + throw new InvalidOperationException ("Internal error: savedState not an instance of ParameterState"); + } + + if (oldState.Owner != this) { + throw new InvalidOperationException ("Internal error: savedState not saved by this instance"); + } + + state = oldState; + } } - class LlvmIrFunctionParameter : LlvmIrFunctionLocalVariable + interface ILlvmIrSavedFunctionSignatureState {} + + class LlvmIrFunctionSignature : IEquatable { - public bool IsCplusPlusReference { get; } + public sealed class ReturnTypeAttributes + { + public bool? InReg; + public bool? NoUndef; + public bool? SignExt; + public bool? ZeroExt; + + public ReturnTypeAttributes () + {} + + public ReturnTypeAttributes (ReturnTypeAttributes other) + { + InReg = other.InReg; + NoUndef = other.NoUndef; + SignExt = other.SignExt; + ZeroExt = other.ZeroExt; + } + } - public LlvmIrFunctionParameter (Type type, string? name = null, bool isNativePointer = false, bool isCplusPlusReference = false) - : base (type, name, isNativePointer) + sealed class SavedSignatureState : ILlvmIrSavedFunctionSignatureState { - IsCplusPlusReference = isCplusPlusReference; + public readonly LlvmIrFunctionSignature Owner; + public readonly IList ParameterStates; + public readonly ReturnTypeAttributes ReturnAttributes; + + public SavedSignatureState (LlvmIrFunctionSignature owner, IList parameterStates, ReturnTypeAttributes returnAttributes) + { + Owner = owner; + ParameterStates = parameterStates; + ReturnAttributes = returnAttributes; + } } - public LlvmIrFunctionParameter (LlvmNativeFunctionSignature nativeFunction, string? name = null, bool isNativePointer = false, bool isCplusPlusReference = false) - : base (nativeFunction, name, isNativePointer) + ReturnTypeAttributes returnAttributes; + + public string Name { get; } + public Type ReturnType { get; } + public ReturnTypeAttributes ReturnAttributes => returnAttributes; + public IList Parameters { get; } + + public LlvmIrFunctionSignature (string name, Type returnType, IList? parameters = null, ReturnTypeAttributes? returnAttributes = null) { - IsCplusPlusReference = isCplusPlusReference; + if (String.IsNullOrEmpty (name)) { + throw new ArgumentException ("must not be null or empty", nameof (name)); + } + + Name = name; + ReturnType = returnType; + this.returnAttributes = returnAttributes ?? new ReturnTypeAttributes (); + Parameters = parameters ?? new List (); } - } - class LlvmIrFunctionArgument - { - public object Value { get; } - public Type Type { get; } + /// + /// Create new signature using data from the one, with the exception of name. + /// Useful when there are several functions with different names but identical parameters and return types. + /// + public LlvmIrFunctionSignature (string name, LlvmIrFunctionSignature templateSignature) + : this (name, templateSignature.ReturnType, templateSignature.Parameters) + {} - public LlvmIrFunctionArgument (Type type, object? value = null) + /// + /// Save (opaque) signature state. This includes states of all the parameters. + /// for more information. + /// + public ILlvmIrSavedFunctionSignatureState SaveState () { - Type = type ?? throw new ArgumentNullException (nameof (type)); + var list = new List (); - if (value != null && value.GetType () != type) { - throw new ArgumentException ($"value type '{value.GetType ()}' does not match the argument type '{type}'"); + foreach (LlvmIrFunctionParameter parameter in Parameters) { + list.Add (parameter.SaveState ()); } - Value = value; + var ret = new SavedSignatureState (this, list.AsReadOnly (), returnAttributes); + returnAttributes = new ReturnTypeAttributes (returnAttributes); + return ret; } - public LlvmIrFunctionArgument (LlvmIrFunctionLocalVariable variable) + /// + /// Restore (opaque) signature state. This includes states of all the parameters. + /// for more information. + /// + public void RestoreState (ILlvmIrSavedFunctionSignatureState savedState) { - Type = typeof(LlvmIrFunctionLocalVariable); - Value = variable; + var oldState = savedState as SavedSignatureState; + if (oldState == null) { + throw new InvalidOperationException ($"Internal error: savedState not an instance of {nameof(SavedSignatureState)}"); + } + + if (oldState.Owner != this) { + throw new InvalidOperationException ("Internal error: savedState not saved by this instance"); + } + + for (int i = 0; i < oldState.ParameterStates.Count; i++) { + ILlvmIrSavedFunctionParameterState parameterState = oldState.ParameterStates[i]; + Parameters[i].RestoreState (parameterState); + } + returnAttributes = new ReturnTypeAttributes (oldState.ReturnAttributes); + } + + public override int GetHashCode () + { + int hc = + Name.GetHashCode () ^ + Parameters.GetHashCode () ^ + ReturnType.GetHashCode (); + + foreach (LlvmIrFunctionParameter p in Parameters) { + hc ^= p.GetHashCode (); + } + + return hc; + } + + public override bool Equals (object obj) + { + var sig = obj as LlvmIrFunctionSignature; + if (sig == null) { + return false; + } + + return Equals (sig); + } + + public bool Equals (LlvmIrFunctionSignature other) + { + if (other == null) { + return false; + } + + if (Parameters.Count != other.Parameters.Count || + ReturnType != other.ReturnType || + String.Compare (Name, other.Name, StringComparison.Ordinal) != 0 + ) { + return false; + } + + for (int i = 0; i < Parameters.Count; i++) { + if (Parameters[i] != other.Parameters[i]) { + return false; + } + } + + return true; } } + interface ILlvmIrSavedFunctionState {} + /// - /// Describes a native function to be emitted and keeps code emitting state between calls to various generator + /// Describes a native function to be emitted or declared and keeps code emitting state between calls to various generator. /// methods. /// - class LlvmIrFunction + class LlvmIrFunction : IEquatable { - const string Indent1 = LlvmIrGenerator.Indent; - const string Indent2 = LlvmIrGenerator.Indent + LlvmIrGenerator.Indent; - - // Function signature - public string Name { get; } - public Type ReturnType { get; } - public int AttributeSetID { get; } - public IList? Parameters { get; } - public string ImplicitFuncTopLabel { get; } - public IList? ParameterVariables { get; } + public class FunctionState + { + // Counter shared by unnamed local variables (including function parameters) and unnamed labels. + ulong unnamedTemporaryCounter = 0; - // Function writing state - public string Indent { get; private set; } = LlvmIrGenerator.Indent; + // Implicit unnamed label at the start of the function + ulong? startingBlockNumber; - // Used for unnamed function parameters as well as unnamed local variables - uint localSlot = 0; - uint indentLevel = 1; + public ulong StartingBlockNumber { + get { + if (startingBlockNumber.HasValue) { + return startingBlockNumber.Value; + } - public LlvmIrFunction (string name, Type returnType, int attributeSetID, List? parameters = null) - { - if (String.IsNullOrEmpty (name)) { - throw new ArgumentException ("must not be null or empty", nameof (name)); + throw new InvalidOperationException ($"Internal error: starting block number not set"); + } } - Name = name; - ReturnType = returnType ?? throw new ArgumentNullException (nameof (returnType)); - AttributeSetID = attributeSetID; - Parameters = parameters?.Select (p => EnsureParameterName (p))?.ToList ()?.AsReadOnly (); - ParameterVariables = Parameters?.Select (p => new LlvmIrFunctionLocalVariable (p.Type, p.Name))?.ToList ()?.AsReadOnly (); - // Unnamed local variables need to start from the value which equals [number_of_unnamed_parameters] + 1, - // since there's an implicit label created for the top of the function whose name is `[number_of_unnamed_parameters]` - ImplicitFuncTopLabel = localSlot.ToString (CultureInfo.InvariantCulture); - localSlot++; + public FunctionState () + {} - LlvmIrFunctionParameter EnsureParameterName (LlvmIrFunctionParameter parameter) + public ulong NextTemporary () { - if (parameter == null) { - throw new InvalidOperationException ("null parameters aren't allowed"); - } + ulong ret = unnamedTemporaryCounter++; + return ret; + } - if (!String.IsNullOrEmpty (parameter.Name)) { - return parameter; + public void ConfigureStartingBlockNumber () + { + if (startingBlockNumber.HasValue) { + return; } - string name = GetNextSlotName (); - if (parameter.NativeFunction != null) { - return new LlvmIrFunctionParameter (parameter.NativeFunction, name, parameter.IsNativePointer, parameter.IsCplusPlusReference); - } - return new LlvmIrFunctionParameter (parameter.Type, name, parameter.IsNativePointer, parameter.IsCplusPlusReference); + startingBlockNumber = unnamedTemporaryCounter++; } } - public LlvmIrFunctionLocalVariable MakeLocalVariable (Type type, string? name = null) + sealed class SavedFunctionState : ILlvmIrSavedFunctionState { - if (String.IsNullOrEmpty (name)) { - name = GetNextSlotName (); + public readonly LlvmIrFunction Owner; + public readonly ILlvmIrSavedFunctionSignatureState SignatureState; + + public SavedFunctionState (LlvmIrFunction owner, ILlvmIrSavedFunctionSignatureState signatureState) + { + Owner = owner; + SignatureState = signatureState; + } + } + + FunctionState functionState; + + public LlvmIrFunctionSignature Signature { get; } + public LlvmIrAddressSignificance AddressSignificance { get; set; } = LlvmIrAddressSignificance.LocalUnnamed; + public LlvmIrFunctionAttributeSet? AttributeSet { get; set; } + public LlvmIrLinkage Linkage { get; set; } = LlvmIrLinkage.Default; + public LlvmIrRuntimePreemption RuntimePreemption { get; set; } = LlvmIrRuntimePreemption.Default; + public LlvmIrVisibility Visibility { get; set; } = LlvmIrVisibility.Default; + public LlvmIrFunctionBody Body { get; } + public string? Comment { get; set; } + public bool ReturnsValue => Signature.ReturnType != typeof(void); + + public LlvmIrFunction (LlvmIrFunctionSignature signature, LlvmIrFunctionAttributeSet? attributeSet = null) + { + Signature = signature; + AttributeSet = attributeSet; + + functionState = new FunctionState (); + foreach (LlvmIrFunctionParameter parameter in signature.Parameters) { + if (!String.IsNullOrEmpty (parameter.Name)) { + continue; + } + + parameter.AssignNumber (functionState.NextTemporary ()); } + functionState.ConfigureStartingBlockNumber (); - return new LlvmIrFunctionLocalVariable (type, name); + Body = new LlvmIrFunctionBody (this, functionState); } - public LlvmIrFunctionLocalVariable MakeLocalVariable (LlvmIrVariable variable, string? name = null) + /// + /// Create new function using data from the signature, with the exception of name. + /// Useful when there are several functions with different names but identical parameters and return types. + /// + public LlvmIrFunction (string name, LlvmIrFunctionSignature templateSignature, LlvmIrFunctionAttributeSet? attributeSet = null) + : this (new LlvmIrFunctionSignature (name, templateSignature), attributeSet) + {} + + public LlvmIrFunction (string name, Type returnType, List? parameters = null, LlvmIrFunctionAttributeSet? attributeSet = null) + : this (new LlvmIrFunctionSignature (name, returnType, parameters), attributeSet) + {} + + /// + /// Creates a local variable which, if is null or empty, is assinged the correct + /// name based on a counter local to the function. + /// + public LlvmIrLocalVariable CreateLocalVariable (Type type, string? name = null) { + var ret = new LlvmIrLocalVariable (type, name); if (String.IsNullOrEmpty (name)) { - name = GetNextSlotName (); + ret.AssignNumber (functionState.NextTemporary ()); } - return new LlvmIrFunctionLocalVariable (variable, name); + return ret; } - public void IncreaseIndent () + /// + /// Save (opaque) function state. This includes signature state. + /// for more information. + /// + public ILlvmIrSavedFunctionState SaveState () { - indentLevel++; - Indent = MakeIndent (indentLevel); + return new SavedFunctionState (this, Signature.SaveState ()); } - public void DecreaseIndent () + /// + /// Restore (opaque) function state. This includes signature state. + /// for more information. + /// + public void RestoreState (ILlvmIrSavedFunctionState savedState) { - if (indentLevel == 0) { - return; + var oldState = savedState as SavedFunctionState; + if (oldState == null) { + throw new InvalidOperationException ($"Internal error: savedState not an instance of {nameof(SavedFunctionState)}"); + } + + if (oldState.Owner != this) { + throw new InvalidOperationException ("Internal error: savedState not saved by this instance"); } - indentLevel--; - Indent = MakeIndent (indentLevel); + Signature.RestoreState (oldState.SignatureState); } - string MakeIndent (uint level) + public override int GetHashCode () { - switch (level) { - case 0: - return String.Empty; - - case 1: - return Indent1; - - case 2: - return Indent2; + return Signature.GetHashCode (); + } - default: - var sb = new StringBuilder (); - for (uint i = 0; i < level; i++) { - sb.Append (LlvmIrGenerator.Indent); - } - return sb.ToString (); + public override bool Equals (object obj) + { + var func = obj as LlvmIrFunction; + if (func == null) { + return false; } + + return Equals (func); } - string GetNextSlotName () + public bool Equals (LlvmIrFunction other) { - string name = $"{localSlot.ToString (CultureInfo.InvariantCulture)}"; - localSlot++; - return name; + if (other == null) { + return false; + } + + return Signature == other.Signature; } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunctionAttributeSet.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunctionAttributeSet.cs new file mode 100644 index 00000000000..f8d91e83058 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunctionAttributeSet.cs @@ -0,0 +1,106 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; + +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks.LLVMIR; + +class LlvmIrFunctionAttributeSet : IEnumerable, IEquatable +{ + public uint Number { get; set; } = 0; + + HashSet attributes; + Dictionary>? privateTargetSpecificAttributes; + + public LlvmIrFunctionAttributeSet () + { + attributes = new HashSet (); + } + + public LlvmIrFunctionAttributeSet (LlvmIrFunctionAttributeSet other) + { + attributes = new HashSet (other); + Number = other.Number; + } + + public IList? GetPrivateTargetAttributes (AndroidTargetArch targetArch) + { + if (privateTargetSpecificAttributes == null || !privateTargetSpecificAttributes.TryGetValue (targetArch, out List list)) { + return null; + } + + return list.AsReadOnly (); + } + + public void Add (LlvmIrFunctionAttribute attr) + { + if (attr == null) { + throw new ArgumentNullException (nameof (attr)); + } + + if (!attributes.Contains (attr)) { + attributes.Add (attr); + } + } + + public void Add (IList attrList) + { + foreach (LlvmIrFunctionAttribute attr in attrList) { + Add (attr); + } + } + + public string Render () + { + List list = attributes.ToList (); + list.Sort ((LlvmIrFunctionAttribute a, LlvmIrFunctionAttribute b) => a.Name.CompareTo (b.Name)); + + return String.Join (" ", list.Select (a => a.Render ())); + } + + public IEnumerator GetEnumerator () => attributes.GetEnumerator (); + + IEnumerator IEnumerable.GetEnumerator () => GetEnumerator (); + + public bool Equals (LlvmIrFunctionAttributeSet other) + { + if (other == null) { + return false; + } + + if (attributes.Count != other.attributes.Count) { + return false; + } + + foreach (LlvmIrFunctionAttribute attr in attributes) { + if (!other.attributes.Contains (attr)) { + return false; + } + } + + return true; + } + + public override bool Equals (object obj) + { + var attrSet = obj as LlvmIrFunctionAttributeSet; + if (attrSet == null) { + return false; + } + + return Equals (attrSet); + } + + public override int GetHashCode() + { + int hc = 0; + + foreach (LlvmIrFunctionAttribute attr in attributes) { + hc ^= attr?.GetHashCode () ?? 0; + } + + return hc; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunctionBody.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunctionBody.cs new file mode 100644 index 00000000000..3336b5d0b7d --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunctionBody.cs @@ -0,0 +1,262 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text; + +namespace Xamarin.Android.Tasks.LLVMIR; + +/// +/// Abstract class from which all of the items (labels, function parameters, +/// local variables and instructions) derive. +/// +abstract class LlvmIrFunctionBodyItem +{ + /// + /// If an item has this property set to true, it won't be written to output when + /// code is generated. This is used for implicit items that don't need to be part of + /// the generated code (e.g. the starting block label) + /// + public bool SkipInOutput { get; protected set; } + public string? Comment { get; set; } + + public void Write (GeneratorWriteContext context, LlvmIrGenerator generator) + { + DoWrite (context, generator); + if (!String.IsNullOrEmpty (Comment)) { + context.Output.Write (' '); + generator.WriteComment (context, Comment); + } + context.Output.WriteLine (); + } + + protected abstract void DoWrite (GeneratorWriteContext context, LlvmIrGenerator generator); +} + +/// +/// Base class for function labels and local variables (including parameters), which +/// obtain automatic names derived from a shared counter, unless explicitly named. +/// +abstract class LlvmIrFunctionLocalItem : LlvmIrFunctionBodyItem +{ + string? name; + + public string Name { + get { + if (String.IsNullOrEmpty (name)) { + throw new InvalidOperationException ("Internal error: name hasn't been set yet"); + } + return name; + } + + protected set { + if (String.IsNullOrEmpty (value)) { + throw new InvalidOperationException ("Internal error: value must not be null or empty"); + } + name = value; + } + } + + protected LlvmIrFunctionLocalItem (string? name) + { + if (name != null) { + Name = name; + } + } + + protected LlvmIrFunctionLocalItem (LlvmIrFunction.FunctionState state, string? name) + { + if (name != null) { + if (name.Length == 0) { + throw new ArgumentException ("must not be an empty string", nameof (name)); + } + + Name = name; + return; + } + + SetName (state.NextTemporary ()); + } + + protected void SetName (ulong num) + { + Name = num.ToString (CultureInfo.InvariantCulture); + } + + protected bool NameIsSet () => !String.IsNullOrEmpty (name); +} + +class LlvmIrFunctionLabelItem : LlvmIrFunctionLocalItem +{ + /// + /// Labels are a bit peculiar in that they must not have their name set to the automatic value (based on + /// a counter shared with function parameters) at creation time, but only when they are actually added to + /// the function body. The reason is that LLVM IR requires all the unnamed temporaries (function parameters and + /// labels) to be named sequentially, but sometimes a label must be referenced before it is added to the instruction + /// stream, e.g. in the br instruction. On the other hand, it is perfectly fine to assign label a name that + /// isn't an integer at **instantiation** time, which is why we have the parameter here. + /// + public LlvmIrFunctionLabelItem (string? name = null) + : base (name) + {} + + public void WillAddToBody (LlvmIrFunctionBody functionBody, LlvmIrFunction.FunctionState state) + { + if (NameIsSet ()) { + return; + } + + SetName (state.NextTemporary ()); + } + + protected override void DoWrite (GeneratorWriteContext context, LlvmIrGenerator generator) + { + context.DecreaseIndent (); + + context.Output.WriteLine (); + context.Output.Write (context.CurrentIndent); + context.Output.Write (Name); + context.Output.Write (':'); + + context.IncreaseIndent (); + } +} + +class LlvmIrFunctionBodyComment : LlvmIrFunctionBodyItem +{ + public string Text { get; } + + public LlvmIrFunctionBodyComment (string comment) + { + Text = comment; + } + + protected override void DoWrite (GeneratorWriteContext context, LlvmIrGenerator generator) + { + context.Output.Write (context.CurrentIndent); + generator.WriteCommentLine (context, Text); + } +} + +class LlvmIrFunctionBody +{ + sealed class LlvmIrFunctionImplicitStartLabel : LlvmIrFunctionLabelItem + { + public LlvmIrFunctionImplicitStartLabel (ulong num) + { + SetName (num); + SkipInOutput = true; + } + } + + List items; + HashSet definedLabels; + LlvmIrFunction ownerFunction; + LlvmIrFunction.FunctionState functionState; + LlvmIrFunctionLabelItem implicitStartBlock; + + LlvmIrFunctionLabelItem? precedingBlock1; + LlvmIrFunctionLabelItem? precedingBlock2; + LlvmIrFunctionLabelItem? previousLabel; + + public IList Items => items.AsReadOnly (); + public LlvmIrFunctionLabelItem? PrecedingBlock1 => precedingBlock1; + public LlvmIrFunctionLabelItem? PrecedingBlock2 => precedingBlock2; + + public LlvmIrFunctionBody (LlvmIrFunction func, LlvmIrFunction.FunctionState functionState) + { + ownerFunction = func; + this.functionState = functionState; + definedLabels = new HashSet (StringComparer.Ordinal); + items = new List (); + previousLabel = implicitStartBlock = new LlvmIrFunctionImplicitStartLabel (functionState.StartingBlockNumber); + } + + public LlvmIrInstructions.Br Br (LlvmIrFunctionLabelItem label) + { + var ret = new LlvmIrInstructions.Br (label); + Add (ret); + return ret; + } + + public LlvmIrInstructions.Br Br (LlvmIrVariable cond, LlvmIrFunctionLabelItem ifTrue, LlvmIrFunctionLabelItem ifFalse) + { + var ret = new LlvmIrInstructions.Br (cond, ifTrue, ifFalse); + Add (ret); + return ret; + } + + public LlvmIrInstructions.Call Call (LlvmIrFunction function, LlvmIrVariable? result = null, ICollection? arguments = null, LlvmIrVariable? funcPointer = null) + { + var ret = new LlvmIrInstructions.Call (function, result, arguments, funcPointer); + Add (ret); + return ret; + } + + public LlvmIrInstructions.Icmp Icmp (LlvmIrIcmpCond cond, LlvmIrVariable op1, object? op2, LlvmIrVariable result) + { + var ret = new LlvmIrInstructions.Icmp (cond, op1, op2, result); + Add (ret); + return ret; + } + + public LlvmIrInstructions.Load Load (LlvmIrVariable source, LlvmIrVariable result, LlvmIrMetadataItem? tbaa = null) + { + var ret = new LlvmIrInstructions.Load (source, result) { + TBAA = tbaa, + }; + Add (ret); + return ret; + } + + /// + /// Creates the `phi` instruction form we use the most throughout marshal methods generator - one which refers to an if/else block and where + /// **both** value:label pairs are **required**. Parameters and are nullable because, in theory, + /// it is possible that hasn't had the required blocks defined prior to adding the `phi` instruction and, thus, + /// we must check for the possibility here. + /// + public LlvmIrInstructions.Phi Phi (LlvmIrVariable result, LlvmIrVariable val1, LlvmIrFunctionLabelItem? label1, LlvmIrVariable val2, LlvmIrFunctionLabelItem? label2) + { + var ret = new LlvmIrInstructions.Phi (result, val1, label1, val2, label2); + Add (ret); + return ret; + } + + public LlvmIrInstructions.Ret Ret (Type retvalType, object? retval = null) + { + var ret = new LlvmIrInstructions.Ret (retvalType, retval); + Add (ret); + return ret; + } + + public void Add (LlvmIrFunctionLabelItem label) + { + label.WillAddToBody (this, functionState); + if (definedLabels.Contains (label.Name)) { + throw new InvalidOperationException ($"Internal error: label with name '{label.Name}' already added to function '{ownerFunction.Signature.Name}' body"); + } + items.Add (label); + definedLabels.Add (label.Name); + + // Rotate preceding blocks + if (precedingBlock2 != null) { + precedingBlock2 = null; + } + + precedingBlock2 = precedingBlock1; + precedingBlock1 = previousLabel; + previousLabel = label; + + var comment = new StringBuilder (" preds = %"); + comment.Append (precedingBlock1.Name); + if (precedingBlock2 != null) { + comment.Append (", %"); + comment.Append (precedingBlock2.Name); + } + label.Comment = comment.ToString (); + } + + public void Add (LlvmIrFunctionBodyItem item) + { + items.Add (item); + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.Code.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.Code.cs deleted file mode 100644 index 94f90dae375..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.Code.cs +++ /dev/null @@ -1,552 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Text; - -namespace Xamarin.Android.Tasks.LLVMIR -{ - abstract partial class LlvmIrGenerator - { - // In code generated by clang, function attributes are determined based on the compiler optimization, - // security arguments, architecture specific flags and so on. For our needs we will have but a - // handful of such sets, based on what clang generates for our native runtime. As such, there is nothing - // "smart" about how we select the attributes, they must match the compiler output for XA runtime, that's all. - // - // Sets are initialized here with the options common to all architectures, the rest is added in the architecture - // specific derived classes. - // - public const int FunctionAttributesXamarinAppInit = 0; - public const int FunctionAttributesJniMethods = 1; - public const int FunctionAttributesCall = 2; - - protected readonly Dictionary FunctionAttributes = new Dictionary (); - - bool codeOutputInitialized = false; - - /// - /// Writes the function definition up to the opening curly brace - /// - public void WriteFunctionStart (LlvmIrFunction function, string? comment = null) - { - if (function == null) { - throw new ArgumentNullException (nameof (function)); - } - - LlvmFunctionAttributeSet? attributes = null; - if (function.AttributeSetID >= 0 && !FunctionAttributes.TryGetValue (function.AttributeSetID, out attributes)) { - throw new InvalidOperationException ($"Function '{function.Name}' refers to attribute set that does not exist (ID: {function.AttributeSetID})"); - } - - Output.WriteLine (); - if (!String.IsNullOrEmpty (comment)) { - foreach (string line in comment.Split ('\n')) { - WriteCommentLine (line); - } - } - - if (attributes != null) { - WriteCommentLine ($"Function attributes: {attributes.Render ()}"); - } - - Output.Write ($"define {GetKnownIRType (function.ReturnType)} @{function.Name} ("); - WriteFunctionParameters (function.Parameters, writeNames: true); - Output.Write(") local_unnamed_addr "); - if (attributes != null) { - Output.Write ($"#{function.AttributeSetID.ToString (CultureInfo.InvariantCulture)}"); - } - Output.WriteLine (); - Output.WriteLine ("{"); - } - - void CodeRenderType (LlvmIrVariable variable, StringBuilder? builder = null) - { - if (variable.NativeFunction != null) { - if (builder == null) { - WriteFunctionSignature (variable.NativeFunction); - } else { - builder.Append (RenderFunctionSignature (variable.NativeFunction)); - } - return; - } - - string extraPointer = variable.IsNativePointer ? "*" : String.Empty; - string irType = $"{GetKnownIRType (variable.Type)}{extraPointer}"; - if (builder == null) { - Output.Write (irType); - } else { - builder.Append (irType); - } - } - - void WriteFunctionParameters (IList? parameters, bool writeNames) - { - string rendered = RenderFunctionParameters (parameters, writeNames); - if (String.IsNullOrEmpty (rendered)) { - return; - } - - Output.Write (rendered); - } - - public string RenderFunctionParameters (IList? parameters, bool writeNames) - { - if (parameters == null || parameters.Count == 0) { - return String.Empty; - } - - var sb = new StringBuilder (); - bool first = true; - foreach (LlvmIrFunctionParameter p in parameters) { - if (!first) { - sb.Append (", "); - } else { - first = false; - } - - CodeRenderType (p, sb); - - if (writeNames) { - sb.Append ($" %{p.Name}"); - } - } - - return sb.ToString (); - } - - public void WriteFunctionSignature (LlvmNativeFunctionSignature sig, bool isPointer = true) - { - Output.Write (RenderFunctionSignature (sig, isPointer)); - } - - public string RenderFunctionSignature (LlvmNativeFunctionSignature sig, bool isPointer = true) - { - if (sig == null) { - throw new ArgumentNullException (nameof (sig)); - } - - var sb = new StringBuilder (); - sb.Append (GetKnownIRType (sig.ReturnType)); - sb.Append (" ("); - sb.Append (RenderFunctionParameters (sig.Parameters, writeNames: false)); - sb.Append (")"); - if (isPointer) { - sb.Append ('*'); - } - - return sb.ToString (); - } - - /// - /// Writes the epilogue of a function, including the return statement if the function return - /// type is void. - /// - public void WriteFunctionEnd (LlvmIrFunction function) - { - if (function == null) { - throw new ArgumentNullException (nameof (function)); - } - - if (function.ReturnType == typeof (void)) { - EmitReturnInstruction (function); - } - - Output.WriteLine ("}"); - } - - /// - /// Emits the ret statement using as the returned value. If - /// is null, void is used as the return value. - /// - public void EmitReturnInstruction (LlvmIrFunction function, LlvmIrFunctionLocalVariable? retVar = null) - { - if (function == null) { - throw new ArgumentNullException (nameof (function)); - } - - string ret = retVar != null ? $"{GetKnownIRType(retVar.Type)} %{retVar.Name}" : "void"; - Output.WriteLine ($"{function.Indent}ret {ret}"); - } - - /// - /// Emits the store instruction (https://llvm.org/docs/LangRef.html#store-instruction), which stores data from a local - /// variable into either local or global destination. If types of and - /// differ, is bitcast to the type of . It is responsibility of the - /// caller to make sure the two types are compatible and/or convertible to each other. - /// - public void EmitStoreInstruction (LlvmIrFunction function, LlvmIrFunctionLocalVariable source, LlvmIrVariableReference destination) - { - if (function == null) { - throw new ArgumentNullException (nameof (function)); - } - - // TODO: implement bitcast, if necessary - Output.Write ($"{function.Indent}store "); - CodeRenderType (source); - Output.Write ($" %{source.Name}, "); - CodeRenderType (destination); - Output.WriteLine ($"* {destination.Reference}, align {GetTypeSize (destination.Type).ToString (CultureInfo.InvariantCulture)}"); - } - - /// - /// Emits the load instruction (https://llvm.org/docs/LangRef.html#load-instruction) - /// - public LlvmIrFunctionLocalVariable EmitLoadInstruction (LlvmIrFunction function, LlvmIrVariableReference source, string? resultVariableName = null) - { - if (function == null) { - throw new ArgumentNullException (nameof (function)); - } - - var sb = new StringBuilder (); - CodeRenderType (source, sb); - - string variableType = sb.ToString (); - LlvmIrFunctionLocalVariable result = function.MakeLocalVariable (source, resultVariableName); - Output.WriteLine ($"{function.Indent}%{result.Name} = load {variableType}, {variableType}* @{source.Name}, align {PointerSize.ToString (CultureInfo.InvariantCulture)}"); - - return result; - } - - /// - /// Emits the icmp comparison instruction (https://llvm.org/docs/LangRef.html#icmp-instruction) - /// - public LlvmIrFunctionLocalVariable EmitIcmpInstruction (LlvmIrFunction function, LlvmIrIcmpCond cond, LlvmIrVariableReference variable, string expectedValue, string? resultVariableName = null) - { - if (function == null) { - throw new ArgumentNullException (nameof (function)); - } - - string condOp; - switch (cond) { - case LlvmIrIcmpCond.Equal: // equal - condOp = "eq"; - break; - - case LlvmIrIcmpCond.NotEqual: // not equal - condOp = "ne"; - break; - - case LlvmIrIcmpCond.UnsignedGreaterThan: // unsigned greater than - condOp = "ugt"; - break; - - case LlvmIrIcmpCond.UnsignedGreaterOrEqual: // unsigned greater or equal - condOp = "uge"; - break; - - case LlvmIrIcmpCond.UnsignedLessThan: // unsigned less than - condOp = "ult"; - break; - - case LlvmIrIcmpCond.UnsignedLessOrEqual: // unsigned less or equal - condOp = "ule"; - break; - - case LlvmIrIcmpCond.SignedGreaterThan: // signed greater than, - condOp = "sgt"; - break; - - case LlvmIrIcmpCond.SignedGreaterOrEqual: // signed greater or equal - condOp = "sge"; - break; - - case LlvmIrIcmpCond.SignedLessThan: // signed less than - condOp = "slt"; - break; - - case LlvmIrIcmpCond.SignedLessOrEqual: // signed less or equal - condOp = "sle"; - break; - - default: - throw new InvalidOperationException ($"Unsupported `icmp` conditional '{cond}'"); - } - - var sb = new StringBuilder (); - CodeRenderType (variable, sb); - - string variableType = sb.ToString (); - LlvmIrFunctionLocalVariable result = function.MakeLocalVariable (variable.Type, resultVariableName); - - Output.WriteLine ($"{function.Indent}%{result.Name} = icmp {condOp} {variableType} {variable.Reference}, {expectedValue}"); - - return result; - } - - public void EmitBrInstruction (LlvmIrFunction function, LlvmIrVariableReference condVariable, string labelTrue, string labelFalse) - { - if (function == null) { - throw new ArgumentNullException (nameof (function)); - } - - Output.WriteLine ($"{function.Indent}br i1 {condVariable.Reference}, label %{labelTrue}, label %{labelFalse}"); - } - - public void EmitBrInstruction (LlvmIrFunction function, string label) - { - if (function == null) { - throw new ArgumentNullException (nameof (function)); - } - - Output.WriteLine ($"{function.Indent}br label %{label}"); - } - - public void EmitLabel (LlvmIrFunction function, string labelName) - { - if (function == null) { - throw new ArgumentNullException (nameof (function)); - } - - Output.WriteLine ($"{labelName}:"); - } - - public LlvmIrFunctionLocalVariable? EmitCall (LlvmIrFunction function, LlvmIrVariableReference targetRef, List? arguments = null, - string? resultVariableName = null, LlvmIrCallMarker marker = LlvmIrCallMarker.Tail, int AttributeSetID = FunctionAttributesCall) - { - if (function == null) { - throw new ArgumentNullException (nameof (function)); - } - - if (targetRef == null) { - throw new ArgumentNullException (nameof (targetRef)); - } - - LlvmNativeFunctionSignature targetSignature = targetRef.NativeFunction; - if (targetSignature == null) { - throw new ArgumentException ("must be reference to native function", nameof (targetRef)); - } - - if (targetSignature.Parameters.Count > 0) { - if (arguments == null) { - throw new ArgumentNullException (nameof (arguments)); - } - - if (targetSignature.Parameters.Count != arguments.Count) { - throw new ArgumentException ($"number of passed parameters ({arguments.Count}) does not match number of parameters in function signature ({targetSignature.Parameters.Count})", nameof (arguments)); - } - } - - bool returnsValue = targetSignature.ReturnType != typeof(void); - LlvmIrFunctionLocalVariable? result = null; - - Output.Write (function.Indent); - if (returnsValue) { - result = function.MakeLocalVariable (targetSignature.ReturnType, resultVariableName); - Output.Write ($"%{result.Name} = "); - } - - switch (marker) { - case LlvmIrCallMarker.Tail: - Output.Write ("tail "); - break; - - case LlvmIrCallMarker.MustTail: - Output.Write ("musttail "); - break; - - case LlvmIrCallMarker.NoTail: - Output.Write ("notail "); - break; - - case LlvmIrCallMarker.None: - break; - - default: - throw new InvalidOperationException ($"Unsupported call marker '{marker}'"); - } - - Output.Write ($"call {GetKnownIRType (targetSignature.ReturnType)} {targetRef.Reference} ("); - - if (targetSignature.Parameters.Count > 0) { - for (int i = 0; i < targetSignature.Parameters.Count; i++) { - LlvmIrFunctionParameter parameter = targetSignature.Parameters[i]; - LlvmIrFunctionArgument argument = arguments[i]; - - AssertValidType (i, parameter, argument); - - if (i > 0) { - Output.Write (", "); - } - - string extra = parameter.IsNativePointer ? "*" : String.Empty; - string paramType = $"{GetKnownIRType (parameter.Type)}{extra}"; - Output.Write ($"{paramType} "); - - if (argument.Value is LlvmIrFunctionLocalVariable variable) { - Output.Write ($"%{variable.Name}"); - } else if (parameter.Type.IsNativePointer () || parameter.IsNativePointer) { - if (parameter.IsCplusPlusReference) { - Output.Write ("nonnull "); - } - - string ptrSize = PointerSize.ToString (CultureInfo.InvariantCulture); - Output.Write ($"align {ptrSize} dereferenceable({ptrSize}) "); - - if (argument.Value is LlvmIrVariableReference variableRef) { - bool needBitcast = parameter.Type != argument.Type; - - if (needBitcast) { - Output.Write ("bitcast ("); - CodeRenderType (variableRef); - Output.Write ("* "); - } - - Output.Write (variableRef.Reference); - - if (needBitcast) { - Output.Write ($" to {paramType})"); - } - } else { - throw new InvalidOperationException ($"Unexpected pointer type in argument {i}, '{argument.Type}'"); - } - } else { - Output.Write (argument.Value.ToString ()); - } - } - } - - Output.Write (")"); - - if (AttributeSetID >= 0) { - if (!FunctionAttributes.ContainsKey (AttributeSetID)) { - throw new InvalidOperationException ($"Unknown attribute set ID {AttributeSetID}"); - } - Output.Write ($" #{AttributeSetID.ToString (CultureInfo.InvariantCulture)}"); - } - Output.WriteLine (); - - return result; - - static void AssertValidType (int index, LlvmIrFunctionParameter parameter, LlvmIrFunctionArgument argument) - { - if (argument.Type == typeof(LlvmIrFunctionLocalVariable) || argument.Type == typeof(LlvmIrVariableReference)) { - return; - } - - if (parameter.Type != typeof(IntPtr)) { - if (argument.Type != parameter.Type) { - ThrowException (); - } - return; - } - - if (argument.Type.IsNativePointer ()) { - return; - } - - if (typeof(LlvmIrVariable).IsAssignableFrom (argument.Type) && - argument.Value is LlvmIrVariable variable && - (variable.IsNativePointer || variable.NativeFunction != null)) { - return; - } - - ThrowException (); - - void ThrowException () - { - throw new InvalidOperationException ($"Argument {index} type '{argument.Type}' does not match the expected function parameter type '{parameter.Type}'"); - } - } - } - - /// - /// Emits the phi instruction (https://llvm.org/docs/LangRef.html#phi-instruction) for a function pointer type - /// - public LlvmIrFunctionLocalVariable EmitPhiInstruction (LlvmIrFunction function, LlvmIrVariableReference target, List<(LlvmIrVariableReference variableRef, string label)> pairs, string? resultVariableName = null) - { - if (function == null) { - throw new ArgumentNullException (nameof (function)); - } - - LlvmIrFunctionLocalVariable result = function.MakeLocalVariable (target, resultVariableName); - Output.Write ($"{function.Indent}%{result.Name} = phi "); - CodeRenderType (target); - - bool first = true; - foreach ((LlvmIrVariableReference variableRef, string label) in pairs) { - if (first) { - first = false; - Output.Write (' '); - } else { - Output.Write (", "); - } - - Output.Write ($"[{variableRef.Reference}, %{label}]"); - } - Output.WriteLine (); - - return result; - } - - public void InitCodeOutput () - { - if (codeOutputInitialized) { - return; - } - - InitFunctionAttributes (); - InitCodeMetadata (); - codeOutputInitialized = true; - } - - protected virtual void InitCodeMetadata () - { - MetadataManager.Add ("llvm.linker.options"); - } - - protected virtual void InitFunctionAttributes () - { - FunctionAttributes[FunctionAttributesXamarinAppInit] = new LlvmFunctionAttributeSet { - new MinLegalVectorWidthFunctionAttribute (0), - new MustprogressFunctionAttribute (), - new NofreeFunctionAttribute (), - new NorecurseFunctionAttribute (), - new NosyncFunctionAttribute (), - new NoTrappingMathFunctionAttribute (true), - new NounwindFunctionAttribute (), - new SspstrongFunctionAttribute (), - new StackProtectorBufferSizeFunctionAttribute (8), - new UwtableFunctionAttribute (), - new WillreturnFunctionAttribute (), - new WriteonlyFunctionAttribute (), - }; - - FunctionAttributes[FunctionAttributesJniMethods] = new LlvmFunctionAttributeSet { - new MinLegalVectorWidthFunctionAttribute (0), - new MustprogressFunctionAttribute (), - new NoTrappingMathFunctionAttribute (true), - new NounwindFunctionAttribute (), - new SspstrongFunctionAttribute (), - new StackProtectorBufferSizeFunctionAttribute (8), - new UwtableFunctionAttribute (), - }; - - FunctionAttributes[FunctionAttributesCall] = new LlvmFunctionAttributeSet { - new NounwindFunctionAttribute (), - }; - } - - void WriteAttributeSets () - { - if (!codeOutputInitialized) { - return; - } - - WriteSet (FunctionAttributesXamarinAppInit, Output); - WriteSet (FunctionAttributesJniMethods, Output); - WriteSet (FunctionAttributesCall, Output); - - Output.WriteLine (); - - void WriteSet (int id, TextWriter output) - { - output.Write ($"attributes #{id.ToString (CultureInfo.InvariantCulture)} = {{ "); - foreach (LLVMFunctionAttribute attr in FunctionAttributes[id]) { - output.Write (attr.Render ()); - output.Write (' '); - } - output.WriteLine ("}"); - } - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.Constants.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.Constants.cs new file mode 100644 index 00000000000..9600dbc81e1 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.Constants.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; + +namespace Xamarin.Android.Tasks.LLVMIR +{ + partial class LlvmIrGenerator + { + // https://llvm.org/docs/LangRef.html#linkage-types + static readonly Dictionary llvmLinkage = new Dictionary { + { LlvmIrLinkage.Default, String.Empty }, + { LlvmIrLinkage.Private, "private" }, + { LlvmIrLinkage.Internal, "internal" }, + { LlvmIrLinkage.AvailableExternally, "available_externally" }, + { LlvmIrLinkage.LinkOnce, "linkonce" }, + { LlvmIrLinkage.Weak, "weak" }, + { LlvmIrLinkage.Common, "common" }, + { LlvmIrLinkage.Appending, "appending" }, + { LlvmIrLinkage.ExternWeak, "extern_weak" }, + { LlvmIrLinkage.LinkOnceODR, "linkonce_odr" }, + { LlvmIrLinkage.External, "external" }, + }; + + // https://llvm.org/docs/LangRef.html#runtime-preemption-specifiers + static readonly Dictionary llvmRuntimePreemption = new Dictionary { + { LlvmIrRuntimePreemption.Default, String.Empty }, + { LlvmIrRuntimePreemption.DSOPreemptable, "dso_preemptable" }, + { LlvmIrRuntimePreemption.DSOLocal, "dso_local" }, + }; + + // https://llvm.org/docs/LangRef.html#visibility-styles + static readonly Dictionary llvmVisibility = new Dictionary { + { LlvmIrVisibility.Default, "default" }, + { LlvmIrVisibility.Hidden, "hidden" }, + { LlvmIrVisibility.Protected, "protected" }, + }; + + // https://llvm.org/docs/LangRef.html#global-variables + static readonly Dictionary llvmAddressSignificance = new Dictionary { + { LlvmIrAddressSignificance.Default, String.Empty }, + { LlvmIrAddressSignificance.Unnamed, "unnamed_addr" }, + { LlvmIrAddressSignificance.LocalUnnamed, "local_unnamed_addr" }, + }; + + // https://llvm.org/docs/LangRef.html#global-variables + static readonly Dictionary llvmWritability = new Dictionary { + { LlvmIrWritability.Constant, "constant" }, + { LlvmIrWritability.Writable, "global" }, + }; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs index 469133189d6..202d04a48c3 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs @@ -1,5 +1,6 @@ using System; using System.Buffers; +using System.Collections; using System.Collections.Generic; using System.Globalization; using System.IO; @@ -9,1565 +10,1284 @@ namespace Xamarin.Android.Tasks.LLVMIR { - /// - /// Base class for all classes which implement architecture-specific code generators. - /// - abstract partial class LlvmIrGenerator + sealed class GeneratorStructureInstance : StructureInstance { - internal sealed class StructureBodyWriterOptions - { - public readonly bool WriteFieldComment; - public readonly string FieldIndent; - public readonly string StructIndent; - public readonly TextWriter? StructureOutput; - public readonly TextWriter? StringsOutput; - public readonly TextWriter? BuffersOutput; + public GeneratorStructureInstance (StructureInfo info, object instance) + : base (info, instance) + {} + } - public StructureBodyWriterOptions (bool writeFieldComment, string fieldIndent = "", string structIndent = "", - TextWriter? structureOutput = null, TextWriter? stringsOutput = null, TextWriter? buffersOutput = null) - { - WriteFieldComment = writeFieldComment; - FieldIndent = fieldIndent; - StructIndent = structIndent; - StructureOutput = structureOutput; - StringsOutput = stringsOutput; - BuffersOutput = buffersOutput; - } + sealed class GeneratorWriteContext + { + const char IndentChar = '\t'; + + int currentIndentLevel = 0; + + public readonly TextWriter Output; + public readonly LlvmIrModule Module; + public readonly LlvmIrModuleTarget Target; + public readonly LlvmIrMetadataManager MetadataManager; + public string CurrentIndent { get; private set; } = String.Empty; + public bool InVariableGroup { get; set; } + + public GeneratorWriteContext (TextWriter writer, LlvmIrModule module, LlvmIrModuleTarget target, LlvmIrMetadataManager metadataManager) + { + Output = writer; + Module = module; + Target = target; + MetadataManager = metadataManager; } - sealed class PackedStructureMember + public void IncreaseIndent () { - public readonly string ValueIRType; - public readonly string? PaddingIRType; - public readonly object? Value; - public readonly bool IsPadded; - public readonly StructureMemberInfo MemberInfo; + currentIndentLevel++; + CurrentIndent = MakeIndentString (); + } - public PackedStructureMember (StructureMemberInfo memberInfo, object? value, string? valueIRType = null, string? paddingIRType = null) - { - ValueIRType = valueIRType ?? memberInfo.IRType; - Value = value; - MemberInfo = memberInfo; - PaddingIRType = paddingIRType; - IsPadded = !String.IsNullOrEmpty (paddingIRType); + public void DecreaseIndent () + { + if (currentIndentLevel > 0) { + currentIndentLevel--; } + CurrentIndent = MakeIndentString (); } - public sealed class StringSymbolInfo + string MakeIndentString () => currentIndentLevel > 0 ? new String (IndentChar, currentIndentLevel) : String.Empty; + } + + partial class LlvmIrGenerator + { + sealed class LlvmTypeInfo { - public readonly string SymbolName; + public readonly bool IsPointer; + public readonly bool IsAggregate; + public readonly bool IsStructure; public readonly ulong Size; + public readonly ulong MaxFieldAlignment; - public StringSymbolInfo (string symbolName, ulong size) + public LlvmTypeInfo (bool isPointer, bool isAggregate, bool isStructure, ulong size, ulong maxFieldAlignment) { - SymbolName = symbolName; + IsPointer = isPointer; + IsAggregate = isAggregate; + IsStructure = isStructure; Size = size; + MaxFieldAlignment = maxFieldAlignment; } } - static readonly Dictionary typeMap = new Dictionary { - { typeof (bool), "i8" }, - { typeof (byte), "i8" }, - { typeof (char), "i8" }, - { typeof (sbyte), "i8" }, - { typeof (short), "i16" }, - { typeof (ushort), "i16" }, - { typeof (int), "i32" }, - { typeof (uint), "i32" }, - { typeof (long), "i64" }, - { typeof (ulong), "i64" }, - { typeof (float), "float" }, - { typeof (double), "double" }, - { typeof (string), "i8*" }, - { typeof (IntPtr), "i8*" }, - { typeof (void), "void" }, - }; - - // https://llvm.org/docs/LangRef.html#single-value-types - static readonly Dictionary typeSizes = new Dictionary { - { typeof (bool), 1 }, - { typeof (byte), 1 }, - { typeof (char), 1 }, - { typeof (sbyte), 1 }, - { typeof (short), 2 }, - { typeof (ushort), 2 }, - { typeof (int), 4 }, - { typeof (uint), 4 }, - { typeof (long), 8 }, - { typeof (ulong), 8 }, - { typeof (float), 4 }, // floats are 32-bit - { typeof (double), 8 }, // doubles are 64-bit - }; - - // https://llvm.org/docs/LangRef.html#linkage-types - static readonly Dictionary llvmLinkage = new Dictionary { - { LlvmIrLinkage.Default, String.Empty }, - { LlvmIrLinkage.Private, "private" }, - { LlvmIrLinkage.Internal, "internal" }, - { LlvmIrLinkage.AvailableExternally, "available_externally" }, - { LlvmIrLinkage.LinkOnce, "linkonce" }, - { LlvmIrLinkage.Weak, "weak" }, - { LlvmIrLinkage.Common, "common" }, - { LlvmIrLinkage.Appending, "appending" }, - { LlvmIrLinkage.ExternWeak, "extern_weak" }, - { LlvmIrLinkage.LinkOnceODR, "linkonce_odr" }, - { LlvmIrLinkage.External, "external" }, - }; - - // https://llvm.org/docs/LangRef.html#runtime-preemption-specifiers - static readonly Dictionary llvmRuntimePreemption = new Dictionary { - { LlvmIrRuntimePreemption.Default, String.Empty }, - { LlvmIrRuntimePreemption.DSOPreemptable, "dso_preemptable" }, - { LlvmIrRuntimePreemption.DSOLocal, "dso_local" }, - }; - - // https://llvm.org/docs/LangRef.html#visibility-styles - static readonly Dictionary llvmVisibility = new Dictionary { - { LlvmIrVisibility.Default, "default" }, - { LlvmIrVisibility.Hidden, "hidden" }, - { LlvmIrVisibility.Protected, "protected" }, - }; - - // https://llvm.org/docs/LangRef.html#global-variables - static readonly Dictionary llvmAddressSignificance = new Dictionary { - { LlvmIrAddressSignificance.Default, String.Empty }, - { LlvmIrAddressSignificance.Unnamed, "unnamed_addr" }, - { LlvmIrAddressSignificance.LocalUnnamed, "local_unnamed_addr" }, - }; - - // https://llvm.org/docs/LangRef.html#global-variables - static readonly Dictionary llvmWritability = new Dictionary { - { LlvmIrWritability.Constant, "constant" }, - { LlvmIrWritability.Writable, "global" }, - }; + sealed class BasicType + { + public readonly string Name; + public readonly ulong Size; + public readonly bool IsNumeric; - static readonly LlvmIrVariableOptions preAllocatedBufferVariableOptions = new LlvmIrVariableOptions { - Writability = LlvmIrWritability.Writable, - Linkage = LlvmIrLinkage.Internal, + public BasicType (string name, ulong size, bool isNumeric = true) + { + Name = name; + Size = size; + IsNumeric = isNumeric; + } + } + + public const string IRPointerType = "ptr"; + + static readonly Dictionary basicTypeMap = new Dictionary { + { typeof (bool), new ("i8", 1, isNumeric: false) }, + { typeof (byte), new ("i8", 1) }, + { typeof (char), new ("i16", 2) }, + { typeof (sbyte), new ("i8", 1) }, + { typeof (short), new ("i16", 2) }, + { typeof (ushort), new ("i16", 2) }, + { typeof (int), new ("i32", 4) }, + { typeof (uint), new ("i32", 4) }, + { typeof (long), new ("i64", 8) }, + { typeof (ulong), new ("i64", 8) }, + { typeof (float), new ("float", 4) }, + { typeof (double), new ("double", 8) }, + { typeof (void), new ("void", 0, isNumeric: false) }, }; - string fileName; - ulong stringCounter = 0; - ulong structStringCounter = 0; - ulong structBufferCounter = 0; + public string FilePath { get; } + public string FileName { get; } - List structures = new List (); - Dictionary stringSymbolCache = new Dictionary (StringComparer.Ordinal); - LlvmIrMetadataItem llvmModuleFlags; + LlvmIrModuleTarget target; - public const string Indent = "\t"; - - protected abstract string DataLayout { get; } - public abstract int PointerSize { get; } - protected abstract string Triple { get; } - - public bool Is64Bit { get; } - public TextWriter Output { get; } - public AndroidTargetArch TargetArch { get; } - - protected LlvmIrMetadataManager MetadataManager { get; } - - protected LlvmIrGenerator (AndroidTargetArch arch, TextWriter output, string fileName) + protected LlvmIrGenerator (string filePath, LlvmIrModuleTarget target) { - Output = output; - MetadataManager = new LlvmIrMetadataManager (); - TargetArch = arch; - Is64Bit = arch == AndroidTargetArch.X86_64 || arch == AndroidTargetArch.Arm64; - this.fileName = fileName; + FilePath = Path.GetFullPath (filePath); + FileName = Path.GetFileName (filePath); + this.target = target; } - /// - /// Create architecture-specific generator for the given . Contents are written - // to the stream and is used mostly for error - // reporting. - /// - public static LlvmIrGenerator Create (AndroidTargetArch arch, StreamWriter output, string fileName) + public static LlvmIrGenerator Create (AndroidTargetArch arch, string fileName) { - LlvmIrGenerator ret = Instantiate (); - ret.Init (); - return ret; - - LlvmIrGenerator Instantiate () - { - return arch switch { - AndroidTargetArch.Arm => new Arm32LlvmIrGenerator (arch, output, fileName), - AndroidTargetArch.Arm64 => new Arm64LlvmIrGenerator (arch, output, fileName), - AndroidTargetArch.X86 => new X86LlvmIrGenerator (arch, output, fileName), - AndroidTargetArch.X86_64 => new X64LlvmIrGenerator (arch, output, fileName), - _ => throw new InvalidOperationException ($"Unsupported Android target ABI {arch}") - }; - } + return arch switch { + AndroidTargetArch.Arm => new LlvmIrGenerator (fileName, new LlvmIrModuleArmV7a ()), + AndroidTargetArch.Arm64 => new LlvmIrGenerator (fileName, new LlvmIrModuleAArch64 ()), + AndroidTargetArch.X86 => new LlvmIrGenerator (fileName, new LlvmIrModuleX86 ()), + AndroidTargetArch.X86_64 => new LlvmIrGenerator (fileName, new LlvmIrModuleX64 ()), + _ => throw new InvalidOperationException ($"Unsupported Android target ABI {arch}") + }; } - static string EnsureIrType (Type type) + public void Generate (TextWriter writer, LlvmIrModule module) { - if (!typeMap.TryGetValue (type, out string irType)) { - throw new InvalidOperationException ($"Unsupported managed type {type}"); - } - - return irType; - } + LlvmIrMetadataManager metadataManager = module.GetMetadataManagerCopy (); + target.AddTargetSpecificMetadata (metadataManager); - static Type GetActualType (Type type) - { - // Arrays of types are handled elsewhere, so we obtain the array base type here - if (type.IsArray) { - return type.GetElementType (); + var context = new GeneratorWriteContext (writer, module, target, metadataManager); + if (!String.IsNullOrEmpty (FilePath)) { + WriteCommentLine (context, $" ModuleID = '{FileName}'"); + context.Output.WriteLine ($"source_filename = \"{FileName}\""); } - return type; - } - - /// - /// Map managed to its LLVM IR counterpart. Only primitive types, - /// string and IntPtr are supported. - /// - public static string MapManagedTypeToIR (Type type) - { - return EnsureIrType (GetActualType (type)); - } - - /// - /// Map managed type to its LLVM IR counterpart. Only primitive types, string and - /// IntPtr are supported. Additionally, return the native type size (in bytes) in - /// - /// - public string MapManagedTypeToIR (Type type, out ulong size) - { - Type actualType = GetActualType (type); - string irType = EnsureIrType (actualType); - size = GetTypeSize (actualType); + context.Output.WriteLine (target.DataLayout.Render ()); + context.Output.WriteLine ($"target triple = \"{target.Triple}\""); + WriteStructureDeclarations (context); + WriteGlobalVariables (context); + WriteFunctions (context); - return irType; + // Bottom of the file + WriteStrings (context); + WriteExternalFunctionDeclarations (context); + WriteAttributeSets (context); + WriteMetadata (context); } - ulong GetTypeSize (Type actualType) + void WriteStrings (GeneratorWriteContext context) { - if (!typeSizes.TryGetValue (actualType, out ulong size)) { - if (actualType == typeof (string) || actualType == typeof (IntPtr) || actualType == typeof (LlvmNativeFunctionSignature)) { - size = (ulong)PointerSize; - } else { - throw new InvalidOperationException ($"Unsupported managed type {actualType}"); - } + if (context.Module.Strings == null || context.Module.Strings.Count == 0) { + return; } - return size; - } - - /// - /// Map managed type to its LLVM IR counterpart. Only primitive types, - /// string and IntPtr are supported. Additionally, return the native type size - /// (in bytes) in - /// - public string MapManagedTypeToIR (out ulong size) - { - return MapManagedTypeToIR (typeof(T), out size); - } + context.Output.WriteLine (); + WriteComment (context, " Strings"); - /// - /// Map a managed to its C++ counterpart. Only primitive types, - /// string and IntPtr are supported. - /// - public static string MapManagedTypeToNative (Type type) - { - Type baseType = GetActualType (type); + foreach (LlvmIrStringGroup group in context.Module.Strings) { + context.Output.WriteLine (); - if (baseType == typeof (bool)) return "bool"; - if (baseType == typeof (byte)) return "uint8_t"; - if (baseType == typeof (char)) return "char"; - if (baseType == typeof (sbyte)) return "int8_t"; - if (baseType == typeof (short)) return "int16_t"; - if (baseType == typeof (ushort)) return "uint16_t"; - if (baseType == typeof (int)) return "int32_t"; - if (baseType == typeof (uint)) return "uint32_t"; - if (baseType == typeof (long)) return "int64_t"; - if (baseType == typeof (ulong)) return "uint64_t"; - if (baseType == typeof (float)) return "float"; - if (baseType == typeof (double)) return "double"; - if (baseType == typeof (string)) return "char*"; - if (baseType == typeof (IntPtr)) return "void*"; + if (!String.IsNullOrEmpty (group.Comment)) { + WriteCommentLine (context, group.Comment); + } - return type.GetShortName (); - } + foreach (LlvmIrStringVariable info in group.Strings) { + string s = QuoteString ((string)info.Value, out ulong size); - public string GetIRType (out ulong size, T? value = default) - { - if (typeof(T) == typeof(LlvmNativeFunctionSignature)) { - if (value == null) { - throw new ArgumentNullException (nameof (value)); + WriteGlobalVariableStart (context, info); + context.Output.Write ('['); + context.Output.Write (size.ToString (CultureInfo.InvariantCulture)); + context.Output.Write (" x i8] c"); + context.Output.Write (s); + context.Output.Write (", align "); + context.Output.WriteLine (target.GetAggregateAlignment (1, size).ToString (CultureInfo.InvariantCulture)); } - - size = (ulong)PointerSize; - return RenderFunctionSignature ((LlvmNativeFunctionSignature)(object)value); } - - return MapManagedTypeToIR (out size); } - public string GetKnownIRType (Type type) + void WriteGlobalVariables (GeneratorWriteContext context) { - if (type == null) { - throw new ArgumentNullException (nameof (type)); - } - - if (type.IsNativeClass ()) { - IStructureInfo si = GetStructureInfo (type); - return $"%{si.NativeTypeDesignator}.{si.Name}"; + if (context.Module.GlobalVariables == null || context.Module.GlobalVariables.Count == 0) { + return; } - return MapManagedTypeToIR (type); - } + foreach (LlvmIrGlobalVariable gv in context.Module.GlobalVariables) { + if (gv is LlvmIrGroupDelimiterVariable groupDelimiter) { + if (!context.InVariableGroup && !String.IsNullOrEmpty (groupDelimiter.Comment)) { + context.Output.WriteLine (); + context.Output.Write (context.CurrentIndent); + WriteComment (context, groupDelimiter.Comment); + } - public string GetValue (T value) - { - if (typeof(T) == typeof(LlvmNativeFunctionSignature)) { - if (value == null) { - throw new ArgumentNullException (nameof (value)); + context.InVariableGroup = !context.InVariableGroup; + if (context.InVariableGroup) { + context.Output.WriteLine (); + } + continue; } - var v = (LlvmNativeFunctionSignature)(object)value; - if (v.FieldValue != null) { - return MonoAndroidHelper.CultureInvariantToString (v.FieldValue); + if (gv.BeforeWriteCallback != null) { + gv.BeforeWriteCallback (gv, target, gv.BeforeWriteCallbackCallerState); } + WriteGlobalVariable (context, gv); + } + } - return MonoAndroidHelper.CultureInvariantToString (v); + void WriteGlobalVariableStart (GeneratorWriteContext context, LlvmIrGlobalVariable variable) + { + if (!String.IsNullOrEmpty (variable.Comment)) { + WriteCommentLine (context, variable.Comment); } + context.Output.Write ('@'); + context.Output.Write (variable.Name); + context.Output.Write (" = "); - return MonoAndroidHelper.CultureInvariantToString (value) ?? String.Empty; + LlvmIrVariableOptions options = variable.Options ?? LlvmIrGlobalVariable.DefaultOptions; + WriteLinkage (context, options.Linkage); + WritePreemptionSpecifier (context, options.RuntimePreemption); + WriteVisibility (context, options.Visibility); + WriteAddressSignificance (context, options.AddressSignificance); + WriteWritability (context, options.Writability); } - /// - /// Initialize the generator. It involves adding required LLVM IR module metadata (such as data model specification, - /// code generation flags etc) - /// - protected virtual void Init () + void WriteGlobalVariable (GeneratorWriteContext context, LlvmIrGlobalVariable variable) { - llvmModuleFlags = MetadataManager.Add ("llvm.module.flags"); - LlvmIrMetadataItem ident = MetadataManager.Add ("llvm.ident"); + if (!context.InVariableGroup) { + context.Output.WriteLine (); + } - var flagsFields = new List (); - AddModuleFlagsMetadata (flagsFields); + WriteGlobalVariableStart (context, variable); + WriteTypeAndValue (context, variable, out LlvmTypeInfo typeInfo); + context.Output.Write (", align "); - foreach (LlvmIrMetadataItem item in flagsFields) { - llvmModuleFlags.AddReferenceField (item.Name); + ulong alignment; + if (typeInfo.IsAggregate) { + ulong count = GetAggregateValueElementCount (variable); + alignment = (ulong)target.GetAggregateAlignment ((int)typeInfo.MaxFieldAlignment, count * typeInfo.Size); + } else if (typeInfo.IsStructure) { + alignment = (ulong)target.GetAggregateAlignment ((int)typeInfo.MaxFieldAlignment, typeInfo.Size); + } else if (typeInfo.IsPointer) { + alignment = target.NativePointerSize; + } else { + alignment = typeInfo.Size; } - LlvmIrMetadataItem identValue = MetadataManager.AddNumbered ($"Xamarin.Android {XABuildConfig.XamarinAndroidBranch} @ {XABuildConfig.XamarinAndroidCommitHash}"); - ident.AddReferenceField (identValue.Name); + context.Output.WriteLine (alignment.ToString (CultureInfo.InvariantCulture)); } - protected void AddLlvmModuleFlag (LlvmIrMetadataItem flag) + void WriteTypeAndValue (GeneratorWriteContext context, LlvmIrVariable variable, out LlvmTypeInfo typeInfo) { - llvmModuleFlags.AddReferenceField (flag.Name); - } + WriteType (context, variable, out typeInfo); + context.Output.Write (' '); - /// - /// Since LLVM IR is strongly typed, it requires each structure to be properly declared before it is - /// used throughout the code. This method uses reflection to scan the managed type - /// and record the information for future use. The returned structure contains - /// the description. It is used later on not only to declare the structure in output code, but also to generate - /// data from instances of . This method is typically called from the - /// method. - /// - public StructureInfo MapStructure () - { - Type t = typeof(T); - if (!t.IsClass && !t.IsValueType) { - throw new InvalidOperationException ($"{t} must be a class or a struct"); + Type valueType; + if (variable.Value is LlvmIrVariable referencedVariable) { + valueType = referencedVariable.Type; + } else { + valueType = variable.Value?.GetType () ?? variable.Type; } - var ret = new StructureInfo (this); - structures.Add (ret); - - return ret; - } - - internal IStructureInfo GetStructureInfo (Type type) - { - IStructureInfo? ret = null; + if (variable.Value == null) { + // Order of checks is important here. Aggregates can contain pointer types, in which case typeInfo.IsPointer + // will be `true` and the aggregate would be incorrectly initialized with `null` instead of the correct + // `zeroinitializer` + if (typeInfo.IsAggregate) { + WriteValue (context, valueType, variable); + return; + } - foreach (IStructureInfo si in structures) { - if (si.Type != type) { - continue; + if (typeInfo.IsPointer) { + context.Output.Write ("null"); + return; } - ret = si; - break; + throw new InvalidOperationException ($"Internal error: variable of type {variable.Type} must not have a null value"); } - if (ret == null) { - throw new InvalidOperationException ($"Unmapped structure {type}"); + if (valueType != variable.Type && !LlvmIrModule.NameValueArrayType.IsAssignableFrom (variable.Type)) { + throw new InvalidOperationException ($"Internal error: variable type '{variable.Type}' is different to its value type, '{valueType}'"); } - return ret; + WriteValue (context, valueType, variable); } - TextWriter EnsureOutput (TextWriter? output) - { - return output ?? Output; - } + ulong GetAggregateValueElementCount (LlvmIrVariable variable) => GetAggregateValueElementCount (variable.Type, variable.Value, variable as LlvmIrGlobalVariable); - void WriteGlobalSymbolStart (string symbolName, LlvmIrVariableOptions options, TextWriter? output = null) + ulong GetAggregateValueElementCount (Type type, object? value, LlvmIrGlobalVariable? globalVariable = null) { - output = EnsureOutput (output); - - output.Write ('@'); - output.Write (symbolName); - output.Write (" = "); - - string linkage = llvmLinkage [options.Linkage]; - if (!string.IsNullOrEmpty (linkage)) { - output.Write (linkage); - output.Write (' '); + if (!type.IsArray ()) { + throw new InvalidOperationException ($"Internal error: unknown type {type} when trying to determine aggregate type element count"); } - if (options.AddressSignificance != LlvmIrAddressSignificance.Default) { - output.Write (llvmAddressSignificance[options.AddressSignificance]); - output.Write (' '); - } - - output.Write (llvmWritability[options.Writability]); - output.Write (' '); - } - object? GetTypedMemberValue (StructureInfo info, StructureMemberInfo smi, StructureInstance instance, Type expectedType, object? defaultValue = null) - { - object? value = smi.GetValue (instance.Obj); if (value == null) { - return defaultValue; + if (globalVariable != null) { + return globalVariable.ArrayItemCount; + } + return 0; } - if (value.GetType () != expectedType) { - throw new InvalidOperationException ($"Field '{smi.Info.Name}' of structure '{info.Name}' should have a value of '{expectedType}' type, instead it had a '{value.GetType ()}'"); + // TODO: use caching here + if (type.ImplementsInterface (typeof(IDictionary))) { + return (uint)((IDictionary)value).Count * 2; } - if (expectedType == typeof(bool)) { - return (bool)value ? 1 : 0; + if (type.ImplementsInterface (typeof(ICollection))) { + return (uint)((ICollection)value).Count; } - return value; + throw new InvalidOperationException ($"Internal error: should never get here"); } - bool MaybeWriteStructureString (StructureInfo info, StructureMemberInfo smi, StructureInstance instance, TextWriter? output = null) + void WriteType (GeneratorWriteContext context, LlvmIrVariable variable, out LlvmTypeInfo typeInfo) { - if (smi.MemberType != typeof(string)) { - return false; - } - - output = EnsureOutput (output); - string? str = (string?)GetTypedMemberValue (info, smi, instance, typeof(string), null); - if (str == null) { - instance.AddPointerData (smi, null, 0); - return false; - } - - StringSymbolInfo stringSymbol = WriteUniqueString ($"__{info.Name}_{smi.Info.Name}", str, ref structStringCounter); - instance.AddPointerData (smi, stringSymbol.SymbolName, stringSymbol.Size); - - return true; + WriteType (context, variable.Type, variable.Value, out typeInfo, variable as LlvmIrGlobalVariable); } - bool MaybeWritePreAllocatedBuffer (StructureInfo info, StructureMemberInfo smi, StructureInstance instance, TextWriter? output = null) + void WriteType (GeneratorWriteContext context, StructureInstance si, StructureMemberInfo memberInfo, out LlvmTypeInfo typeInfo) { - if (!smi.Info.IsNativePointerToPreallocatedBuffer (out ulong bufferSize)) { - return false; - } - - if (smi.Info.UsesDataProvider ()) { - bufferSize = info.GetBufferSizeFromProvider (smi, instance); - } - - output = EnsureOutput (output); - string irType = MapManagedTypeToIR (smi.MemberType); - string variableName = $"__{info.Name}_{smi.Info.Name}_{structBufferCounter.ToString (CultureInfo.InvariantCulture)}"; - structBufferCounter++; - - WriteGlobalSymbolStart (variableName, preAllocatedBufferVariableOptions, output); - ulong size = bufferSize * smi.BaseTypeSize; - - // WriteLine $"[{bufferSize} x {irType}] zeroinitializer, align {GetAggregateAlignment ((int)smi.BaseTypeSize, size)}" - output.Write ('['); - output.Write (bufferSize.ToString (CultureInfo.InvariantCulture)); - output.Write (" x "); - output.Write (irType); - output.Write ("] zeroinitializer, align "); - output.WriteLine (GetAggregateAlignment ((int) smi.BaseTypeSize, size).ToString (CultureInfo.InvariantCulture)); - - instance.AddPointerData (smi, variableName, size); - return true; - } + if (memberInfo.IsNativePointer) { + typeInfo = new LlvmTypeInfo ( + isPointer: true, + isAggregate: false, + isStructure: false, + size: target.NativePointerSize, + maxFieldAlignment: target.NativePointerSize + ); - bool WriteStructureArrayStart (StructureInfo info, IList>? instances, LlvmIrVariableOptions options, string? symbolName = null, string? initialComment = null, TextWriter? output = null) - { - if (options.IsGlobal && String.IsNullOrEmpty (symbolName)) { - throw new ArgumentException ("must not be null or empty for global symbols", nameof (symbolName)); + context.Output.Write (IRPointerType); + return; } - bool named = !String.IsNullOrEmpty (symbolName); - if (named || !String.IsNullOrEmpty (initialComment)) { - WriteEOL (output: output); - WriteEOL (initialComment ?? symbolName, output); + if (memberInfo.IsInlineArray) { + WriteArrayType (context, memberInfo.MemberType.GetArrayElementType (), memberInfo.ArrayElements, out typeInfo); + return; } - if (named) { - WriteGlobalSymbolStart (symbolName, options, output); + if (memberInfo.IsIRStruct ()) { + var sim = new GeneratorStructureInstance (context.Module.GetStructureInfo (memberInfo.MemberType), memberInfo.GetValue (si.Obj)); + WriteStructureType (context, sim, out typeInfo); + return; } - return named; - } - - void WriteStructureArrayEnd (StructureInfo info, string? symbolName, ulong count, bool named, bool skipFinalComment = false, TextWriter? output = null, bool isArrayOfPointers = false) - { - output = EnsureOutput (output); - - int alignment = isArrayOfPointers ? PointerSize : GetAggregateAlignment (info.MaxFieldAlignment, info.Size * count); - output.Write (", align "); - output.Write (alignment.ToString (CultureInfo.InvariantCulture)); - if (named && !skipFinalComment) { - WriteEOL ($"end of '{symbolName!}' array", output); - } else { - WriteEOL (output: output); - } + WriteType (context, memberInfo.MemberType, value: null, out typeInfo); } - /// - /// Writes an array of zero-initialized entries. specifies the symbol attributes (visibility, writeability etc) - /// - public void WriteStructureArray (StructureInfo info, ulong count, LlvmIrVariableOptions options, string? symbolName = null, bool writeFieldComment = true, string? initialComment = null, bool isArrayOfPointers = false) + void WriteStructureType (GeneratorWriteContext context, StructureInstance si, out LlvmTypeInfo typeInfo) { - bool named = WriteStructureArrayStart (info, null, options, symbolName, initialComment); + ulong alignment = GetStructureMaxFieldAlignment (si.Info); - // $"[{count} x %{info.NativeTypeDesignator}.{info.Name}{pointerAsterisk}] zeroinitializer" - Output.Write ('['); - Output.Write (count.ToString (CultureInfo.InvariantCulture)); - Output.Write (" x %"); - Output.Write (info.NativeTypeDesignator); - Output.Write ('.'); - Output.Write (info.Name); - if (isArrayOfPointers) - Output.Write ('*'); - Output.Write ("] zeroinitializer"); + typeInfo = new LlvmTypeInfo ( + isPointer: false, + isAggregate: false, + isStructure: true, + size: si.Info.Size, + maxFieldAlignment: alignment + ); - WriteStructureArrayEnd (info, symbolName, (ulong)count, named, skipFinalComment: true, isArrayOfPointers: isArrayOfPointers); + context.Output.Write ('%'); + context.Output.Write (si.Info.NativeTypeDesignator); + context.Output.Write ('.'); + context.Output.Write (si.Info.Name); } - /// - /// Writes an array of zero-initialized entries. The array will be generated as a local, writable symbol. - /// - public void WriteStructureArray (StructureInfo info, ulong count, string? symbolName = null, bool writeFieldComment = true, string? initialComment = null, bool isArrayOfPointers = false) + void WriteType (GeneratorWriteContext context, Type type, object? value, out LlvmTypeInfo typeInfo, LlvmIrGlobalVariable? globalVariable = null) { - WriteStructureArray (info, count, LlvmIrVariableOptions.Default, symbolName, writeFieldComment, initialComment, isArrayOfPointers); - } + if (IsStructureInstance (type)) { + if (value == null) { + throw new ArgumentException ("must not be null for structure instances", nameof (value)); + } - /// - /// Writes an array of managed type , with data optionally specified in (if it's null, the array - /// will be zero-initialized). specifies the symbol attributes (visibility, writeability etc) - /// - public void WriteStructureArray (StructureInfo info, IList>? instances, LlvmIrVariableOptions options, - string? symbolName = null, bool writeFieldComment = true, string? initialComment = null, - Action? nestedStructureWriter = null) - { - var arrayOutput = new StringWriter (); - bool named = WriteStructureArrayStart (info, instances, options, symbolName, initialComment, arrayOutput); - int count = instances != null ? instances.Count : 0; - - // $"[{count} x %{info.NativeTypeDesignator}.{info.Name}] " - arrayOutput.Write ('['); - arrayOutput.Write (count.ToString (CultureInfo.InvariantCulture)); - arrayOutput.Write (" x %"); - arrayOutput.Write (info.NativeTypeDesignator); - arrayOutput.Write ('.'); - arrayOutput.Write (info.Name); - arrayOutput.Write ("] "); - - if (instances != null) { - var bodyWriterOptions = new StructureBodyWriterOptions ( - writeFieldComment: true, - fieldIndent: $"{Indent}{Indent}", - structIndent: Indent, - structureOutput: arrayOutput, - stringsOutput: info.HasStrings ? new StringWriter () : null, - buffersOutput: info.HasPreAllocatedBuffers ? new StringWriter () : null - ); + WriteStructureType (context, (StructureInstance)value, out typeInfo); + return; + } - arrayOutput.WriteLine ('['); - for (int i = 0; i < count; i++) { - StructureInstance instance = instances[i]; + string irType; + ulong size; + bool isPointer; - arrayOutput.Write (Indent); - arrayOutput.Write ("; "); - arrayOutput.WriteLine (i.ToString (CultureInfo.InvariantCulture)); - WriteStructureBody (info, instance, bodyWriterOptions, nestedStructureWriter); - if (i < count - 1) { - arrayOutput.Write (", "); - } - WriteEOL (output: arrayOutput); - } - arrayOutput.Write (']'); + if (type.IsArray ()) { + Type elementType = type.GetArrayElementType (); + ulong elementCount = GetAggregateValueElementCount (type, value, globalVariable); - WriteBufferToOutput (bodyWriterOptions.StringsOutput); - WriteBufferToOutput (bodyWriterOptions.BuffersOutput); - } else { - arrayOutput.Write ("zeroinitializer"); + WriteArrayType (context, elementType, elementCount, out typeInfo); + return; } - WriteStructureArrayEnd (info, symbolName, (ulong)count, named, skipFinalComment: instances == null, output: arrayOutput); - WriteBufferToOutput (arrayOutput); - } - - /// - /// Writes an array of managed type , with data optionally specified in (if it's null, the array - /// will be zero-initialized). The array will be generated as a local, writable symbol. - /// - public void WriteStructureArray (StructureInfo info, IList>? instances, string? symbolName = null, bool writeFieldComment = true, string? initialComment = null) - { - WriteStructureArray (info, instances, LlvmIrVariableOptions.Default, symbolName, writeFieldComment, initialComment); + irType = GetIRType (type, out size, out isPointer); + typeInfo = new LlvmTypeInfo ( + isPointer: isPointer, + isAggregate: false, + isStructure: false, + size: size, + maxFieldAlignment: size + ); + context.Output.Write (irType); } - public void WriteArray (IList values, string symbolName, string? initialComment = null) + void WriteArrayType (GeneratorWriteContext context, Type elementType, ulong elementCount, out LlvmTypeInfo typeInfo) { - WriteEOL (); - WriteEOL (initialComment ?? symbolName); + string irType; + ulong size; + ulong maxFieldAlignment; + bool isPointer; - ulong arrayStringCounter = 0; - var strings = new List (); - - foreach (string s in values) { - StringSymbolInfo symbol = WriteUniqueString ($"__{symbolName}", s, ref arrayStringCounter, LlvmIrVariableOptions.LocalConstexprString); - strings.Add (new StringSymbolInfo (symbol.SymbolName, symbol.Size)); - } + if (elementType.IsStructureInstance (out Type? structureType)) { + StructureInfo si = context.Module.GetStructureInfo (structureType); - if (strings.Count > 0) { - Output.WriteLine (); - } + irType = $"%{si.NativeTypeDesignator}.{si.Name}"; + size = si.Size; + maxFieldAlignment = GetStructureMaxFieldAlignment (si); + isPointer = false; + } else { + irType = GetIRType (elementType, out size, out isPointer); + maxFieldAlignment = size; + } + typeInfo = new LlvmTypeInfo ( + isPointer: isPointer, + isAggregate: true, + isStructure: false, + size: size, + maxFieldAlignment: maxFieldAlignment + ); - WriteStringArray (symbolName, LlvmIrVariableOptions.GlobalConstantStringPointer, strings); + context.Output.Write ('['); + context.Output.Write (elementCount.ToString (CultureInfo.InvariantCulture)); + context.Output.Write (" x "); + context.Output.Write (irType); + context.Output.Write (']'); } - public void WriteArray (IList values, LlvmIrVariableOptions options, string symbolName, Func? commentProvider = null) where T: struct + ulong GetStructureMaxFieldAlignment (StructureInfo si) { - bool optimizeOutput = commentProvider == null; - - WriteGlobalSymbolStart (symbolName, options); - string elementType = MapManagedTypeToIR (typeof (T), out ulong size); + if (si.HasPointers && target.NativePointerSize > si.MaxFieldAlignment) { + return target.NativePointerSize; + } - // WriteLine $"[{values.Count} x {elementType}] [" - Output.Write ('['); - Output.Write (values.Count.ToString (CultureInfo.InvariantCulture)); - Output.Write (" x "); - Output.Write (elementType); - Output.WriteLine ("] ["); + return si.MaxFieldAlignment; + } - Output.Write (Indent); - for (int i = 0; i < values.Count; i++) { - if (i != 0) { - if (optimizeOutput) { - Output.Write (','); - if (i % 8 == 0) { - Output.Write (" ; "); - Output.Write (i - 8); - Output.Write (".."); - Output.WriteLine (i - 1); + bool IsStructureInstance (Type t) => typeof(StructureInstance).IsAssignableFrom (t); - Output.Write (Indent); - } else { - Output.Write (' '); - } - } else { - Output.Write (Indent); - } + void WriteValue (GeneratorWriteContext context, Type valueType, LlvmIrVariable variable) + { + if (variable.Type.IsArray ()) { + bool zeroInitialize = false; + if (variable is LlvmIrGlobalVariable gv) { + zeroInitialize = gv.ZeroInitializeArray || variable.Value == null; + } else { + zeroInitialize = GetAggregateValueElementCount (variable) == 0; } - Output.Write (elementType); - Output.Write (' '); - Output.Write (MonoAndroidHelper.CultureInvariantToString (values [i])); - - if (!optimizeOutput) { - bool last = i == values.Count - 1; - if (!last) { - Output.Write (','); - } - - string? comment = commentProvider (i, values[i]); - if (!String.IsNullOrEmpty (comment)) { - Output.Write (" ; "); - Output.Write (comment); - } - - if (!last) { - Output.WriteLine (); - } + if (zeroInitialize) { + context.Output.Write ("zeroinitializer"); + return; } - } - if (optimizeOutput && values.Count / 8 != 0) { - int idx = values.Count - (values.Count % 8); - Output.Write (" ; "); - Output.Write (idx); - Output.Write (".."); - Output.Write (values.Count - 1); + + WriteArrayValue (context, variable); + return; } - Output.WriteLine (); - Output.Write ("], align "); - Output.WriteLine (GetAggregateAlignment ((int) size, size * (ulong) values.Count).ToString (CultureInfo.InvariantCulture)); + WriteValue (context, valueType, variable.Value); } - void AssertArraySize (StructureInfo info, StructureMemberInfo smi, ulong length, ulong expectedLength) + void AssertArraySize (StructureInstance si, StructureMemberInfo smi, ulong length, ulong expectedLength) { if (length == expectedLength) { return; } - throw new InvalidOperationException ($"Invalid array size in field '{smi.Info.Name}' of structure '{info.Name}', expected {expectedLength}, found {length}"); - } - - void RenderArray (StructureInfo info, StructureMemberInfo smi, byte[] bytes, TextWriter output, ulong? expectedArraySize = null) - { - // Byte arrays are represented in the same way as strings, without the explicit NUL termination byte - AssertArraySize (info, smi, expectedArraySize ?? (ulong)bytes.Length, smi.ArrayElements); - output.Write ('c'); - output.Write (QuoteString (bytes, bytes.Length, out _, nullTerminated: false)); + throw new InvalidOperationException ($"Invalid array size in field '{smi.Info.Name}' of structure '{si.Info.Name}', expected {expectedLength}, found {length}"); } - void MaybeWriteStructureStringsAndBuffers (StructureInfo info, StructureMemberInfo smi, StructureInstance instance, StructureBodyWriterOptions options) + void WriteValue (GeneratorWriteContext context, StructureInstance structInstance, StructureMemberInfo smi, object? value) { - if (options.StringsOutput != null) { - MaybeWriteStructureString (info, smi, instance, options.StringsOutput); + if (smi.IsNativePointer) { + if (WriteNativePointerValue (context, structInstance, smi, value)) { + return; + } } - if (options.BuffersOutput != null) { - MaybeWritePreAllocatedBuffer (info, smi, instance, options.BuffersOutput); - } - } + if (smi.IsInlineArray) { + Array a = (Array)value; + ulong length = smi.ArrayElements == 0 ? (ulong)a.Length : smi.ArrayElements; - void WriteStructureField (StructureInfo info, StructureInstance instance, StructureMemberInfo smi, int fieldIndex, - StructureBodyWriterOptions options, TextWriter output, object? valueOverride = null, ulong? expectedArraySize = null, - Action? nestedStructureWriter = null) - { - object? value = null; + if (smi.MemberType == typeof(byte[])) { + var bytes = (byte[])value; - if (smi.IsIRStruct ()) { - if (nestedStructureWriter == null) { - throw new InvalidOperationException ($"Nested structure found in type {typeof(T)}, field {smi.Info.Name} but no nested structure writer provided"); - } - nestedStructureWriter (this, options, smi.MemberType, valueOverride ?? GetTypedMemberValue (info, smi, instance, smi.MemberType)); - } else if (smi.IsNativePointer) { - output.Write (options.FieldIndent); - WritePointer (info, smi, instance, output); - } else if (smi.IsNativeArray) { - if (!smi.IsInlineArray) { - throw new InvalidOperationException ($"Out of line arrays aren't supported at this time (structure '{info.Name}', field '{smi.Info.Name}')"); + // Byte arrays are represented in the same way as strings, without the explicit NUL termination byte + AssertArraySize (structInstance, smi, length, smi.ArrayElements); + context.Output.Write ('c'); + context.Output.Write (QuoteString (bytes, bytes.Length, out _, nullTerminated: false)); + return; } - output.Write (options.FieldIndent); - output.Write (smi.IRType); - output.Write (" "); - value = valueOverride ?? GetTypedMemberValue (info, smi, instance, smi.MemberType); + throw new NotSupportedException ($"Internal error: inline arrays of type {smi.MemberType} aren't supported at this point. Field {smi.Info.Name} in structure {structInstance.Info.Name}"); + } - if (smi.MemberType == typeof(byte[])) { - RenderArray (info, smi, (byte[])value, output, expectedArraySize); - } else { - throw new InvalidOperationException ($"Arrays of type '{smi.MemberType}' aren't supported at this point (structure '{info.Name}', field '{smi.Info.Name}')"); - } - } else { - value = valueOverride; - output.Write (options.FieldIndent); - WritePrimitiveField (info, smi, instance, output); + if (smi.IsIRStruct ()) { + StructureInfo si = context.Module.GetStructureInfo (smi.MemberType); + WriteValue (context, typeof(GeneratorStructureInstance), new GeneratorStructureInstance (si, value)); + return; + } + + if (smi.Info.IsNativePointerToPreallocatedBuffer (out _)) { + string bufferVariableName = context.Module.LookupRequiredBufferVariableName (structInstance, smi); + context.Output.Write ('@'); + context.Output.Write (bufferVariableName); + return; } - FinishStructureField (info, smi, instance, options, fieldIndex, value, output); + WriteValue (context, smi.MemberType, value); } - void WriteStructureBody (StructureInfo info, StructureInstance? instance, StructureBodyWriterOptions options, Action? nestedStructureWriter = null) + bool WriteNativePointerValue (GeneratorWriteContext context, StructureInstance si, StructureMemberInfo smi, object? value) { - TextWriter structureOutput = EnsureOutput (options.StructureOutput); - - // $"{options.StructIndent}%{info.NativeTypeDesignator}.{info.Name} " - structureOutput.Write (options.StructIndent); - structureOutput.Write ('%'); - structureOutput.Write (info.NativeTypeDesignator); - structureOutput.Write ('.'); - structureOutput.Write (info.Name); - structureOutput.Write (' '); + // Structure members decorated with the [NativePointer] attribute cannot have a + // value other than `null`, unless they are strings or references to symbols - if (instance != null) { - structureOutput.WriteLine ('{'); - for (int i = 0; i < info.Members.Count; i++) { - StructureMemberInfo smi = info.Members[i]; + if (smi.Info.PointsToSymbol (out string? symbolName)) { + if (String.IsNullOrEmpty (symbolName) && smi.Info.UsesDataProvider ()) { + if (si.Info.DataProvider == null) { + throw new InvalidOperationException ($"Field '{smi.Info.Name}' of structure '{si.Info.Name}' points to a symbol, but symbol name wasn't provided and there's no configured data context provider"); + } + symbolName = si.Info.DataProvider.GetPointedToSymbolName (si.Obj, smi.Info.Name); + } - MaybeWriteStructureStringsAndBuffers (info, smi, instance, options); - WriteStructureField (info, instance, smi, i, options, structureOutput, nestedStructureWriter: nestedStructureWriter); + if (String.IsNullOrEmpty (symbolName)) { + context.Output.Write ("null"); + } else { + context.Output.Write ('@'); + context.Output.Write (symbolName); } + return true; + } - structureOutput.Write (options.StructIndent); - structureOutput.Write ('}'); - } else { - structureOutput.Write ("zeroinitializer"); + if (smi.MemberType != typeof(string)) { + context.Output.Write ("null"); + return true; } + + return false; } - void MaybeWriteFieldComment (StructureInfo info, StructureMemberInfo smi, StructureInstance instance, StructureBodyWriterOptions options, object? value, TextWriter output) + void WriteValue (GeneratorWriteContext context, Type type, object? value) { - if (!options.WriteFieldComment) { + if (value is LlvmIrVariable variableRef) { + context.Output.Write (variableRef.Reference); return; } - string? comment = info.GetCommentFromProvider (smi, instance); - if (String.IsNullOrEmpty (comment)) { - var sb = new StringBuilder (smi.Info.Name); - if (value != null && smi.MemberType.IsPrimitive && smi.MemberType != typeof(bool)) { - sb.Append (" (0x"); - sb.Append ($"{value:x}"); - sb.Append (')'); - } - comment = sb.ToString (); + if (IsNumeric (type)) { + context.Output.Write (MonoAndroidHelper.CultureInvariantToString (value)); + return; } - WriteComment (output, comment); - } - void FinishStructureField (StructureInfo info, StructureMemberInfo smi, StructureInstance instance, StructureBodyWriterOptions options, int fieldIndex, object? value, TextWriter output) - { - if (fieldIndex < info.Members.Count - 1) { - output.Write (", "); + if (type == typeof(bool)) { + context.Output.Write ((bool)value ? '1' : '0'); + return; } - MaybeWriteFieldComment (info, smi, instance, options, value, output); - WriteEOL (output); - } - - void WritePrimitiveField (StructureInfo info, StructureMemberInfo smi, StructureInstance instance, TextWriter output, object? overrideValue = null) - { - object? value = overrideValue ?? GetTypedMemberValue (info, smi, instance, smi.MemberType); - output.Write (smi.IRType); - output.Write (' '); - output.Write (MonoAndroidHelper.CultureInvariantToString (value)); - } - void WritePointer (StructureInfo info, StructureMemberInfo smi, StructureInstance instance, TextWriter output, object? overrideValue = null) - { - if (info.HasStrings) { - StructurePointerData? spd = instance.GetPointerData (smi); - if (spd != null) { - WriteGetStringPointer (spd.VariableName, spd.Size, indent: false, output: output); - return; - } + if (IsStructureInstance (type)) { + WriteStructureValue (context, (StructureInstance?)value); + return; } - if (info.HasPreAllocatedBuffers) { - StructurePointerData? spd = instance.GetPointerData (smi); - if (spd != null) { - WriteGetBufferPointer (spd.VariableName, smi.IRType, spd.Size, indent: false, output: output); - return; - } + if (type == typeof(IntPtr)) { + // Pointers can only be `null` or a reference to variable + context.Output.Write ("null"); + return; } - if (smi.Info.PointsToSymbol (out string? symbolName)) { - if (String.IsNullOrEmpty (symbolName) && smi.Info.UsesDataProvider ()) { - if (info.DataProvider == null) { - throw new InvalidOperationException ($"Field '{smi.Info.Name}' of structure '{info.Name}' points to a symbol, but symbol name wasn't provided and there's no configured data context provider"); - } - symbolName = info.DataProvider.GetPointedToSymbolName (instance.Obj, smi.Info.Name); - } - - if (String.IsNullOrEmpty (symbolName)) { - WriteNullPointer (smi, output); + if (type == typeof(string)) { + if (value == null) { + context.Output.Write ("null"); return; } - ulong bufferSize = info.GetBufferSizeFromProvider (smi, instance); - WriteGetBufferPointer (symbolName, smi.IRType, bufferSize, indent: false, output: output); + LlvmIrStringVariable sv = context.Module.LookupRequiredVariableForString ((string)value); + context.Output.Write (sv.Reference); return; } - object? value = overrideValue ?? smi.GetValue (instance.Obj); - if (value == null || ((value is IntPtr) && (IntPtr)value == IntPtr.Zero)) { - WriteNullPointer (smi, output); - return; - } + if (type.IsInlineArray ()) { - if (value.GetType ().IsPrimitive) { - ulong v = Convert.ToUInt64 (value); - if (v == 0) { - WriteNullPointer (smi, output); - return; - } } - throw new InvalidOperationException ($"While processing field '{smi.Info.Name}' of type '{info.Name}': non-null pointers to objects of managed type '{smi.MemberType}' (IR type '{smi.IRType}') currently not supported (value: {value})"); + throw new NotSupportedException ($"Internal error: value type '{type}' is unsupported"); } - void WriteNullPointer (StructureMemberInfo smi, TextWriter output) + void WriteStructureValue (GeneratorWriteContext context, StructureInstance? instance) { - output.Write (smi.IRType); - output.Write (" null"); - } + if (instance == null || instance.IsZeroInitialized) { + context.Output.Write ("zeroinitializer"); + return; + } - // In theory, functionality implemented here should be folded into WriteStructureArray, but in practice it would slow processing for most of the structures we - // write, thus we'll keep this one separate, even at the cost of some code duplication - // - // This code is extremely ugly, one day it should be made look nicer (right... :D) - // - public void WritePackedStructureArray (StructureInfo info, IList> instances, LlvmIrVariableOptions options, string? symbolName = null, bool writeFieldComment = true, string? initialComment = null) - { - StructureBodyWriterOptions bodyWriterOptions = InitStructureWrite (info, options, symbolName, writeFieldComment, fieldIndent: $"{Indent}{Indent}"); - TextWriter structureOutput = EnsureOutput (bodyWriterOptions.StructureOutput); - var structureBodyOutput = new StringWriter (); - var structureTypeOutput = new StringWriter (); - - bool firstInstance = true; - var members = new List> (); - var instanceType = new StringBuilder (); - foreach (StructureInstance instance in instances) { - members.Clear (); - bool hasPaddedFields = false; - - if (!firstInstance) { - structureTypeOutput.WriteLine (','); - structureBodyOutput.WriteLine (','); - } else { - firstInstance = false; - } + context.Output.WriteLine ('{'); + context.IncreaseIndent (); - foreach (StructureMemberInfo smi in info.Members) { - object? value = GetTypedMemberValue (info, smi, instance, smi.MemberType); + StructureInfo info = instance.Info; + int lastMember = info.Members.Count - 1; - if (!smi.NeedsPadding) { - members.Add (new PackedStructureMember (smi, value)); - continue; - } + for (int i = 0; i < info.Members.Count; i++) { + StructureMemberInfo smi = info.Members[i]; - if (smi.MemberType != typeof(byte[])) { - throw new InvalidOperationException ($"Only byte arrays are supported currently (field '{smi.Info.Name}' of structure '{info.Name}')"); - } + context.Output.Write (context.CurrentIndent); + WriteType (context, instance, smi, out _); + context.Output.Write (' '); - var array = (byte[])value; - var arrayLength = (ulong)array.Length; + object? value = GetTypedMemberValue (context, info, smi, instance, smi.MemberType); + WriteValue (context, instance, smi, value); - if (arrayLength > smi.ArrayElements) { - throw new InvalidOperationException ($"Field '{smi.Info.Name}' of structure '{info.Name}' should not have more than {smi.ArrayElements} elements"); - } + if (i < lastMember) { + context.Output.Write (", "); + } - ulong padding = smi.ArrayElements - arrayLength; - if (padding == 0) { - members.Add (new PackedStructureMember (smi, value)); - continue; + string? comment = info.GetCommentFromProvider (smi, instance); + if (String.IsNullOrEmpty (comment)) { + var sb = new StringBuilder (" "); + sb.Append (MapManagedTypeToNative (smi)); + sb.Append (' '); + sb.Append (smi.Info.Name); + if (value != null && smi.MemberType.IsPrimitive && smi.MemberType != typeof(bool)) { + sb.Append ($" (0x{value:x})"); } + comment = sb.ToString (); + } + WriteCommentLine (context, comment); + } - if (padding < 8) { - var paddedValue = new byte[arrayLength + padding]; - Array.Copy (array, paddedValue, array.Length); - for (int i = (int)arrayLength; i < paddedValue.Length; i++) { - paddedValue[i] = 0; - } - members.Add (new PackedStructureMember (smi, paddedValue)); - continue; - } + context.DecreaseIndent (); + context.Output.Write (context.CurrentIndent); + context.Output.Write ('}'); + } - members.Add (new PackedStructureMember (smi, value, valueIRType: $"[{arrayLength} x i8]", paddingIRType: $"[{padding} x i8]")); - hasPaddedFields = true; + void WriteArrayValue (GeneratorWriteContext context, LlvmIrVariable variable) + { + ICollection entries; + if (variable.Type.ImplementsInterface (typeof(IDictionary))) { + var list = new List (); + foreach (var kvp in (IDictionary)variable.Value) { + list.Add (kvp.Key); + list.Add (kvp.Value); } + entries = list; + } else { + entries = (ICollection)variable.Value; + } - bool firstField; - instanceType.Clear (); - if (!hasPaddedFields) { - instanceType.Append ("\t%"); - instanceType.Append (info.NativeTypeDesignator); - instanceType.Append ('.'); - instanceType.Append (info.Name); - } else { - instanceType.Append ("\t{ "); - - firstField = true; - foreach (PackedStructureMember psm in members) { - if (!firstField) { - instanceType.Append (", "); - } else { - firstField = false; - } - - if (!psm.IsPadded) { - instanceType.Append (psm.ValueIRType); - continue; - } - - // $"<{{ {psm.ValueIRType}, {psm.PaddingIRType} }}>" - instanceType.Append ("<{ "); - instanceType.Append (psm.ValueIRType); - instanceType.Append (", "); - instanceType.Append (psm.PaddingIRType); - instanceType.Append (" }>"); - } + if (entries.Count == 0) { + context.Output.Write ("zeroinitializer"); + return; + } - instanceType.Append (" }"); - } - structureTypeOutput.Write (instanceType.ToString ()); + context.Output.WriteLine ('['); + context.IncreaseIndent (); - structureBodyOutput.Write (instanceType.ToString ()); - structureBodyOutput.WriteLine (" {"); + Type elementType = variable.Type.GetArrayElementType (); + bool writeIndices = (variable.WriteOptions & LlvmIrVariableWriteOptions.ArrayWriteIndexComments) == LlvmIrVariableWriteOptions.ArrayWriteIndexComments; + ulong counter = 0; + string? prevItemComment = null; + uint stride; - firstField = true; - bool previousFieldWasPadded = false; - for (int i = 0; i < members.Count; i++) { - PackedStructureMember psm = members[i]; + if ((variable.WriteOptions & LlvmIrVariableWriteOptions.ArrayFormatInRows) == LlvmIrVariableWriteOptions.ArrayFormatInRows) { + stride = variable.ArrayStride > 0 ? variable.ArrayStride : 1; + } else { + stride = 1; + } - if (firstField) { - firstField = false; - } + bool first = true; - if (!psm.IsPadded) { - previousFieldWasPadded = false; - ulong? expectedArraySize = psm.MemberInfo.IsNativeArray ? (ulong)((byte[])psm.Value).Length : null; - WriteStructureField (info, instance, psm.MemberInfo, i, bodyWriterOptions, structureBodyOutput, valueOverride: psm.Value, expectedArraySize: expectedArraySize); - continue; - } + // TODO: implement output in rows + foreach (object entry in entries) { + if (!first) { + context.Output.Write (','); + WritePrevItemCommentOrNewline (); + } else { + first = false; + } - if (!firstField && previousFieldWasPadded) { - structureBodyOutput.Write (", "); - } + prevItemComment = null; + if (variable.GetArrayItemCommentCallback != null) { + prevItemComment = variable.GetArrayItemCommentCallback (variable, target, counter, entry, variable.GetArrayItemCommentCallbackCallerState); + } - // $"{bodyWriterOptions.FieldIndent}<{{ {psm.ValueIRType}, {psm.PaddingIRType} }}> <{{ {psm.ValueIRType} c{QuoteString ((byte[])psm.Value)}, {psm.PaddingIRType} zeroinitializer }}> " - structureBodyOutput.Write (bodyWriterOptions.FieldIndent); - structureBodyOutput.Write ("<{ "); - structureBodyOutput.Write (psm.ValueIRType); - structureBodyOutput.Write (", "); - structureBodyOutput.Write (psm.PaddingIRType); - structureBodyOutput.Write (" }> <{ "); - structureBodyOutput.Write (psm.ValueIRType); - structureBodyOutput.Write (" c"); - structureBodyOutput.Write (QuoteString ((byte []) psm.Value)); - structureBodyOutput.Write (", "); - structureBodyOutput.Write (psm.PaddingIRType); - structureBodyOutput.Write (" zeroinitializer }> "); - - MaybeWriteFieldComment (info, psm.MemberInfo, instance, bodyWriterOptions, value: null, output: structureBodyOutput); - previousFieldWasPadded = true; + if (writeIndices && String.IsNullOrEmpty (prevItemComment)) { + prevItemComment = $" {counter}"; } - structureBodyOutput.WriteLine (); - structureBodyOutput.Write (Indent); - structureBodyOutput.Write ('}'); - } - structureOutput.WriteLine ("<{"); - structureOutput.Write (structureTypeOutput); - structureOutput.WriteLine (); - structureOutput.WriteLine ("}>"); + counter++; + context.Output.Write (context.CurrentIndent); + WriteType (context, elementType, entry, out _); - structureOutput.WriteLine ("<{"); - structureOutput.Write (structureBodyOutput); - structureOutput.WriteLine (); - structureOutput.Write ("}>"); + context.Output.Write (' '); + WriteValue (context, elementType, entry); + } + WritePrevItemCommentOrNewline (); + + context.DecreaseIndent (); + context.Output.Write (']'); - FinishStructureWrite (info, bodyWriterOptions); + void WritePrevItemCommentOrNewline () + { + if (!String.IsNullOrEmpty (prevItemComment)) { + context.Output.Write (' '); + WriteCommentLine (context, prevItemComment); + } else { + context.Output.WriteLine (); + } + } } - StructureBodyWriterOptions InitStructureWrite (StructureInfo info, LlvmIrVariableOptions options, string? symbolName, bool writeFieldComment, string? fieldIndent = null) + void WriteLinkage (GeneratorWriteContext context, LlvmIrLinkage linkage) { - if (options.IsGlobal && String.IsNullOrEmpty (symbolName)) { - throw new ArgumentException ("must not be null or empty for global symbols", nameof (symbolName)); + if (linkage == LlvmIrLinkage.Default) { + return; } - var structureOutput = new StringWriter (); - bool named = !String.IsNullOrEmpty (symbolName); - if (named) { - WriteEOL (output: structureOutput); - WriteEOL (symbolName, structureOutput); - - WriteGlobalSymbolStart (symbolName, options, structureOutput); + try { + WriteAttribute (context, llvmLinkage[linkage]); + } catch (Exception ex) { + throw new InvalidOperationException ($"Internal error: unsupported writability '{linkage}'", ex); } - - return new StructureBodyWriterOptions ( - writeFieldComment: writeFieldComment, - fieldIndent: fieldIndent ?? Indent, - structureOutput: structureOutput, - stringsOutput: info.HasStrings ? new StringWriter () : null, - buffersOutput: info.HasPreAllocatedBuffers ? new StringWriter () : null - ); } - void FinishStructureWrite (StructureInfo info, StructureBodyWriterOptions bodyWriterOptions) + void WriteWritability (GeneratorWriteContext context, LlvmIrWritability writability) { - bodyWriterOptions.StructureOutput.Write (", align "); - bodyWriterOptions.StructureOutput.WriteLine (info.MaxFieldAlignment.ToString (CultureInfo.InvariantCulture)); - - WriteBufferToOutput (bodyWriterOptions.StringsOutput); - WriteBufferToOutput (bodyWriterOptions.BuffersOutput); - WriteBufferToOutput (bodyWriterOptions.StructureOutput); + try { + WriteAttribute (context, llvmWritability[writability]); + } catch (Exception ex) { + throw new InvalidOperationException ($"Internal error: unsupported writability '{writability}'", ex); + } } - public void WriteStructure (StructureInfo info, StructureInstance? instance, StructureBodyWriterOptions bodyWriterOptions, LlvmIrVariableOptions options, string? symbolName = null, bool writeFieldComment = true) + void WriteAddressSignificance (GeneratorWriteContext context, LlvmIrAddressSignificance addressSignificance) { - WriteStructureBody (info, instance, bodyWriterOptions); - FinishStructureWrite (info, bodyWriterOptions); + if (addressSignificance == LlvmIrAddressSignificance.Default) { + return; + } + + try { + WriteAttribute (context, llvmAddressSignificance[addressSignificance]); + } catch (Exception ex) { + throw new InvalidOperationException ($"Internal error: unsupported address significance '{addressSignificance}'", ex); + } } - public void WriteNestedStructure (StructureInfo info, StructureInstance instance, StructureBodyWriterOptions bodyWriterOptions) + void WriteVisibility (GeneratorWriteContext context, LlvmIrVisibility visibility) { - var options = new StructureBodyWriterOptions ( - bodyWriterOptions.WriteFieldComment, - bodyWriterOptions.FieldIndent + Indent, - bodyWriterOptions.FieldIndent, // structure indent should start at the original struct's field column - bodyWriterOptions.StructureOutput, - bodyWriterOptions.StringsOutput, - bodyWriterOptions.BuffersOutput - ); - WriteStructureBody (info, instance, options); + if (visibility == LlvmIrVisibility.Default) { + return; + } + + try { + WriteAttribute (context, llvmVisibility[visibility]); + } catch (Exception ex) { + throw new InvalidOperationException ($"Internal error: unsupported visibility '{visibility}'", ex); + } } - /// - /// Write a structure represented by managed type , with optional data passed in (if null, the structure - /// is zero-initialized). specifies the symbol attributes (visibility, writeability etc) - /// - public void WriteStructure (StructureInfo info, StructureInstance? instance, LlvmIrVariableOptions options, string? symbolName = null, bool writeFieldComment = true) + void WritePreemptionSpecifier (GeneratorWriteContext context, LlvmIrRuntimePreemption preemptionSpecifier) { - StructureBodyWriterOptions bodyWriterOptions = InitStructureWrite (info, options, symbolName, writeFieldComment); - WriteStructure (info, instance, bodyWriterOptions, options, symbolName, writeFieldComment); + if (preemptionSpecifier == LlvmIrRuntimePreemption.Default) { + return; + } + + try { + WriteAttribute (context, llvmRuntimePreemption[preemptionSpecifier]); + } catch (Exception ex) { + throw new InvalidOperationException ($"Internal error: unsupported preemption specifier '{preemptionSpecifier}'", ex); + } } /// - /// Write a structure represented by managed type , with optional data passed in (if null, the structure - /// is zero-initialized). The structure will be generated as a local, writable symbol. + /// Write attribute named in followed by a single space /// - public void WriteStructure (StructureInfo info, StructureInstance? instance, string? symbolName = null, bool writeFieldComment = true) + void WriteAttribute (GeneratorWriteContext context, string attr) { - WriteStructure (info, instance, LlvmIrVariableOptions.Default, symbolName, writeFieldComment); + context.Output.Write (attr); + context.Output.Write (' '); } - void WriteBufferToOutput (TextWriter? writer) + void WriteStructureDeclarations (GeneratorWriteContext context) { - if (writer == null) { + if (context.Module.Structures == null || context.Module.Structures.Count == 0) { return; } - writer.Flush (); - string text = writer.ToString (); - if (text.Length > 0) { - Output.WriteLine (text); + foreach (StructureInfo si in context.Module.Structures) { + context.Output.WriteLine (); + WriteStructureDeclaration (context, si); } } - void WriteGetStringPointer (string? variableName, ulong size, bool indent = true, TextWriter? output = null) + void WriteStructureDeclaration (GeneratorWriteContext context, StructureInfo si) { - WriteGetBufferPointer (variableName, "i8*", size, indent, output); - } + // $"%{typeDesignator}.{name} = type " + context.Output.Write ('%'); + context.Output.Write (si.NativeTypeDesignator); + context.Output.Write ('.'); + context.Output.Write (si.Name); + context.Output.Write (" = type "); + + if (si.IsOpaque) { + context.Output.WriteLine ("opaque"); + } else { + context.Output.WriteLine ('{'); + } - void WriteGetBufferPointer (string? variableName, string irType, ulong size, bool indent = true, TextWriter? output = null) - { - output = EnsureOutput (output); - if (indent) { - output.Write (Indent); + if (si.IsOpaque) { + return; } - if (String.IsNullOrEmpty (variableName)) { - output.Write (irType); - output.Write (" null"); - } else { - string irBaseType; - if (irType[irType.Length - 1] == '*') { - irBaseType = irType.Substring (0, irType.Length - 1); + context.IncreaseIndent (); + for (int i = 0; i < si.Members.Count; i++) { + StructureMemberInfo info = si.Members[i]; + string nativeType = MapManagedTypeToNative (info.MemberType); + + // TODO: nativeType can be an array, update to indicate that (and get the size) + string arraySize; + if (info.IsNativeArray) { + arraySize = $"[{info.ArrayElements}]"; + } else { + arraySize = String.Empty; + } + + var comment = $" {nativeType} {info.Info.Name}{arraySize}"; + WriteStructureDeclarationField (info.IRType, comment, i == si.Members.Count - 1); + } + context.DecreaseIndent (); + + context.Output.WriteLine ('}'); + + void WriteStructureDeclarationField (string typeName, string comment, bool last) + { + context.Output.Write (context.CurrentIndent); + context.Output.Write (typeName); + if (!last) { + context.Output.Write (", "); } else { - irBaseType = irType; + context.Output.Write (' '); } - // $"{irType} getelementptr inbounds ([{size} x {irBaseType}], [{size} x {irBaseType}]* @{variableName}, i32 0, i32 0)" - string sizeStr = size.ToString (CultureInfo.InvariantCulture); - output.Write (irType); - output.Write (" getelementptr inbounds (["); - output.Write (sizeStr); - output.Write (" x "); - output.Write (irBaseType); - output.Write ("], ["); - output.Write (sizeStr); - output.Write (" x "); - output.Write (irBaseType); - output.Write ("]* @"); - output.Write (variableName); - output.Write (", i32 0, i32 0)"); + if (!String.IsNullOrEmpty (comment)) { + WriteCommentLine (context, comment); + } else { + context.Output.WriteLine (); + } } } - /// - /// Write an array of name/value pairs. The array symbol will be global and non-writable. - /// - public void WriteNameValueArray (string symbolName, IDictionary arrayContents) + // + // Functions syntax: https://llvm.org/docs/LangRef.html#functions + // + void WriteFunctions (GeneratorWriteContext context) { - WriteEOL (); - WriteEOL (symbolName); + if (context.Module.Functions == null || context.Module.Functions.Count == 0) { + return; + } - var strings = new List (); - long i = 0; - ulong arrayStringCounter = 0; + context.Output.WriteLine (); + WriteComment (context, " Functions"); - foreach (var kvp in arrayContents) { - string name = kvp.Key; - string value = kvp.Value; - string iStr = i.ToString (CultureInfo.InvariantCulture); + foreach (LlvmIrFunction function in context.Module.Functions) { + context.Output.WriteLine (); + WriteFunctionComment (context, function); - WriteArrayString (name, $"n_{iStr}"); - WriteArrayString (value, $"v_{iStr}"); - i++; + // Must preserve state between calls, different targets may modify function state differently (e.g. set different parameter flags + ILlvmIrSavedFunctionState funcState = WriteFunctionPreamble (context, function, "define"); + WriteFunctionDefinitionLeadingDecorations (context, function); + WriteFunctionSignature (context, function, writeParameterNames: true); + WriteFunctionDefinitionTrailingDecorations (context, function); + WriteFunctionBody (context, function); + function.RestoreState (funcState); } + } - if (strings.Count > 0) { - Output.WriteLine (); + void WriteFunctionComment (GeneratorWriteContext context, LlvmIrFunction function) + { + if (String.IsNullOrEmpty (function.Comment)) { + return; } - WriteStringArray (symbolName, LlvmIrVariableOptions.GlobalConstantStringPointer, strings); - - void WriteArrayString (string str, string symbolSuffix) - { - StringSymbolInfo symbol = WriteUniqueString ($"__{symbolName}_{symbolSuffix}", str, ref arrayStringCounter, LlvmIrVariableOptions.LocalConstexprString); - strings.Add (new StringSymbolInfo (symbol.SymbolName, symbol.Size)); + foreach (string commentLine in function.Comment.Split ('\n')) { + context.Output.Write (context.CurrentIndent); + WriteCommentLine (context, commentLine); } } - void WriteStringArray (string symbolName, LlvmIrVariableOptions options, List strings) + void WriteFunctionBody (GeneratorWriteContext context, LlvmIrFunction function) { - WriteGlobalSymbolStart (symbolName, options); + context.Output.WriteLine (); + context.Output.WriteLine ('{'); + context.IncreaseIndent (); - // $"[{strings.Count} x i8*]" - Output.Write ('['); - Output.Write (strings.Count.ToString (CultureInfo.InvariantCulture)); - Output.Write (" x i8*]"); + foreach (LlvmIrFunctionBodyItem item in function.Body.Items) { + item.Write (context, this); + } - if (strings.Count > 0) { - Output.WriteLine (" ["); + context.DecreaseIndent (); + context.Output.WriteLine ('}'); + } - for (int j = 0; j < strings.Count; j++) { - ulong size = strings[j].Size; - string varName = strings[j].SymbolName; + ILlvmIrSavedFunctionState WriteFunctionPreamble (GeneratorWriteContext context, LlvmIrFunction function, string keyword) + { + ILlvmIrSavedFunctionState funcState = function.SaveState (); - // - // Syntax: https://llvm.org/docs/LangRef.html#getelementptr-instruction - // the two indices following {varName} have the following meanings: - // - // - The first index is into the **pointer** itself - // - The second index is into the **pointed to** value - // - // Better explained here: https://llvm.org/docs/GetElementPtr.html#id4 - // - WriteGetStringPointer (varName, size); - if (j < strings.Count - 1) { - Output.WriteLine (','); - } - } - WriteEOL (); - } else { - Output.Write (" zeroinitializer"); + foreach (LlvmIrFunctionParameter parameter in function.Signature.Parameters) { + target.SetParameterFlags (parameter); } - var arraySize = (ulong)(strings.Count * PointerSize); - if (strings.Count > 0) { - Output.Write (']'); - } - Output.Write (", align "); - Output.WriteLine (GetAggregateAlignment (PointerSize, arraySize).ToString (CultureInfo.InvariantCulture)); + WriteFunctionAttributesComment (context, function); + context.Output.Write (keyword); + context.Output.Write (' '); + + return funcState; } - /// - /// Wries a global, constant variable - /// - public void WriteVariable (string symbolName, T value) + void WriteExternalFunctionDeclarations (GeneratorWriteContext context) { - WriteVariable (symbolName, value, LlvmIrVariableOptions.GlobalConstant); + if (context.Module.ExternalFunctions == null || context.Module.ExternalFunctions.Count == 0) { + return; + } + + context.Output.WriteLine (); + WriteComment (context, " External functions"); + foreach (LlvmIrFunction function in context.Module.ExternalFunctions) { + context.Output.WriteLine (); + + // Must preserve state between calls, different targets may modify function state differently (e.g. set different parameter flags) + ILlvmIrSavedFunctionState funcState = WriteFunctionPreamble (context, function, "declare"); + WriteFunctionDeclarationLeadingDecorations (context, function); + WriteFunctionSignature (context, function, writeParameterNames: false); + WriteFunctionDeclarationTrailingDecorations (context, function); + + function.RestoreState (funcState); + } } - public void WriteVariable (string symbolName, T value, LlvmIrVariableOptions options) + void WriteFunctionAttributesComment (GeneratorWriteContext context, LlvmIrFunction func) { - if (typeof(T) == typeof(string)) { - WriteString (symbolName, (string)(object)value, options); + if (func.AttributeSet == null) { return; } - WriteEOL (); - string irType = GetIRType (out ulong size, value); - WriteGlobalSymbolStart (symbolName, options); + if (String.IsNullOrEmpty (func.Comment)) { + context.Output.WriteLine (); + } + WriteCommentLine (context, $" Function attributes: {func.AttributeSet.Render ()}"); + } - Output.Write (irType); - Output.Write (' '); - Output.Write (GetValue (value)); - Output.Write (", align "); - Output.WriteLine (size); + void WriteFunctionDeclarationLeadingDecorations (GeneratorWriteContext context, LlvmIrFunction func) + { + WriteFunctionLeadingDecorations (context, func, declaration: true); } - /// - /// Writes a private string. Strings without symbol names aren't exported, but they may be referenced by other - /// symbols - /// - public string WriteString (string value) + void WriteFunctionDefinitionLeadingDecorations (GeneratorWriteContext context, LlvmIrFunction func) { - return WriteString (value, LlvmIrVariableOptions.LocalString); + WriteFunctionLeadingDecorations (context, func, declaration: false); } - /// - /// Writes a string with automatically generated symbol name and symbol options (writeability, visibility etc) specified in the parameter. - /// - public string WriteString (string value, LlvmIrVariableOptions options) + void WriteFunctionLeadingDecorations (GeneratorWriteContext context, LlvmIrFunction func, bool declaration) { - string name = $"@.str"; - if (stringCounter > 0) { - name += $".{stringCounter.ToString (CultureInfo.InvariantCulture)}"; + if (func.Linkage != LlvmIrLinkage.Default) { + context.Output.Write (llvmLinkage[func.Linkage]); + context.Output.Write (' '); + } + + if (!declaration && func.RuntimePreemption != LlvmIrRuntimePreemption.Default) { + context.Output.Write (llvmRuntimePreemption[func.RuntimePreemption]); + context.Output.Write (' '); + } + + if (func.Visibility != LlvmIrVisibility.Default) { + context.Output.Write (llvmVisibility[func.Visibility]); + context.Output.Write (' '); } - stringCounter++; - return WriteString (name, value, options); } - /// - /// Writes a global, C++ constexpr style string - /// - public string WriteString (string symbolName, string value) + void WriteFunctionDeclarationTrailingDecorations (GeneratorWriteContext context, LlvmIrFunction func) { - return WriteString (symbolName, value, LlvmIrVariableOptions.GlobalConstexprString); + WriteFunctionTrailingDecorations (context, func, declaration: true); } - /// - /// Writes a string with symbol options (writeability, visibility) options specified in the parameter. - /// - public string WriteString (string symbolName, string value, LlvmIrVariableOptions options) + void WriteFunctionDefinitionTrailingDecorations (GeneratorWriteContext context, LlvmIrFunction func) { - return WriteString (symbolName, value, options, out _); + WriteFunctionTrailingDecorations (context, func, declaration: false); } - /// - /// Writes a local, constexpr style string and returns its size in - /// - public string WriteString (string symbolName, string value, out ulong stringSize) + void WriteFunctionTrailingDecorations (GeneratorWriteContext context, LlvmIrFunction func, bool declaration) { - return WriteString (symbolName, value, LlvmIrVariableOptions.LocalConstexprString, out stringSize); + if (func.AddressSignificance != LlvmIrAddressSignificance.Default) { + context.Output.Write ($" {llvmAddressSignificance[func.AddressSignificance]}"); + } + + if (func.AttributeSet != null) { + context.Output.Write ($" #{func.AttributeSet.Number}"); + } } - /// - /// Writes a string with specified , and symbol options (writeability, visibility etc) specified in the - /// parameter. Returns string size (in bytes) in - /// - public string WriteString (string symbolName, string value, LlvmIrVariableOptions options, out ulong stringSize) + public static void WriteReturnAttributes (GeneratorWriteContext context, LlvmIrFunctionSignature.ReturnTypeAttributes returnAttrs) { - string strSymbolName; - bool global = options.IsGlobal; - if (global) { - strSymbolName = $"__{symbolName}"; - } else { - strSymbolName = symbolName; + if (AttributeIsSet (returnAttrs.NoUndef)) { + context.Output.Write ("noundef "); } - string quotedString = QuoteString (value, out stringSize); - - // It might seem counter-intuitive that when we're requested to write a global string, here we generate a **local** one, - // but global strings are actually pointers to local storage. - WriteGlobalSymbolStart (strSymbolName, global ? LlvmIrVariableOptions.LocalConstexprString : options); + if (AttributeIsSet (returnAttrs.SignExt)) { + context.Output.Write ("signext "); + } - string stringSizeStr = stringSize.ToString (CultureInfo.InvariantCulture); - // WriteLine $"[{stringSize} x i8] c{quotedString}, align {GetAggregateAlignment (1, stringSize)}" - Output.Write ('['); - Output.Write (stringSizeStr); - Output.Write (" x i8] c"); - Output.Write (quotedString); - Output.Write (", align "); - Output.WriteLine (GetAggregateAlignment (1, stringSize).ToString (CultureInfo.InvariantCulture)); + if (AttributeIsSet (returnAttrs.ZeroExt)) { + context.Output.Write ("zeroext "); + } + } - if (!global) { - return symbolName; + void WriteFunctionSignature (GeneratorWriteContext context, LlvmIrFunction func, bool writeParameterNames) + { + if (func.ReturnsValue) { + WriteReturnAttributes (context, func.Signature.ReturnAttributes); } - string indexType = Is64Bit ? "i64" : "i32"; - WriteGlobalSymbolStart (symbolName, LlvmIrVariableOptions.GlobalConstantStringPointer); + context.Output.Write (MapToIRType (func.Signature.ReturnType)); + context.Output.Write (" @"); + context.Output.Write (func.Signature.Name); + context.Output.Write ('('); - // WriteLine $"i8* getelementptr inbounds ([{stringSize} x i8], [{stringSize} x i8]* @{strSymbolName}, {indexType} 0, {indexType} 0), align {GetAggregateAlignment (PointerSize, stringSize)}" - Output.Write ("i8* getelementptr inbounds (["); - Output.Write (stringSizeStr); - Output.Write (" x i8], ["); - Output.Write (stringSizeStr); - Output.Write (" x i8]* @"); - Output.Write (strSymbolName); - Output.Write (", "); - Output.Write (indexType); - Output.Write (" 0, "); - Output.Write (indexType); - Output.Write (" 0), align "); - Output.WriteLine (GetAggregateAlignment (PointerSize, stringSize).ToString (CultureInfo.InvariantCulture)); + bool first = true; + foreach (LlvmIrFunctionParameter parameter in func.Signature.Parameters) { + if (!first) { + context.Output.Write (", "); + } else { + first = false; + } - return symbolName; - } + context.Output.Write (MapToIRType (parameter.Type)); + WriteParameterAttributes (context, parameter); + if (writeParameterNames) { + if (String.IsNullOrEmpty (parameter.Name)) { + throw new InvalidOperationException ($"Internal error: parameter must have a name"); + } + context.Output.Write (" %"); // Function arguments are always local variables + context.Output.Write (parameter.Name); + } + } - /// - /// Writes a string, creating a new symbol if the is unique or returns name of a previously created symbol with the same - /// string value. If a new symbol is written, its name is constructed by combining prefix () with value - /// of a string counter referenced by the parameter. Symbol is created as a local, C++ constexpr style string. - /// - public StringSymbolInfo WriteUniqueString (string potentialSymbolName, string value, ref ulong counter) - { - return WriteUniqueString (potentialSymbolName, value, ref counter, LlvmIrVariableOptions.LocalConstexprString); + context.Output.Write (')'); } - /// - /// Writes a string, creating a new symbol if the is unique or returns name of a previously created symbol with the same - /// string value. If a new symbol is written, its name is constructed by combining prefix () with value - /// of a string counter referenced by the parameter. Symbol options (writeability, visibility etc) are specified in the parameter. String size (in bytes) is returned in . - /// - public StringSymbolInfo WriteUniqueString (string potentialSymbolNamePrefix, string value, ref ulong counter, LlvmIrVariableOptions options) + public static void WriteParameterAttributes (GeneratorWriteContext context, LlvmIrFunctionParameter parameter) { - if (value == null) { - return null; + var attributes = new List (); + if (AttributeIsSet (parameter.ImmArg)) { + attributes.Add ("immarg"); } - StringSymbolInfo info; - if (stringSymbolCache.TryGetValue (value, out info)) { - return info; + if (AttributeIsSet (parameter.AllocPtr)) { + attributes.Add ("allocptr"); } - string newSymbolName = $"{potentialSymbolNamePrefix}.{counter.ToString (CultureInfo.InvariantCulture)}"; - counter++; + if (AttributeIsSet (parameter.NoCapture)) { + attributes.Add ("nocapture"); + } - WriteString (newSymbolName, value, options, out ulong stringSize); - info = new StringSymbolInfo (newSymbolName, stringSize); - stringSymbolCache.Add (value, info); + if (AttributeIsSet (parameter.NonNull)) { + attributes.Add ("nonnull"); + } - return info; - } + if (AttributeIsSet (parameter.NoUndef)) { + attributes.Add ("noundef"); + } - public virtual void WriteFileTop () - { - WriteCommentLine ($"ModuleID = '{fileName}'"); - WriteDirective ("source_filename", QuoteStringNoEscape (fileName)); - WriteDirective ("target datalayout", QuoteStringNoEscape (DataLayout)); - WriteDirective ("target triple", QuoteStringNoEscape (Triple)); - } + if (AttributeIsSet (parameter.ReadNone)) { + attributes.Add ("readnone"); + } - public virtual void WriteFileEnd () - { - Output.WriteLine (); + if (AttributeIsSet (parameter.SignExt)) { + attributes.Add ("signext"); + } + + if (AttributeIsSet (parameter.ZeroExt)) { + attributes.Add ("zeroext"); + } + + if (parameter.Align.HasValue) { + attributes.Add ($"align({ValueOrPointerSize (parameter.Align.Value)})"); + } + + if (parameter.Dereferenceable.HasValue) { + attributes.Add ($"dereferenceable({ValueOrPointerSize (parameter.Dereferenceable.Value)})"); + } + + if (attributes.Count == 0) { + return; + } - WriteAttributeSets (); + context.Output.Write (' '); + context.Output.Write (String.Join (" ", attributes)); + + uint ValueOrPointerSize (uint? value) + { + if (value.Value == 0) { + return context.Target.NativePointerSize; + } - foreach (LlvmIrMetadataItem metadata in MetadataManager.Items) { - Output.WriteLine (metadata.Render ()); + return value.Value; } } - public void WriteStructureDeclarations () + static bool AttributeIsSet (bool? attr) => attr.HasValue && attr.Value; + + void WriteAttributeSets (GeneratorWriteContext context) { - if (structures.Count == 0) { + if (context.Module.AttributeSets == null || context.Module.AttributeSets.Count == 0) { return; } - Output.WriteLine (); - foreach (IStructureInfo si in structures) { - si.RenderDeclaration (this); + context.Output.WriteLine (); + foreach (LlvmIrFunctionAttributeSet attrSet in context.Module.AttributeSets) { + // Must not modify the original set, it is shared with other targets. + var targetSet = new LlvmIrFunctionAttributeSet (attrSet); + target.AddTargetSpecificAttributes (targetSet); + + IList? privateTargetSet = attrSet.GetPrivateTargetAttributes (target.TargetArch); + if (privateTargetSet != null) { + targetSet.Add (privateTargetSet); + } + + context.Output.WriteLine ($"attributes #{targetSet.Number} = {{ {targetSet.Render ()} }}"); } } - public void WriteStructureDeclarationStart (string typeDesignator, string name, bool forOpaqueType = false) + void WriteMetadata (GeneratorWriteContext context) { - WriteEOL (); + if (context.MetadataManager.Items.Count == 0) { + return; + } - // $"%{typeDesignator}.{name} = type " - Output.Write ('%'); - Output.Write (typeDesignator); - Output.Write ('.'); - Output.Write (name); - Output.Write (" = type "); - - if (forOpaqueType) { - Output.WriteLine ("opaque"); - } else { - Output.WriteLine ("{"); + context.Output.WriteLine (); + WriteCommentLine (context, " Metadata"); + foreach (LlvmIrMetadataItem metadata in context.MetadataManager.Items) { + context.Output.WriteLine (metadata.Render ()); } } - public void WriteStructureDeclarationEnd () + public void WriteComment (GeneratorWriteContext context, string comment) { - Output.WriteLine ('}'); + context.Output.Write (';'); + context.Output.Write (comment); } - public void WriteStructureDeclarationField (string typeName, string comment, bool last) + public void WriteCommentLine (GeneratorWriteContext context, string comment) { - Output.Write (Indent); - Output.Write (typeName); - if (!last) { - Output.Write (","); - } - - if (!String.IsNullOrEmpty (comment)) { - WriteCommentLine (comment); - } else { - WriteEOL (); - } + WriteComment (context, comment); + context.Output.WriteLine (); } - protected virtual void AddModuleFlagsMetadata (List flagsFields) + static Type GetActualType (Type type) { - flagsFields.Add (MetadataManager.AddNumbered (LlvmIrModuleMergeBehavior.Error, "wchar_size", 4)); - flagsFields.Add (MetadataManager.AddNumbered (LlvmIrModuleMergeBehavior.Max, "PIC Level", 2)); + // Arrays of types are handled elsewhere, so we obtain the array base type here + if (type.IsArray) { + return type.GetElementType (); + } + + return type; } - // Alignment for arrays, structures and unions - protected virtual int GetAggregateAlignment (int maxFieldAlignment, ulong dataSize) + /// + /// Map a managed to its C++ counterpart. Only primitive types, + /// string and IntPtr are supported. + /// + public static string MapManagedTypeToNative (Type type) { - return maxFieldAlignment; + Type baseType = GetActualType (type); + + if (baseType == typeof (bool)) return "bool"; + if (baseType == typeof (byte)) return "uint8_t"; + if (baseType == typeof (char)) return "char"; + if (baseType == typeof (sbyte)) return "int8_t"; + if (baseType == typeof (short)) return "int16_t"; + if (baseType == typeof (ushort)) return "uint16_t"; + if (baseType == typeof (int)) return "int32_t"; + if (baseType == typeof (uint)) return "uint32_t"; + if (baseType == typeof (long)) return "int64_t"; + if (baseType == typeof (ulong)) return "uint64_t"; + if (baseType == typeof (float)) return "float"; + if (baseType == typeof (double)) return "double"; + if (baseType == typeof (string)) return "char*"; + if (baseType == typeof (IntPtr)) return "void*"; + + return type.GetShortName (); } - public void WriteCommentLine (string? comment = null, bool indent = false) + static string MapManagedTypeToNative (StructureMemberInfo smi) { - WriteCommentLine (Output, comment, indent); + string nativeType = MapManagedTypeToNative (smi.MemberType); + // Silly, but effective + if (nativeType[nativeType.Length - 1] == '*') { + return nativeType; + } + + if (!smi.IsNativePointer) { + return nativeType; + } + + return $"{nativeType}*"; } - public void WriteComment (TextWriter writer, string? comment = null, bool indent = false) + static bool IsNumeric (Type type) => basicTypeMap.TryGetValue (type, out BasicType typeDesc) && typeDesc.IsNumeric; + + object? GetTypedMemberValue (GeneratorWriteContext context, StructureInfo info, StructureMemberInfo smi, StructureInstance instance, Type expectedType, object? defaultValue = null) { - if (indent) { - writer.Write (Indent); + object? value = smi.GetValue (instance.Obj); + if (value == null) { + return defaultValue; } - writer.Write (';'); + Type valueType = value.GetType (); + if (valueType != expectedType) { + throw new InvalidOperationException ($"Field '{smi.Info.Name}' of structure '{info.Name}' should have a value of '{expectedType}' type, instead it had a '{value.GetType ()}'"); + } - if (!String.IsNullOrEmpty (comment)) { - writer.Write (' '); - writer.Write (comment); + if (valueType == typeof(string)) { + return context.Module.LookupRequiredVariableForString ((string)value); } + + return value; } - public void WriteCommentLine (TextWriter writer, string? comment = null, bool indent = false) + public static string MapToIRType (Type type) { - WriteComment (writer, comment, indent); - writer.WriteLine (); + return MapToIRType (type, out _, out _); } - public void WriteEOL (string? comment = null, TextWriter? output = null) + public static string MapToIRType (Type type, out ulong size) { - WriteEOL (EnsureOutput (output), comment); + return MapToIRType (type, out size, out _); } - public void WriteEOL (TextWriter writer, string? comment = null) + public static string MapToIRType (Type type, out bool isPointer) { - if (!String.IsNullOrEmpty (comment)) { - WriteCommentLine (writer, comment); - return; - } - writer.WriteLine (); + return MapToIRType (type, out _, out isPointer); } - public void WriteDirectiveWithComment (TextWriter writer, string name, string? comment, string? value) + /// + /// Maps managed type to equivalent IR type. Puts type size in and whether or not the type + /// is a pointer in . When a type is determined to be a pointer, + /// will be set to 0, because this method doesn't have access to the generator target. In order to adjust pointer + /// size, the instance method must be called (private to the generator as other classes should not + /// have any need to know the pointer size). + /// + public static string MapToIRType (Type type, out ulong size, out bool isPointer) { - writer.Write (name); - - if (!String.IsNullOrEmpty (value)) { - writer.Write (" = "); - writer.Write (value); + type = GetActualType (type); + if (!type.IsNativePointer () && basicTypeMap.TryGetValue (type, out BasicType typeDesc)) { + size = typeDesc.Size; + isPointer = false; + return typeDesc.Name; } - WriteEOL (writer, comment); + // if it's not a basic type, then it's an opaque pointer + size = 0; // Will be determined by the specific target architecture class + isPointer = true; + return IRPointerType; } - public void WriteDirectiveWithComment (string name, string? comment, string? value) + string GetIRType (Type type, out ulong size, out bool isPointer) { - WriteDirectiveWithComment (name, comment, value); - } + string ret = MapToIRType (type, out size, out isPointer); + if (isPointer && size == 0) { + size = target.NativePointerSize; + } - public void WriteDirective (TextWriter writer, string name, string? value) - { - WriteDirectiveWithComment (writer, name, comment: null, value: value); + return ret; } - public void WriteDirective (string name, string value) + public static bool IsFirstClassNonPointerType (Type type) { - WriteDirective (Output, name, value); + if (type == typeof(void)) { + return false; + } + + return basicTypeMap.ContainsKey (type); } public static string QuoteStringNoEscape (string s) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrInstructions.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrInstructions.cs new file mode 100644 index 00000000000..2efa5e45db8 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrInstructions.cs @@ -0,0 +1,479 @@ +using System; +using System.Collections.Generic; +using System.Globalization; + +namespace Xamarin.Android.Tasks.LLVMIR; + +abstract class LlvmIrInstruction : LlvmIrFunctionBodyItem +{ + public string Mnemonic { get; } + public LlvmIrFunctionAttributeSet? AttributeSet { get; set; } + + /// + /// TBAA (Type Based Alias Analysis) metadata item the instruction references, if any. + /// for more information about TBAA. + /// + public LlvmIrMetadataItem? TBAA { get; set; } + + protected LlvmIrInstruction (string mnemonic) + { + if (String.IsNullOrEmpty (mnemonic)) { + throw new ArgumentException ("must not be null or empty", nameof (mnemonic)); + } + + Mnemonic = mnemonic; + } + + protected override void DoWrite (GeneratorWriteContext context, LlvmIrGenerator generator) + { + context.Output.Write (context.CurrentIndent); + WriteValueAssignment (context); + WritePreamble (context); + context.Output.Write (Mnemonic); + context.Output.Write (' '); + WriteBody (context); + + if (TBAA != null) { + context.Output.Write (", !tbaa !"); + context.Output.Write (TBAA.Name); + } + + if (AttributeSet != null) { + context.Output.Write (" #"); + context.Output.Write (AttributeSet.Number.ToString (CultureInfo.InvariantCulture)); + } + + if (!String.IsNullOrEmpty (Comment)) { + generator.WriteComment (context, Comment); + } + } + + /// + /// Write the '<variable_reference> = ' part of the instruction line. + /// + protected virtual void WriteValueAssignment (GeneratorWriteContext context) + {} + + /// + /// Write part of the instruction that comes between the optional value assignment and the instruction + /// mnemonic. If any text is written, it must end with a whitespace. + /// + protected virtual void WritePreamble (GeneratorWriteContext context) + {} + + /// + /// Write the "body" of the instruction, that is the part that follows instruction mnemonic but precedes the + /// metadata and attribute set references. + /// + protected virtual void WriteBody (GeneratorWriteContext context) + {} + + protected void WriteValue (GeneratorWriteContext context, Type type, object? value, bool isPointer) + { + if (value == null) { + if (!isPointer) { + throw new InvalidOperationException ($"Internal error: non-pointer type '{type}' must not have a `null` value"); + } + context.Output.Write ("null"); + } else if (value is LlvmIrVariable variable) { + context.Output.Write (variable.Reference); + } else { + context.Output.Write (MonoAndroidHelper.CultureInvariantToString (value)); + } + } + + protected void WriteAlignment (GeneratorWriteContext context, ulong typeSize, bool isPointer) + { + context.Output.Write (", align "); + + ulong alignment; + if (isPointer) { + alignment = context.Target.NativePointerSize; + } else { + alignment = typeSize; + } + context.Output.Write (alignment.ToString (CultureInfo.InvariantCulture)); + } +} + +abstract class LlvmIrInstructionArgumentValuePlaceholder +{ + protected LlvmIrInstructionArgumentValuePlaceholder () + {} + + public abstract object? GetValue (LlvmIrModuleTarget target); +} + +sealed class LlvmIrInstructions +{ + public class Br : LlvmIrInstruction + { + const string OpName = "br"; + + LlvmIrVariable? cond; + LlvmIrFunctionLabelItem ifTrue; + LlvmIrFunctionLabelItem? ifFalse; + + /// + /// Outputs a conditional branch to label if condition is + /// true, and to label otherwise. must be a variable + /// of type bool + /// + public Br (LlvmIrVariable cond, LlvmIrFunctionLabelItem ifTrue, LlvmIrFunctionLabelItem ifFalse) + : base (OpName) + { + if (cond.Type != typeof(bool)) { + throw new ArgumentException ($"Internal error: condition must refer to a variable of type 'bool', was 'cond.Type' instead", nameof (cond)); + } + + this.cond = cond; + this.ifTrue = ifTrue; + this.ifFalse = ifFalse; + } + + /// + /// Outputs an unconditional branch to label + /// + public Br (LlvmIrFunctionLabelItem label) + : base (OpName) + { + ifTrue = label; + } + + protected override void WriteBody (GeneratorWriteContext context) + { + if (cond == null) { + context.Output.Write ("label %"); + context.Output.Write (ifTrue.Name); + return; + } + + context.Output.Write ("i1 "); + context.Output.Write (cond.Reference); + context.Output.Write (", label %"); + context.Output.Write (ifTrue.Name); + context.Output.Write (", label %"); + context.Output.Write (ifFalse.Name); + } + } + + public class Call : LlvmIrInstruction + { + LlvmIrFunction function; + IList? arguments; + LlvmIrVariable? result; + + public LlvmIrCallMarker CallMarker { get; set; } = LlvmIrCallMarker.None; + + /// + /// This needs to be set if we want to call a function via a local or global variable. passed + /// to the constructor is then used only to generate a type safe call, while function address comes from the variable assigned + /// to this property. + /// + public LlvmIrVariable? FuncPointer { get; set; } + + public Call (LlvmIrFunction function, LlvmIrVariable? result = null, ICollection? arguments = null, LlvmIrVariable? funcPointer = null) + : base ("call") + { + this.function = function; + this.result = result; + this.FuncPointer = funcPointer; + + if (function.Signature.ReturnType != typeof(void)) { + if (result == null) { + throw new ArgumentNullException ($"Internal error: function '{function.Signature.Name}' returns '{function.Signature.ReturnType} and thus requires a result variable", nameof (result)); + } + } else if (result != null) { + throw new ArgumentException ($"Internal error: function '{function.Signature.Name}' returns no value and yet a result variable was provided", nameof (result)); + } + + int argCount = function.Signature.Parameters.Count; + if (argCount != 0) { + if (arguments == null) { + throw new ArgumentNullException ($"Internal error: function '{function.Signature.Name}' requires {argCount} arguments", nameof (arguments)); + } + + if (arguments.Count != argCount) { + throw new ArgumentException ($"Internal error: function '{function.Signature.Name}' requires {argCount} arguments, but {arguments.Count} were provided", nameof (arguments)); + } + + this.arguments = new List (arguments).AsReadOnly (); + } + } + + protected override void WriteValueAssignment (GeneratorWriteContext context) + { + if (result == null) { + return; + } + + context.Output.Write (result.Reference); + context.Output.Write (" = "); + } + + protected override void WritePreamble (GeneratorWriteContext context) + { + string? callMarker = CallMarker switch { + LlvmIrCallMarker.None => null, + LlvmIrCallMarker.Tail => "tail", + LlvmIrCallMarker.NoTail => "notail", + LlvmIrCallMarker.MustTail => "musttail", + _ => throw new InvalidOperationException ($"Internal error: call marker '{CallMarker}' not supported"), + }; + + if (!String.IsNullOrEmpty (callMarker)) { + context.Output.Write (callMarker); + context.Output.Write (' '); + } + } + + protected override void WriteBody (GeneratorWriteContext context) + { + if (function.ReturnsValue) { + LlvmIrGenerator.WriteReturnAttributes (context, function.Signature.ReturnAttributes); + } + + context.Output.Write (LlvmIrGenerator.MapToIRType (function.Signature.ReturnType)); + if (FuncPointer == null) { + context.Output.Write (" @"); + context.Output.Write (function.Signature.Name); + } else { + context.Output.Write (' '); + context.Output.Write (FuncPointer.Reference); + } + context.Output.Write ('('); + + for (int i = 0; i < function.Signature.Parameters.Count; i++) { + if (i > 0) { + context.Output.Write (", "); + } + + WriteArgument (context, function.Signature.Parameters[i], i); + } + + context.Output.Write (')'); + } + + void WriteArgument (GeneratorWriteContext context, LlvmIrFunctionParameter parameter, int index) + { + context.Output.Write (LlvmIrGenerator.MapToIRType (parameter.Type)); + LlvmIrGenerator.WriteParameterAttributes (context, parameter); + context.Output.Write (' '); + + object? value = arguments[index]; + if (value is LlvmIrInstructionArgumentValuePlaceholder placeholder) { + value = placeholder.GetValue (context.Target); + } + + if (value == null) { + if (!parameter.Type.IsNativePointer ()) { + throw new InvalidOperationException ($"Internal error: value for argument {index} to function '{function.Signature.Name}' must not be null"); + } + + context.Output.Write ("null"); + return; + } + + if (value is LlvmIrVariable variable) { + context.Output.Write (variable.Reference); + return; + } + + if (!parameter.Type.IsAssignableFrom (value.GetType ())) { + throw new InvalidOperationException ($"Internal error: value type '{value.GetType ()}' for argument {index} to function '{function.Signature.Name}' is invalid. Expected '{parameter.Type}' or compatible"); + } + + if (value is string str) { + context.Output.Write (context.Module.LookupRequiredVariableForString (str).Reference); + return; + } + + if (LlvmIrGenerator.IsFirstClassNonPointerType (value.GetType ())) { + context.Output.Write (MonoAndroidHelper.CultureInvariantToString (value)); + return; + } + + throw new InvalidOperationException ($"Internal error: unsupported type '{value.GetType ()}' in call to function '{function.Signature.Name}'"); + } + } + + public class Icmp : LlvmIrInstruction + { + LlvmIrIcmpCond cond; + LlvmIrVariable op1; + object? op2; + LlvmIrVariable result; + + public Icmp (LlvmIrIcmpCond cond, LlvmIrVariable op1, object? op2, LlvmIrVariable result) + : base ("icmp") + { + if (result.Type != typeof(bool)) { + throw new ArgumentException ($"Internal error: result must be a variable of type 'bool', was '{result.Type}' instead", nameof (result)); + } + + this.cond = cond; + this.op1 = op1; + this.op2 = op2; + this.result = result; + } + + protected override void WriteValueAssignment (GeneratorWriteContext context) + { + context.Output.Write (result.Reference); + context.Output.Write (" = "); + } + + protected override void WriteBody (GeneratorWriteContext context) + { + string irType = LlvmIrGenerator.MapToIRType (op1.Type, out ulong size, out bool isPointer); + string condOp = cond switch { + LlvmIrIcmpCond.Equal => "eq", + LlvmIrIcmpCond.NotEqual => "ne", + LlvmIrIcmpCond.UnsignedGreaterThan => "ugt", + LlvmIrIcmpCond.UnsignedGreaterOrEqual => "uge", + LlvmIrIcmpCond.UnsignedLessThan => "ult", + LlvmIrIcmpCond.UnsignedLessOrEqual => "ule", + LlvmIrIcmpCond.SignedGreaterThan => "sgt", + LlvmIrIcmpCond.SignedGreaterOrEqual => "sge", + LlvmIrIcmpCond.SignedLessThan => "slt", + LlvmIrIcmpCond.SignedLessOrEqual => "sle", + _ => throw new InvalidOperationException ($"Unsupported `icmp` conditional '{cond}'"), + }; + + context.Output.Write (condOp); + context.Output.Write (' '); + context.Output.Write (irType); + context.Output.Write (' '); + context.Output.Write (op1.Reference); + context.Output.Write (", "); + WriteValue (context, op1.Type, op2, isPointer); + } + } + + public class Load : LlvmIrInstruction + { + LlvmIrVariable source; + LlvmIrVariable result; + + public Load (LlvmIrVariable source, LlvmIrVariable result) + : base ("load") + { + this.source = source; + this.result = result; + } + + protected override void WriteValueAssignment (GeneratorWriteContext context) + { + context.Output.Write (result.Reference); + context.Output.Write (" = "); + } + + protected override void WriteBody (GeneratorWriteContext context) + { + string irType = LlvmIrGenerator.MapToIRType (result.Type, out ulong size, out bool isPointer); + context.Output.Write (irType); + context.Output.Write (", ptr "); + WriteValue (context, result.Type, source, isPointer); + WriteAlignment (context, size, isPointer); + } + } + + public class Phi : LlvmIrInstruction + { + LlvmIrVariable result; + LlvmIrVariable val1; + LlvmIrFunctionLabelItem label1; + LlvmIrVariable val2; + LlvmIrFunctionLabelItem label2; + + /// + /// Represents the `phi` instruction form we use the most throughout marshal methods generator - one which refers to an if/else block and where + /// **both** value:label pairs are **required**. Parameters and are nullable because, in theory, + /// it is possible that hasn't had the required blocks defined prior to adding the `phi` instruction and, thus, + /// we must check for the possibility here. + /// + public Phi (LlvmIrVariable result, LlvmIrVariable val1, LlvmIrFunctionLabelItem? label1, LlvmIrVariable val2, LlvmIrFunctionLabelItem? label2) + : base ("phi") + { + this.result = result; + this.val1 = val1; + this.label1 = label1 ?? throw new ArgumentNullException (nameof (label1)); + this.val2 = val2; + this.label2 = label2 ?? throw new ArgumentNullException (nameof (label2)); + } + + protected override void WriteValueAssignment (GeneratorWriteContext context) + { + context.Output.Write (result.Reference); + context.Output.Write (" = "); + } + + protected override void WriteBody (GeneratorWriteContext context) + { + context.Output.Write (LlvmIrGenerator.MapToIRType (result.Type)); + context.Output.Write (" ["); + context.Output.Write (val1.Reference); + context.Output.Write (", %"); + context.Output.Write (label1.Name); + context.Output.Write ("], ["); + context.Output.Write (val2.Reference); + context.Output.Write (", %"); + context.Output.Write (label2.Name); + context.Output.Write (']'); + } + } + + public class Ret : LlvmIrInstruction + { + Type retvalType; + object? retVal; + + public Ret (Type retvalType, object? retval = null) + : base ("ret") + { + this.retvalType = retvalType; + retVal = retval; + } + + protected override void WriteBody (GeneratorWriteContext context) + { + if (retvalType == typeof(void)) { + context.Output.Write ("void"); + return; + } + + string irType = LlvmIrGenerator.MapToIRType (retvalType, out bool isPointer); + context.Output.Write (irType); + context.Output.Write (' '); + + WriteValue (context, retvalType, retVal, isPointer); + } + } + + public class Store : LlvmIrInstruction + { + object? from; + LlvmIrVariable to; + + public Store (LlvmIrVariable from, LlvmIrVariable to) + : base ("store") + { + this.from = from; + this.to = to; + } + + protected override void WriteBody (GeneratorWriteContext context) + { + string irType = LlvmIrGenerator.MapToIRType (to.Type, out ulong size, out bool isPointer); + context.Output.Write (irType); + context.Output.Write (' '); + + WriteValue (context, to.Type, from, isPointer); + + context.Output.Write (", ptr "); + context.Output.Write (to.Reference); + + WriteAlignment (context, size, isPointer); + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrKnownMetadata.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrKnownMetadata.cs new file mode 100644 index 00000000000..dbd6ce0f69c --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrKnownMetadata.cs @@ -0,0 +1,7 @@ +namespace Xamarin.Android.Tasks.LLVMIR; + +sealed class LlvmIrKnownMetadata +{ + public const string LlvmModuleFlags = "llvm.module.flags"; + public const string LlvmIdent = "llvm.ident"; +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrMetadataManager.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrMetadataManager.cs index b2d8bf7379a..37acb17ce44 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrMetadataManager.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrMetadataManager.cs @@ -10,6 +10,12 @@ class LlvmIrMetadataField public string Contents { get; } public bool IsReference { get; } + public LlvmIrMetadataField (LlvmIrMetadataField other) + { + Contents = other.Contents; + IsReference = other.IsReference; + } + public LlvmIrMetadataField (string value, bool isReference = false) { if (isReference) { @@ -35,7 +41,7 @@ string FormatValue (object value) return QuoteString ((string)value); } - string irType = LlvmIrGenerator.MapManagedTypeToIR (vt); + string irType = LlvmIrGenerator.MapToIRType (vt); return $"{irType} {MonoAndroidHelper.CultureInvariantToString (value)}"; } @@ -51,6 +57,15 @@ class LlvmIrMetadataItem public string Name { get; } + public LlvmIrMetadataItem (LlvmIrMetadataItem other) + { + Name = other.Name; + fields = new List (); + foreach (LlvmIrMetadataField field in other.fields) { + fields.Add (new LlvmIrMetadataField (field)); + } + } + public LlvmIrMetadataItem (string name) { if (name.Length == 0) { @@ -66,9 +81,19 @@ public void AddReferenceField (string referenceName) fields.Add (new LlvmIrMetadataField (referenceName, isReference: true)); } + public void AddReferenceField (LlvmIrMetadataItem referencedItem) + { + AddReferenceField (referencedItem.Name); + } + public void AddField (object value) { - fields.Add (new LlvmIrMetadataField (value)); + AddField (new LlvmIrMetadataField (value)); + } + + public void AddField (LlvmIrMetadataField field) + { + fields.Add (field); } public string Render () @@ -96,11 +121,29 @@ class LlvmIrMetadataManager { ulong counter = 0; List items = new List (); + Dictionary nameToItem = new Dictionary (StringComparer.Ordinal); public List Items => items; + public LlvmIrMetadataManager () + {} + + public LlvmIrMetadataManager (LlvmIrMetadataManager other) + { + foreach (LlvmIrMetadataItem item in other.items) { + var newItem = new LlvmIrMetadataItem (item); + items.Add (newItem); + nameToItem.Add (newItem.Name, newItem); + } + counter = other.counter; + } + public LlvmIrMetadataItem Add (string name, params object[]? values) { + if (nameToItem.ContainsKey (name)) { + throw new InvalidOperationException ($"Internal error: metadata item '{name}' has already been added"); + } + var ret = new LlvmIrMetadataItem (name); if (values != null && values.Length > 0) { @@ -110,6 +153,7 @@ public LlvmIrMetadataItem Add (string name, params object[]? values) } items.Add (ret); + nameToItem.Add (name, ret); return ret; } @@ -119,5 +163,14 @@ public LlvmIrMetadataItem AddNumbered (params object[]? values) counter++; return Add (name, values); } + + public LlvmIrMetadataItem? GetItem (string name) + { + if (nameToItem.TryGetValue (name, out LlvmIrMetadataItem? item)) { + return item; + } + + return null; + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModule.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModule.cs new file mode 100644 index 00000000000..f7094dbbd42 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModule.cs @@ -0,0 +1,561 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks.LLVMIR +{ + partial class LlvmIrModule + { + /// + /// Global variable type to be used to output name:value string arrays. This is a notational shortcut, + /// do **NOT** change the type without understanding how it affects the rest of code. + /// + public static readonly Type NameValueArrayType = typeof(IDictionary); + + public IList? ExternalFunctions { get; private set; } + public IList? Functions { get; private set; } + public IList? AttributeSets { get; private set; } + public IList? Structures { get; private set; } + public IList? GlobalVariables { get; private set; } + public IList? Strings { get; private set; } + + /// + /// TBAA stands for "Type Based Alias Analysis" and is used by LLVM to implemente a description of + /// a higher level language typesystem to LLVM IR (in which memory doesn't have types). This metadata + /// item describes pointer usage for certain instructions we output and is common enough to warrant + /// a shortcut property like that. More information about TBAA can be found at https://llvm.org/docs/LangRef.html#tbaa-metadata + /// + public LlvmIrMetadataItem TbaaAnyPointer => tbaaAnyPointer; + + Dictionary? attributeSets; + Dictionary? externalFunctions; + Dictionary? functions; + Dictionary? structures; + LlvmIrStringManager? stringManager; + LlvmIrMetadataManager metadataManager; + LlvmIrMetadataItem tbaaAnyPointer; + LlvmIrBufferManager? bufferManager; + + List? globalVariables; + + public LlvmIrModule () + { + metadataManager = new LlvmIrMetadataManager (); + + // Only model agnostic items can be added here + LlvmIrMetadataItem flags = metadataManager.Add (LlvmIrKnownMetadata.LlvmModuleFlags); + flags.AddReferenceField (metadataManager.AddNumbered (LlvmIrModuleMergeBehavior.Error, "wchar_size", 4)); + flags.AddReferenceField (metadataManager.AddNumbered (LlvmIrModuleMergeBehavior.Max, "PIC Level", 2)); + + LlvmIrMetadataItem ident = metadataManager.Add (LlvmIrKnownMetadata.LlvmIdent); + LlvmIrMetadataItem identValue = metadataManager.AddNumbered ($"Xamarin.Android {XABuildConfig.XamarinAndroidBranch} @ {XABuildConfig.XamarinAndroidCommitHash}"); + ident.AddReferenceField (identValue.Name); + + tbaaAnyPointer = metadataManager.AddNumbered (); + LlvmIrMetadataItem anyPointer = metadataManager.AddNumbered ("any pointer"); + LlvmIrMetadataItem omnipotentChar = metadataManager.AddNumbered ("omnipotent char"); + LlvmIrMetadataItem simpleCppTBAA = metadataManager.AddNumbered ("Simple C++ TBAA"); + + anyPointer.AddReferenceField (omnipotentChar.Name); + anyPointer.AddField ((ulong)0); + + omnipotentChar.AddReferenceField (simpleCppTBAA); + omnipotentChar.AddField ((ulong)0); + + tbaaAnyPointer.AddReferenceField (anyPointer); + tbaaAnyPointer.AddReferenceField (anyPointer); + tbaaAnyPointer.AddField ((ulong)0); + } + + /// + /// Return a metadata manager instance which includes copies of all the target-agnostic metadata items. + /// We must not modify the original manager since each target may have conflicting values for certain + /// flags. + /// + public LlvmIrMetadataManager GetMetadataManagerCopy () => new LlvmIrMetadataManager (metadataManager); + + /// + /// Perform any tasks that need to be done after construction is complete. + /// + public void AfterConstruction () + { + if (externalFunctions != null) { + List list = externalFunctions.Values.ToList (); + list.Sort ((LlvmIrFunction a, LlvmIrFunction b) => a.Signature.Name.CompareTo (b.Signature.Name)); + ExternalFunctions = list.AsReadOnly (); + } + + if (functions != null) { + List list = functions.Values.ToList (); + // TODO: sort or not? + Functions = list.AsReadOnly (); + } + + if (attributeSets != null) { + List list = attributeSets.Values.ToList (); + list.Sort ((LlvmIrFunctionAttributeSet a, LlvmIrFunctionAttributeSet b) => a.Number.CompareTo (b.Number)); + AttributeSets = list.AsReadOnly (); + } + + if (structures != null) { + List list = structures.Values.ToList (); + list.Sort ((StructureInfo a, StructureInfo b) => a.Name.CompareTo (b.Name)); + Structures = list.AsReadOnly (); + } + + if (stringManager != null && stringManager.StringGroups.Count > 0) { + Strings = stringManager.StringGroups.AsReadOnly (); + } + + GlobalVariables = globalVariables?.AsReadOnly (); + } + + public void Add (LlvmIrFunction func) + { + if (functions == null) { + functions = new Dictionary (); + } + + if (functions.TryGetValue (func, out LlvmIrFunction existingFunc)) { + throw new InvalidOperationException ($"Internal error: identical function has already been added (\"{func.Signature.Name}\")"); + } + + functions.Add (func, func); + } + + /// + /// A shortcut way to add a global variable without first having to create an instance of first. This overload + /// requires the parameter to not be null. + /// + public LlvmIrGlobalVariable AddGlobalVariable (string name, object value, LlvmIrVariableOptions? options = null, string? comment = null) + { + if (value == null) { + throw new ArgumentNullException (nameof (value)); + } + + return AddGlobalVariable (value.GetType (), name, value, options, comment); + } + + /// + /// A shortcut way to add a global variable without first having to create an instance of first. + /// + public LlvmIrGlobalVariable AddGlobalVariable (Type type, string name, object? value, LlvmIrVariableOptions? options = null, string? comment = null) + { + var ret = new LlvmIrGlobalVariable (type, name, options) { + Value = value, + Comment = comment, + }; + Add (ret); + return ret; + } + + public void Add (LlvmIrGlobalVariable variable, string stringGroupName, string? stringGroupComment = null, string? symbolSuffix = null) + { + EnsureValidGlobalVariableType (variable); + + if (IsStringVariable (variable)) { + AddStringGlobalVariable (variable, stringGroupName, stringGroupComment, symbolSuffix); + return; + } + + if (IsStringArrayVariable (variable)) { + AddStringArrayGlobalVariable (variable, stringGroupName, stringGroupComment, symbolSuffix); + return; + } + + throw new InvalidOperationException ("Internal error: this overload is ONLY for adding string or array-of-string variables"); + } + + public void Add (IList variables) + { + foreach (LlvmIrGlobalVariable variable in variables) { + Add (variable); + } + } + + public void Add (LlvmIrGlobalVariable variable) + { + EnsureValidGlobalVariableType (variable); + + if (IsStringVariable (variable)) { + AddStringGlobalVariable (variable); + return; + } + + if (IsStringArrayVariable (variable)) { + AddStringArrayGlobalVariable (variable); + return; + } + + if (IsStructureArrayVariable (variable)) { + AddStructureArrayGlobalVariable (variable); + return; + } + + if (IsStructureVariable (variable)) { + PrepareStructure (variable); + } + + AddStandardGlobalVariable (variable); + } + + void PrepareStructure (LlvmIrGlobalVariable variable) + { + var structure = variable.Value as StructureInstance; + if (structure == null) { + return; + } + + PrepareStructure (structure); + } + + void PrepareStructure (StructureInstance structure) + { + foreach (StructureMemberInfo smi in structure.Info.Members) { + if (smi.IsIRStruct ()) { + object? instance = structure.Obj == null ? null : smi.GetValue (structure.Obj); + if (instance == null) { + continue; + } + + StructureInfo si = GetStructureInfo (smi.MemberType); + PrepareStructure (new GeneratorStructureInstance (si, instance)); + continue; + } + + if (smi.Info.IsNativePointerToPreallocatedBuffer (out ulong bufferSize)) { + if (bufferSize == 0) { + bufferSize = structure.Info.GetBufferSizeFromProvider (smi, structure); + } + + AddAutomaticBuffer (structure, smi, bufferSize); + continue; + } + + if (smi.MemberType != typeof(string)) { + continue; + } + + string? value = smi.GetValue (structure.Obj) as string; + if (value != null) { + RegisterString (value, stringGroupName: structure.Info.Name, symbolSuffix: smi.Info.Name); + } + } + } + + void AddAutomaticBuffer (StructureInstance structure, StructureMemberInfo smi, ulong bufferSize) + { + if (bufferManager == null) { + bufferManager = new LlvmIrBufferManager (); + } + + string bufferName = bufferManager.Allocate (structure, smi, bufferSize); + var buffer = new LlvmIrGlobalVariable (typeof(List), bufferName, LlvmIrVariableOptions.LocalWritable) { + ZeroInitializeArray = true, + ArrayItemCount = bufferSize, + }; + Add (buffer); + } + + void AddStandardGlobalVariable (LlvmIrGlobalVariable variable) + { + if (globalVariables == null) { + globalVariables = new List (); + } + + globalVariables.Add (variable); + } + + void AddStringGlobalVariable (LlvmIrGlobalVariable variable, string? stringGroupName = null, string? stringGroupComment = null, string? symbolSuffix = null) + { + RegisterString (variable, stringGroupName, stringGroupComment, symbolSuffix); + AddStandardGlobalVariable (variable); + } + + void RegisterString (LlvmIrGlobalVariable variable, string? stringGroupName = null, string? stringGroupComment = null, string? symbolSuffix = null) + { + RegisterString ((string)variable.Value, stringGroupName, stringGroupComment, symbolSuffix); + } + + void RegisterString (string value, string? stringGroupName = null, string? stringGroupComment = null, string? symbolSuffix = null) + { + if (stringManager == null) { + stringManager = new LlvmIrStringManager (); + } + + stringManager.Add (value, stringGroupName, stringGroupComment, symbolSuffix); + } + + void AddStructureArrayGlobalVariable (LlvmIrGlobalVariable variable) + { + if (variable.Value == null) { + AddStandardGlobalVariable (variable); + return; + } + + // For simplicity we support only arrays with homogenous entry types + StructureInfo? info = null; + ulong index = 0; + + foreach (StructureInstance structure in (IEnumerable)variable.Value) { + if (info == null) { + info = structure.Info; + if (info.HasPreAllocatedBuffers) { + // let's group them... + Add (new LlvmIrGroupDelimiterVariable ()); + } + } + + if (structure.Type != info.Type) { + throw new InvalidOperationException ($"Internal error: only arrays with homogenous element types are currently supported. All entries were expected to be of type '{info.Type}', but the '{structure.Type}' type was encountered."); + } + + // This is a bit of a kludge to make a specific corner case work seamlessly from the LlvmIrModule user's point of view. + // The scenario is used in ApplicationConfigNativeAssemblyGenerator and it involves an array of structures where each + // array index contains the same object in structure.Obj but each instance needs to allocate a unique buffer at runtime. + // LlvmIrBufferManager makes it possible, but it must be able to uniquely identify each instance, which in this scenario + // wouldn't be possible if we had to rely only on the StructureInstance contents. Enter `StructureInstance.IndexInArray`, + // which is used to create unique buffers and unambiguously assign them to each structure instance. + // + // See LlvmIrBufferManager for how it is used. + structure.IndexInArray = index++; + + PrepareStructure (structure); + } + + if (info != null && info.HasPreAllocatedBuffers) { + Add (new LlvmIrGroupDelimiterVariable ()); + } + + AddStandardGlobalVariable (variable); + } + + void AddStringArrayGlobalVariable (LlvmIrGlobalVariable variable, string? stringGroupName = null, string? stringGroupComment = null, string? symbolSuffix = null) + { + if (variable.Value == null) { + AddStandardGlobalVariable (variable); + return; + } + + List? entries = null; + if (NameValueArrayType.IsAssignableFrom (variable.Type)) { + entries = new List (); + var dict = (IDictionary)variable.Value; + foreach (var kvp in dict) { + Register (kvp.Key); + Register (kvp.Value); + } + } else if (typeof(ICollection).IsAssignableFrom (variable.Type)) { + foreach (string s in (ICollection)variable.Value) { + Register (s); + } + } else { + throw new InvalidOperationException ($"Internal error: unsupported string array type `{variable.Type}'"); + } + + AddStandardGlobalVariable (variable); + + void Register (string value) + { + RegisterString (value, stringGroupName, stringGroupComment, symbolSuffix); + } + } + + bool IsStringArrayVariable (LlvmIrGlobalVariable variable) + { + if (NameValueArrayType.IsAssignableFrom (variable.Type)) { + if (variable.Value != null && !NameValueArrayType.IsAssignableFrom (variable.Value.GetType ())) { + throw new InvalidOperationException ($"Internal error: name:value array variable must have its value set to either `null` or `{NameValueArrayType}`"); + } + + return true; + } + + var ctype = typeof(ICollection); + if (ctype.IsAssignableFrom (variable.Type)) { + if (variable.Value != null && !ctype.IsAssignableFrom (variable.Value.GetType ())) { + throw new InvalidOperationException ($"Internal error: string array variable must have its value set to either `null` or implement `{ctype}`"); + } + + return true; + } + + if (variable.Type == typeof(string[])) { + if (variable.Value != null && variable.Value.GetType () != typeof(string[])) { + throw new InvalidOperationException ($"Internal error: string array variable must have its value set to either `null` or be `{typeof(string[])}`"); + } + + return true; + } + + return false; + } + + bool IsStringVariable (LlvmIrGlobalVariable variable) + { + if (variable.Type != typeof(string)) { + return false; + } + + if (variable.Value != null && variable.Value.GetType () != typeof(string)) { + throw new InvalidOperationException ("Internal error: variable of string type must have its value set to either `null` or a string"); + } + + return true; + } + + bool IsStructureArrayVariable (LlvmIrGlobalVariable variable) + { + if (typeof(StructureInstance[]).IsAssignableFrom (variable.Type)) { + return true; + } + + if (!variable.Type.IsArray ()) { + return false; + } + + Type elementType = variable.Type.GetArrayElementType (); + return typeof(StructureInstance).IsAssignableFrom (elementType); + } + + bool IsStructureVariable (LlvmIrGlobalVariable variable) + { + if (!typeof(StructureInstance).IsAssignableFrom (variable.Type)) { + return false; + } + + if (variable.Value != null && !typeof(StructureInstance).IsAssignableFrom (variable.Value.GetType ())) { + throw new InvalidOperationException ("Internal error: variable referring to a structure instance must have its value set to either `null` or an instance of the StructureInstance class"); + } + + return true; + } + + void EnsureValidGlobalVariableType (LlvmIrGlobalVariable variable) + { + if (variable is LlvmIrStringVariable) { + throw new ArgumentException ("Internal error: do not add instances of LlvmIrStringVariable, simply set variable value to the desired string instead"); + } + } + + /// + /// Looks up LLVM variable for a previously registered string given in . If a variable isn't found, + /// an exception is thrown. This is primarily used by to look up variables related to strings which + /// are part of structure instances. Such strings **MUST** be registered by and, thus, failure to do + /// so is an internal error. + /// + public LlvmIrStringVariable LookupRequiredVariableForString (string value) + { + LlvmIrStringVariable? sv = stringManager?.Lookup (value); + if (sv == null) { + throw new InvalidOperationException ($"Internal error: string '{value}' wasn't registered with string manager"); + } + + return sv; + } + + public string LookupRequiredBufferVariableName (StructureInstance structure, StructureMemberInfo smi) + { + if (bufferManager == null) { + throw new InvalidOperationException ("Internal error: no buffer variables have been registed with the buffer manager"); + } + + string? variableName = bufferManager.GetBufferVariableName (structure, smi); + if (String.IsNullOrEmpty (variableName)) { + throw new InvalidOperationException ($"Internal error: buffer for member '{smi.Info.Name}' of structure '{structure.Info.Name}' (index {structure.IndexInArray}) not found"); + } + + return variableName; + } + + /// + /// Add a new attribute set. The caller MUST use the returned value to refer to the set, instead of the one passed + /// as parameter, since this function de-duplicates sets and may return a previously added one that's identical to + /// the new one. + /// + public LlvmIrFunctionAttributeSet AddAttributeSet (LlvmIrFunctionAttributeSet attrSet) + { + if (attributeSets == null) { + attributeSets = new Dictionary (); + } + + if (attributeSets.TryGetValue (attrSet, out LlvmIrFunctionAttributeSet existingSet)) { + return existingSet; + } + attrSet.Number = (uint)attributeSets.Count; + attributeSets.Add (attrSet, attrSet); + + return attrSet; + } + + /// + /// Add a new external function declaration. The caller MUST use the returned value to refer to the function, instead + /// of the one passed as parameter, since this function de-duplicates function declarations and may return a previously + /// added one that's identical to the new one. + /// + public LlvmIrFunction DeclareExternalFunction (LlvmIrFunction func) + { + if (externalFunctions == null) { + externalFunctions = new Dictionary (); + } + + if (externalFunctions.TryGetValue (func, out LlvmIrFunction existingFunc)) { + return existingFunc; + } + + externalFunctions.Add (func, func); + return func; + } + + /// + /// Since LLVM IR is strongly typed, it requires each structure to be properly declared before it is + /// used throughout the code. This method uses reflection to scan the managed type + /// and record the information for future use. The returned structure contains + /// the description. It is used later on not only to declare the structure in output code, but also to generate + /// data from instances of . This method is typically called from the + /// method. + /// + public StructureInfo MapStructure () + { + Console.WriteLine ($"Mapping structure: {typeof(T)}"); + if (structures == null) { + structures = new Dictionary (); + } + + Type t = typeof(T); + if (!t.IsClass && !t.IsValueType) { + throw new InvalidOperationException ($"{t} must be a class or a struct"); + } + + // TODO: check if already there + if (structures.TryGetValue (t, out StructureInfo sinfo)) { + return (StructureInfo)sinfo; + } + + var ret = new StructureInfo (this, t); + structures.Add (t, ret); + + return ret; + } + + internal StructureInfo GetStructureInfo (Type type) + { + if (structures == null) { + throw new InvalidOperationException ($"Internal error: no structures have been mapped, cannot return info for {type}"); + } + + foreach (var kvp in structures) { + StructureInfo si = kvp.Value; + if (si.Type != type) { + continue; + } + + return si; + } + + throw new InvalidOperationException ($"Internal error: unmapped structure {type}"); + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModuleAArch64.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModuleAArch64.cs new file mode 100644 index 00000000000..131b79dcaa6 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModuleAArch64.cs @@ -0,0 +1,52 @@ +using System.Collections.Generic; + +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks.LLVMIR; + +class LlvmIrModuleAArch64 : LlvmIrModuleTarget +{ + public override LlvmIrDataLayout DataLayout { get; } + public override string Triple => "aarch64-unknown-linux-android21"; + public override AndroidTargetArch TargetArch => AndroidTargetArch.Arm64; + public override uint NativePointerSize => 8; + public override bool Is64Bit => true; + + public LlvmIrModuleAArch64 () + { + // + // As per Android NDK: + // target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" + // + DataLayout = new LlvmIrDataLayout { + LittleEndian = true, + Mangling = new LlvmIrDataLayoutMangling (LlvmIrDataLayoutManglingOption.ELF), + + IntegerAlignment = new List { + new LlvmIrDataLayoutIntegerAlignment (size: 8, abi: 8, pref: 32), // i8 + new LlvmIrDataLayoutIntegerAlignment (size: 16, abi: 16, pref: 32), // i16 + new LlvmIrDataLayoutIntegerAlignment (size: 64, abi: 64), // i64 + new LlvmIrDataLayoutIntegerAlignment (size: 128, abi: 128), // i128 + }, + + NativeIntegerWidths = new List { 32, 64}, + StackAlignment = 128, + }; + } + + public override void AddTargetSpecificAttributes (LlvmIrFunctionAttributeSet attrSet) + { + attrSet.Add (new TargetCpuFunctionAttribute ("generic")); + attrSet.Add (new TargetFeaturesFunctionAttribute ("+fix-cortex-a53-835769,+neon,+outline-atomics,+v8a")); + } + + public override void AddTargetSpecificMetadata (LlvmIrMetadataManager manager) + { + LlvmIrMetadataItem flags = GetFlagsMetadata (manager); + + flags.AddReferenceField (manager.AddNumbered (LlvmIrModuleMergeBehavior.Error, "branch-target-enforcement", 0)); + flags.AddReferenceField (manager.AddNumbered (LlvmIrModuleMergeBehavior.Error, "sign-return-address", 0)); + flags.AddReferenceField (manager.AddNumbered (LlvmIrModuleMergeBehavior.Error, "sign-return-address-all", 0)); + flags.AddReferenceField (manager.AddNumbered (LlvmIrModuleMergeBehavior.Error, "sign-return-address-with-bkey", 0)); + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModuleArmV7a.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModuleArmV7a.cs new file mode 100644 index 00000000000..f7ff54f8a48 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModuleArmV7a.cs @@ -0,0 +1,63 @@ +using System.Collections.Generic; + +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks.LLVMIR; + +class LlvmIrModuleArmV7a : LlvmIrModuleTarget +{ + public override LlvmIrDataLayout DataLayout { get; } + public override string Triple => "armv7-unknown-linux-android21"; + public override AndroidTargetArch TargetArch => AndroidTargetArch.Arm; + public override uint NativePointerSize => 4; + public override bool Is64Bit => false; + + public LlvmIrModuleArmV7a () + { + // + // As per Android NDK: + // target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64" + // + DataLayout = new LlvmIrDataLayout { + LittleEndian = true, + Mangling = new LlvmIrDataLayoutMangling (LlvmIrDataLayoutManglingOption.ELF), + + PointerSize = new List { + new LlvmIrDataLayoutPointerSize (size: 32, abi: 32), + }, + + FunctionPointerAlignment = new LlvmIrDataLayoutFunctionPointerAlignment (LlvmIrDataLayoutFunctionPointerAlignmentType.Independent, abi: 8), + + IntegerAlignment = new List { + new LlvmIrDataLayoutIntegerAlignment (size: 64, abi: 64), // i64 + }, + + VectorAlignment = new List { + new LlvmIrDataLayoutVectorAlignment (size: 128, abi: 64, pref: 128), // v128 + }, + + AggregateObjectAlignment = new LlvmIrDataLayoutAggregateObjectAlignment (abi: 0, pref: 32), + NativeIntegerWidths = new List { 32 }, + StackAlignment = 64, + }; + } + + public override void AddTargetSpecificAttributes (LlvmIrFunctionAttributeSet attrSet) + { + attrSet.Add (new TargetCpuFunctionAttribute ("generic")); + attrSet.Add (new TargetFeaturesFunctionAttribute ("+armv7-a,+d32,+dsp,+fp64,+neon,+vfp2,+vfp2sp,+vfp3,+vfp3d16,+vfp3d16sp,+vfp3sp,-aes,-fp-armv8,-fp-armv8d16,-fp-armv8d16sp,-fp-armv8sp,-fp16,-fp16fml,-fullfp16,-sha2,-thumb-mode,-vfp4,-vfp4d16,-vfp4d16sp,-vfp4sp")); + } + + public override void SetParameterFlags (LlvmIrFunctionParameter parameter) + { + base.SetParameterFlags (parameter); + SetIntegerParameterUpcastFlags (parameter); + } + + public override void AddTargetSpecificMetadata (LlvmIrMetadataManager manager) + { + LlvmIrMetadataItem flags = GetFlagsMetadata (manager); + + flags.AddReferenceField (manager.AddNumbered (LlvmIrModuleMergeBehavior.Error, "min_enum_size", 4)); + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModuleTarget.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModuleTarget.cs new file mode 100644 index 00000000000..ad9a1d9c604 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModuleTarget.cs @@ -0,0 +1,73 @@ +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks.LLVMIR; + +abstract class LlvmIrModuleTarget +{ + public abstract LlvmIrDataLayout DataLayout { get; } + public abstract string Triple { get; } + public abstract AndroidTargetArch TargetArch { get; } + public abstract uint NativePointerSize { get; } + public abstract bool Is64Bit { get; } + + /// + /// Adds target-specific attributes which are common to many attribute sets. Usually this specifies CPU type, tuning and + /// features. + /// + public virtual void AddTargetSpecificAttributes (LlvmIrFunctionAttributeSet attrSet) + {} + + public virtual void AddTargetSpecificMetadata (LlvmIrMetadataManager manager) + {} + + public virtual void SetParameterFlags (LlvmIrFunctionParameter parameter) + { + if (!parameter.NoUndef.HasValue) { + parameter.NoUndef = true; + } + } + + /// + /// Sets the zeroext or signext attributes on the parameter, if not set previously and if + /// the parameter is a small integral type. Out of our supported architectures, all except AArch64 set + /// the flags, thus the reason to put this method in the base class. + /// + protected void SetIntegerParameterUpcastFlags (LlvmIrFunctionParameter parameter) + { + if (parameter.Type == typeof(bool) || + parameter.Type == typeof(byte) || + parameter.Type == typeof(char) || + parameter.Type == typeof(ushort)) + { + if (!parameter.ZeroExt.HasValue) { + parameter.ZeroExt = true; + parameter.SignExt = false; + } + return; + } + + if (parameter.Type == typeof(sbyte) || + parameter.Type == typeof(short)) + { + if (!parameter.SignExt.HasValue) { + parameter.SignExt = true; + parameter.ZeroExt = false; + } + } + } + + public virtual int GetAggregateAlignment (int maxFieldAlignment, ulong dataSize) + { + return maxFieldAlignment; + } + + protected LlvmIrMetadataItem GetFlagsMetadata (LlvmIrMetadataManager manager) + { + LlvmIrMetadataItem? flags = manager.GetItem (LlvmIrKnownMetadata.LlvmModuleFlags); + if (flags == null) { + flags = manager.Add (LlvmIrKnownMetadata.LlvmModuleFlags); + } + + return flags; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModuleX64.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModuleX64.cs new file mode 100644 index 00000000000..44eede79e54 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModuleX64.cs @@ -0,0 +1,76 @@ +using System.Collections.Generic; + +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks.LLVMIR; + +class LlvmIrModuleX64 : LlvmIrModuleTarget +{ + public override LlvmIrDataLayout DataLayout { get; } + public override string Triple => "x86_64-unknown-linux-android21"; + public override AndroidTargetArch TargetArch => AndroidTargetArch.X86_64; + public override uint NativePointerSize => 8; + public override bool Is64Bit => true; + + public LlvmIrModuleX64 () + { + // + // As per Android NDK: + // target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" + // + DataLayout = new LlvmIrDataLayout { + LittleEndian = true, + Mangling = new LlvmIrDataLayoutMangling (LlvmIrDataLayoutManglingOption.ELF), + + PointerSize = new List { + new LlvmIrDataLayoutPointerSize (size: 32, abi: 32) { + AddressSpace = 270, + }, + new LlvmIrDataLayoutPointerSize (size: 32, abi: 32) { + AddressSpace = 271, + }, + new LlvmIrDataLayoutPointerSize (size: 64, abi: 64) { + AddressSpace = 272, + }, + }, + + IntegerAlignment = new List { + new LlvmIrDataLayoutIntegerAlignment (size: 64, abi: 64), // i64 + }, + + FloatAlignment = new List { + new LlvmIrDataLayoutFloatAlignment (size: 80, abi: 128), // f80 + }, + + NativeIntegerWidths = new List { 8, 16, 32, 64 }, + StackAlignment = 128, + }; + } + + public override void AddTargetSpecificAttributes (LlvmIrFunctionAttributeSet attrSet) + { + attrSet.Add (new TargetCpuFunctionAttribute ("x86-64")); + attrSet.Add (new TargetFeaturesFunctionAttribute ("+crc32,+cx16,+cx8,+fxsr,+mmx,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87")); + attrSet.Add (new TuneCpuFunctionAttribute ("generic")); + } + + public override void SetParameterFlags (LlvmIrFunctionParameter parameter) + { + base.SetParameterFlags (parameter); + SetIntegerParameterUpcastFlags (parameter); + } + + public override int GetAggregateAlignment (int maxFieldAlignment, ulong dataSize) + { + // System V ABI for x86_64 mandates that any aggregates 16 bytes or more long will + // be aligned at at least 16 bytes + // + // See: https://refspecs.linuxbase.org/elf/x86_64-abi-0.99.pdf (Section '3.1.2 Data Representation', "Aggregates and Unions") + // + if (dataSize >= 16 && maxFieldAlignment < 16) { + return 16; + } + + return maxFieldAlignment; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModuleX86.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModuleX86.cs new file mode 100644 index 00000000000..7e43558cb84 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModuleX86.cs @@ -0,0 +1,68 @@ +using System.Collections.Generic; + +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks.LLVMIR; + +class LlvmIrModuleX86 : LlvmIrModuleTarget +{ + public override LlvmIrDataLayout DataLayout { get; } + public override string Triple => "i686-unknown-linux-android21"; + public override AndroidTargetArch TargetArch => AndroidTargetArch.X86; + public override uint NativePointerSize => 4; + public override bool Is64Bit => false; + + public LlvmIrModuleX86 () + { + // + // As per Android NDK: + // target datalayout = "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-f64:32:64-f80:32-n8:16:32-S128" + // + DataLayout = new LlvmIrDataLayout { + LittleEndian = true, + Mangling = new LlvmIrDataLayoutMangling (LlvmIrDataLayoutManglingOption.ELF), + + PointerSize = new List { + new LlvmIrDataLayoutPointerSize (size: 32, abi: 32), + new LlvmIrDataLayoutPointerSize (size: 32, abi: 32) { + AddressSpace = 270, + }, + new LlvmIrDataLayoutPointerSize (size: 32, abi: 32) { + AddressSpace = 271, + }, + new LlvmIrDataLayoutPointerSize (size: 64, abi: 64) { + AddressSpace = 272, + }, + }, + + FloatAlignment = new List { + new LlvmIrDataLayoutFloatAlignment (size: 64, abi: 32, pref: 64), // f64 + new LlvmIrDataLayoutFloatAlignment (size: 80, abi: 32), // f80 + }, + + NativeIntegerWidths = new List { 8, 16, 32 }, + StackAlignment = 128, + }; + } + + public override void AddTargetSpecificAttributes (LlvmIrFunctionAttributeSet attrSet) + { + attrSet.Add (new TargetCpuFunctionAttribute ("i686")); + attrSet.Add (new TargetFeaturesFunctionAttribute ("+cx8,+mmx,+sse,+sse2,+sse3,+ssse3,+x87")); + attrSet.Add (new TuneCpuFunctionAttribute ("generic")); + attrSet.Add (new StackrealignFunctionAttribute ()); + } + + public override void SetParameterFlags (LlvmIrFunctionParameter parameter) + { + base.SetParameterFlags (parameter); + SetIntegerParameterUpcastFlags (parameter); + } + + public override void AddTargetSpecificMetadata (LlvmIrMetadataManager manager) + { + LlvmIrMetadataItem flags = GetFlagsMetadata (manager); + + flags.AddReferenceField (manager.AddNumbered (LlvmIrModuleMergeBehavior.Error, "NumRegisterParameters", 0)); + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrStringGroup.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrStringGroup.cs new file mode 100644 index 00000000000..a20cae23a8b --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrStringGroup.cs @@ -0,0 +1,16 @@ +using System.Collections.Generic; + +namespace Xamarin.Android.Tasks.LLVMIR; + +sealed class LlvmIrStringGroup +{ + public ulong Count; + public readonly string? Comment; + public readonly List Strings = new List (); + + public LlvmIrStringGroup (string? comment = null) + { + Comment = comment; + Count = 0; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrStringManager.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrStringManager.cs new file mode 100644 index 00000000000..9a49e544428 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrStringManager.cs @@ -0,0 +1,71 @@ +using System; +using System.Collections.Generic; + +namespace Xamarin.Android.Tasks.LLVMIR; + +partial class LlvmIrModule +{ + protected class LlvmIrStringManager + { + Dictionary stringSymbolCache = new Dictionary (StringComparer.Ordinal); + Dictionary stringGroupCache = new Dictionary (StringComparer.Ordinal); + List stringGroups = new List (); + + LlvmIrStringGroup defaultGroup; + + public List StringGroups => stringGroups; + + public LlvmIrStringManager () + { + defaultGroup = new LlvmIrStringGroup (); + stringGroupCache.Add (String.Empty, defaultGroup); + stringGroups.Add (defaultGroup); + } + + public LlvmIrStringVariable Add (string value, string? groupName = null, string? groupComment = null, string? symbolSuffix = null) + { + if (value == null) { + throw new ArgumentNullException (nameof (value)); + } + + LlvmIrStringVariable? stringVar; + if (stringSymbolCache.TryGetValue (value, out stringVar) && stringVar != null) { + return stringVar; + } + + LlvmIrStringGroup? group; + string groupPrefix; + if (String.IsNullOrEmpty (groupName) || String.Compare ("str", groupName, StringComparison.Ordinal) == 0) { + group = defaultGroup; + groupPrefix = ".str"; + } else if (!stringGroupCache.TryGetValue (groupName, out group) || group == null) { + group = new LlvmIrStringGroup (groupComment ?? groupName); + stringGroups.Add (group); + stringGroupCache[groupName] = group; + groupPrefix = $".{groupName}"; + } else { + groupPrefix = $".{groupName}"; + } + + string symbolName = $"{groupPrefix}.{group.Count++}"; + if (!String.IsNullOrEmpty (symbolSuffix)) { + symbolName = $"{symbolName}_{symbolSuffix}"; + } + + stringVar = new LlvmIrStringVariable (symbolName, value); + group.Strings.Add (stringVar); + stringSymbolCache.Add (value, stringVar); + + return stringVar; + } + + public LlvmIrStringVariable? Lookup (string value) + { + if (stringSymbolCache.TryGetValue (value, out LlvmIrStringVariable? sv)) { + return sv; + } + + return null; + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs index 0abda63bfdd..7f741490ce0 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs @@ -1,34 +1,226 @@ using System; +using System.Globalization; -namespace Xamarin.Android.Tasks.LLVMIR +namespace Xamarin.Android.Tasks.LLVMIR; + +[Flags] +enum LlvmIrVariableWriteOptions +{ + None = 0x0000, + ArrayWriteIndexComments = 0x0001, + ArrayFormatInRows = 0x0002, +} + +abstract class LlvmIrVariable : IEquatable { + public abstract bool Global { get; } + public abstract string NamePrefix { get; } + + public string? Name { get; protected set; } + public Type Type { get; protected set; } + public LlvmIrVariableWriteOptions WriteOptions { get; set; } = LlvmIrVariableWriteOptions.ArrayWriteIndexComments; + + /// + /// Number of columns an array that is written in rows should have. By default, arrays are written one item in a line, but + /// when the flag is set in , then + /// the value of this property dictates how many items are to be placed in a single row. + /// + public uint ArrayStride { get; set; } = 8; + public object? Value { get; set; } + public string? Comment { get; set; } + + /// + /// Both global and local variables will want their names to matter in equality checks, but function + /// parameters must not take it into account, thus this property. If set to false, + /// will ignore name when checking for equality. + protected bool NameMatters { get; set; } = true; + /// - /// Base class for all the variable (local and global) as well as function parameter classes. + /// Returns a string which constitutes a reference to a local (using the % prefix character) or a global + /// (using the @ prefix character) variable, ready for use in the generated code wherever variables are + /// referenced. /// - abstract class LlvmIrVariable + public virtual string Reference { + get { + if (String.IsNullOrEmpty (Name)) { + throw new InvalidOperationException ("Variable doesn't have a name, it cannot be referenced"); + } + + return $"{NamePrefix}{Name}"; + } + } + + /// + /// + /// Certain data must be calculated when the target architecture is known, because it may depend on certain aspects of + /// the target (e.g. its bitness). This callback, if set, will be invoked before the variable is written to the output + /// stream, allowing updating of any such data as described above. + /// + /// + /// First parameter passed to the callback is the variable itself, second parameter is the current + /// and the third is the value previously assigned to + /// + /// + public Action? BeforeWriteCallback { get; set; } + + /// + /// Object passed to the method, if any, as the caller state. + /// + public object? BeforeWriteCallbackCallerState { get; set; } + + /// + /// + /// Callback used when processing array variables, called for each item of the array in order to obtain the item's comment, if any. + /// + /// + /// The first argument is the variable which contains the array, second is the item index, third is the item value and fourth is + /// the caller state object, previously assigned to the property. The callback + /// can return an empty string or null, in which case no comment is written. + /// + /// + public Func? GetArrayItemCommentCallback { get; set; } + + /// + /// Object passed to the method, if any, as the caller state. + /// + public object? GetArrayItemCommentCallbackCallerState { get; set; } + + /// + /// Constructs an abstract variable. is translated to one of the LLVM IR first class types (see + /// https://llvm.org/docs/LangRef.html#t-firstclass) only if it's an integral or floating point type. In all other cases it + /// is treated as an opaque pointer type. + /// + protected LlvmIrVariable (Type type, string? name = null) { - public LlvmNativeFunctionSignature? NativeFunction { get; } - public string? Name { get; } - public Type Type { get; } + Type = type; + Name = name; + } - // Used when we need a pointer to pointer (etc) or when the type itself is not a pointer but we need one - // in a given context (e.g. function parameters) - public bool IsNativePointer { get; } + public override int GetHashCode () + { + return Type.GetHashCode () ^ (Name?.GetHashCode () ?? 0); + } - protected LlvmIrVariable (Type type, string name, LlvmNativeFunctionSignature? signature, bool isNativePointer) - { - Type = type ?? throw new ArgumentNullException (nameof (type)); - Name = name; - NativeFunction = signature; - IsNativePointer = isNativePointer; + public override bool Equals (object obj) + { + var irVar = obj as LlvmIrVariable; + if (irVar == null) { + return false; } - protected LlvmIrVariable (LlvmIrVariable variable, string name, bool isNativePointer) - { - Type = variable?.Type ?? throw new ArgumentNullException (nameof (variable)); - Name = name; - NativeFunction = variable.NativeFunction; - IsNativePointer = isNativePointer; + return Equals (irVar); + } + + public virtual bool Equals (LlvmIrVariable other) + { + if (other == null) { + return false; } + + return + Global == other.Global && + Type == other.Type && + String.Compare (NamePrefix, other.NamePrefix, StringComparison.Ordinal) == 0 && + (!NameMatters || String.Compare (Name, other.Name, StringComparison.Ordinal) == 0); } } + +class LlvmIrLocalVariable : LlvmIrVariable +{ + public override bool Global => false; + public override string NamePrefix => "%"; + + /// + /// Constructs a local variable. is translated to one of the LLVM IR first class types (see + /// https://llvm.org/docs/LangRef.html#t-firstclass) only if it's an integral or floating point type. In all other cases it + /// is treated as an opaque pointer type. is optional because local variables can be unnamed, in + /// which case they will be assigned a sequential number when function code is generated. + /// + public LlvmIrLocalVariable (Type type, string? name = null) + : base (type, name) + {} + + public void AssignNumber (ulong n) + { + Name = n.ToString (CultureInfo.InvariantCulture); + } +} + +class LlvmIrGlobalVariable : LlvmIrVariable +{ + /// + /// By default a global variable is constant and exported. + /// + public static readonly LlvmIrVariableOptions DefaultOptions = LlvmIrVariableOptions.GlobalConstant; + + public override bool Global => true; + public override string NamePrefix => "@"; + + /// + /// Specify variable options. If omitted, it defaults to . + /// + /// + public virtual LlvmIrVariableOptions? Options { get; set; } + + public bool ZeroInitializeArray { get; set; } + public ulong ArrayItemCount { get; set; } + + /// + /// Constructs a local variable. is translated to one of the LLVM IR first class types (see + /// https://llvm.org/docs/LangRef.html#t-firstclass) only if it's an integral or floating point type. In all other cases it + /// is treated as an opaque pointer type. is required because global variables must be named. + /// + public LlvmIrGlobalVariable (Type type, string name, LlvmIrVariableOptions? options = null) + : base (type, name) + { + if (String.IsNullOrEmpty (name)) { + throw new ArgumentException ("must not be null or empty", nameof (name)); + } + + Options = options; + } + + /// + /// Constructs a local variable and sets the property to and + /// property to its type. For that reason, **must not** be null. is + /// required because global variables must be named. + /// + public LlvmIrGlobalVariable (object value, string name, LlvmIrVariableOptions? options = null) + : this ((value ?? throw new ArgumentNullException (nameof (value))).GetType (), name, options) + { + Value = value; + } + + /// + /// This is, unfortunately, needed to be able to address scenarios when a single symbol can have a different type when + /// generating output for a specific target (e.g. 32-bit vs 64-bit integer variables). If the variable requires such + /// type changes, this should be done at generation time from within the method. + /// + public void OverrideValueAndType (Type newType, object? newValue) + { + Type = newType; + Value = newValue; + } +} + +class LlvmIrStringVariable : LlvmIrGlobalVariable +{ + public LlvmIrStringVariable (string name, string value) + : base (typeof(string), name, LlvmIrVariableOptions.LocalString) + { + Value = value; + } +} + +/// +/// This is to address my dislike to have single-line variables separated by empty lines :P. +/// When an instance of this "variable" is first encountered, it enables variable grouping, that is +/// they will be followed by just a single newline. The next instance of this "variable" turns +/// grouping off, meaning the following variables will be followed by two newlines. +/// +class LlvmIrGroupDelimiterVariable : LlvmIrGlobalVariable +{ + public LlvmIrGroupDelimiterVariable () + : base (typeof(void), ".:!GroupDelimiter!:.", LlvmIrVariableOptions.LocalConstant) + {} +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariableOptions.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariableOptions.cs index 0c0b3c121b4..680881ca31f 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariableOptions.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariableOptions.cs @@ -23,6 +23,7 @@ class LlvmIrVariableOptions /// public static readonly LlvmIrVariableOptions GlobalWritable = new LlvmIrVariableOptions { Writability = LlvmIrWritability.Writable, + AddressSignificance = LlvmIrAddressSignificance.LocalUnnamed, }; /// @@ -57,6 +58,7 @@ class LlvmIrVariableOptions Linkage = LlvmIrLinkage.Private, Writability = LlvmIrWritability.Constant, AddressSignificance = LlvmIrAddressSignificance.Unnamed, + RuntimePreemption = LlvmIrRuntimePreemption.Default, }; /// @@ -83,7 +85,7 @@ class LlvmIrVariableOptions }; public LlvmIrLinkage Linkage { get; set; } = LlvmIrLinkage.Default; - public LlvmIrRuntimePreemption RuntimePreemption { get; set; } = LlvmIrRuntimePreemption.Default; + public LlvmIrRuntimePreemption RuntimePreemption { get; set; } = LlvmIrRuntimePreemption.DSOLocal; public LlvmIrVisibility Visibility { get; set; } = LlvmIrVisibility.Default; public LlvmIrAddressSignificance AddressSignificance { get; set; } = LlvmIrAddressSignificance.Default; public LlvmIrWritability Writability { get; set; } = LlvmIrWritability.Writable; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariableReference.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariableReference.cs deleted file mode 100644 index 628014eeb2a..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariableReference.cs +++ /dev/null @@ -1,50 +0,0 @@ -using System; - -namespace Xamarin.Android.Tasks.LLVMIR -{ - /// - /// References either a local or global variable. - /// - class LlvmIrVariableReference : LlvmIrVariable - { - public string Reference { get; } - - public LlvmIrVariableReference (Type type, string name, bool isGlobal, bool isNativePointer = false) - : base (type, name, signature: null, isNativePointer: isNativePointer) - { - if (String.IsNullOrEmpty (name)) { - throw new ArgumentException ("must not be null or empty", nameof (name)); - } - Reference = MakeReference (isGlobal, name); - } - - public LlvmIrVariableReference (LlvmNativeFunctionSignature signature, string name, bool isGlobal, bool isNativePointer = false) - : base (typeof(LlvmNativeFunctionSignature), name, signature, isNativePointer) - { - if (signature == null) { - throw new ArgumentNullException (nameof (signature)); - } - - if (String.IsNullOrEmpty (name)) { - throw new ArgumentException ("must not be null or empty", nameof (name)); - } - - Reference = MakeReference (isGlobal, name); - } - - public LlvmIrVariableReference (LlvmIrVariable variable, bool isGlobal, bool isNativePointer = false) - : base (variable, variable?.Name, isNativePointer) - { - if (String.IsNullOrEmpty (variable?.Name)) { - throw new ArgumentException ("variable name must not be null or empty", nameof (variable)); - } - - Reference = MakeReference (isGlobal, variable?.Name); - } - - string MakeReference (bool isGlobal, string name) - { - return $"{(isGlobal ? '@' : '%')}{Name}"; - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmNativeFunctionSignature.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmNativeFunctionSignature.cs deleted file mode 100644 index 01471c8199a..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmNativeFunctionSignature.cs +++ /dev/null @@ -1,34 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; - -namespace Xamarin.Android.Tasks.LLVMIR -{ - /// - /// Contains signature/description of a native function. All the types used for parameters or return value must - /// be mappable to LLVM IR types. This class can be used to describe pointers to functions which have no corresponding - /// managed method (e.g. `xamarin_app_init` used by marshal methods). Additionally, an optional default value can be - /// specified, to be used whenever a variable of this type is emitted (e.g. - class LlvmNativeFunctionSignature - { - public Type ReturnType { get; } - public IList? Parameters { get; } - public object? FieldValue { get; set; } - - public LlvmNativeFunctionSignature (Type returnType, List? parameters = null) - { - ReturnType = returnType ?? throw new ArgumentNullException (nameof (returnType)); - Parameters = parameters?.Select (p => EnsureValidParameter (p))?.ToList ()?.AsReadOnly (); - - LlvmIrFunctionParameter EnsureValidParameter (LlvmIrFunctionParameter parameter) - { - if (parameter == null) { - throw new InvalidOperationException ("null parameters aren't allowed"); - } - - return parameter; - } - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/MemberInfoUtilities.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/MemberInfoUtilities.cs index 832a34c0fae..431d92b6229 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/MemberInfoUtilities.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/MemberInfoUtilities.cs @@ -25,7 +25,7 @@ public static bool IsNativePointerToPreallocatedBuffer (this MemberInfo mi, out public static bool PointsToSymbol (this MemberInfo mi, out string? symbolName) { var attr = mi.GetCustomAttribute (); - if (attr == null) { + if (attr == null || attr.PointsToSymbol == null) { symbolName = null; return false; } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/StructureInfo.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/StructureInfo.cs index 5cc1dfa6589..066c788c93e 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/StructureInfo.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/StructureInfo.cs @@ -1,68 +1,37 @@ using System; using System.Collections.Generic; -using System.IO; using System.Reflection; namespace Xamarin.Android.Tasks.LLVMIR { // TODO: add cache for members and data provider info - sealed class StructureInfo : IStructureInfo + sealed class StructureInfo { Type type; public Type Type => type; - public string Name { get; } = String.Empty; - public ulong Size { get; } - public List> Members { get; } = new List> (); + public string Name { get; } = String.Empty; + public ulong Size { get; } + public IList Members { get; } = new List (); public NativeAssemblerStructContextDataProvider? DataProvider { get; } - public int MaxFieldAlignment { get; private set; } = 0; - public bool HasStrings { get; private set; } - public bool HasPreAllocatedBuffers { get; private set; } + public ulong MaxFieldAlignment { get; private set; } = 0; + public bool HasStrings { get; private set; } + public bool HasPreAllocatedBuffers { get; private set; } + public bool HasPointers { get; private set; } - public bool IsOpaque => Members.Count == 0; - public string NativeTypeDesignator { get; } + public bool IsOpaque => Members.Count == 0; + public string NativeTypeDesignator { get; } - public StructureInfo (LlvmIrGenerator generator) + public StructureInfo (LlvmIrModule module, Type type) { - type = typeof(T); + this.type = type; Name = type.GetShortName (); - Size = GatherMembers (type, generator); + Size = GatherMembers (type, module); DataProvider = type.GetDataProvider (); NativeTypeDesignator = type.IsNativeClass () ? "class" : "struct"; } - public void RenderDeclaration (LlvmIrGenerator generator) - { - TextWriter output = generator.Output; - generator.WriteStructureDeclarationStart (NativeTypeDesignator, Name, forOpaqueType: IsOpaque); - - if (IsOpaque) { - return; - } - - for (int i = 0; i < Members.Count; i++) { - StructureMemberInfo info = Members[i]; - string nativeType = LlvmIrGenerator.MapManagedTypeToNative (info.MemberType); - if (info.Info.IsNativePointer ()) { - nativeType += "*"; - } - - // TODO: nativeType can be an array, update to indicate that (and get the size) - string arraySize; - if (info.IsNativeArray) { - arraySize = $"[{info.ArrayElements}]"; - } else { - arraySize = String.Empty; - } - - var comment = $"{nativeType} {info.Info.Name}{arraySize}"; - generator.WriteStructureDeclarationField (info.IRType, comment, i == Members.Count - 1); - } - - generator.WriteStructureDeclarationEnd (); - } - - public string? GetCommentFromProvider (StructureMemberInfo smi, StructureInstance instance) + public string? GetCommentFromProvider (StructureMemberInfo smi, StructureInstance instance) { if (DataProvider == null || !smi.Info.UsesDataProvider ()) { return null; @@ -76,7 +45,7 @@ public void RenderDeclaration (LlvmIrGenerator generator) return ret; } - public ulong GetBufferSizeFromProvider (StructureMemberInfo smi, StructureInstance instance) + public ulong GetBufferSizeFromProvider (StructureMemberInfo smi, StructureInstance instance) { if (DataProvider == null) { return 0; @@ -85,7 +54,7 @@ public ulong GetBufferSizeFromProvider (StructureMemberInfo smi, StructureIns return DataProvider.GetBufferSize (instance.Obj, smi.Info.Name); } - ulong GatherMembers (Type type, LlvmIrGenerator generator, bool storeMembers = true) + ulong GatherMembers (Type type, LlvmIrModule module, bool storeMembers = true) { ulong size = 0; foreach (MemberInfo mi in type.GetMembers ()) { @@ -93,13 +62,17 @@ ulong GatherMembers (Type type, LlvmIrGenerator generator, bool storeMembers = t continue; } - var info = new StructureMemberInfo (mi, generator); + var info = new StructureMemberInfo (mi, module); + if (info.IsNativePointer) { + HasPointers = true; + } + if (storeMembers) { Members.Add (info); size += info.Size; - if ((int)info.Alignment > MaxFieldAlignment) { - MaxFieldAlignment = (int)info.Alignment; + if (info.Alignment > MaxFieldAlignment) { + MaxFieldAlignment = info.Alignment; } } @@ -116,7 +89,7 @@ ulong GatherMembers (Type type, LlvmIrGenerator generator, bool storeMembers = t // The presence of strings/buffers is important at the generation time as it is used to decide whether we need separate stream writers for them and // if the owning structure does **not** have any of those, the generated code would be invalid if (info.IsIRStruct ()) { - GatherMembers (info.MemberType, generator, storeMembers: false); + GatherMembers (info.MemberType, module, storeMembers: false); } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/StructureInstance.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/StructureInstance.cs index 3b171485f5e..3c0c53ab471 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/StructureInstance.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/StructureInstance.cs @@ -1,37 +1,62 @@ using System; -using System.Collections.Generic; -using System.IO; -using System.Reflection; namespace Xamarin.Android.Tasks.LLVMIR { - class StructureInstance + abstract class StructureInstance { - Dictionary, StructurePointerData>? pointees; + StructureInfo info; - public T Obj { get; } + public object? Obj { get; } + public Type Type => info.Type; + public StructureInfo Info => info; - public StructureInstance (T instance) - { - Obj = instance; - } + /// + /// Do **not** set this property, it is used internally by , + /// and when dealing with arrays of objects where each + /// array index contains the same object instance + /// + internal ulong IndexInArray { get; set; } - public void AddPointerData (StructureMemberInfo smi, string? variableName, ulong dataSize) + /// + /// This is a cludge to support zero-initialized structures. In order to output proper variable type + /// when a structure is used, the generator must be able to read the structure descrption, which is + /// provided in the property and, thus, it requires a variable of structural type to + /// **always** have a non-null value. To support zero initialization of such structures, this property + /// can be set to true + /// + public bool IsZeroInitialized { get; set; } + + protected StructureInstance (StructureInfo info, object instance) { - if (pointees == null) { - pointees = new Dictionary, StructurePointerData> (); + if (instance == null) { + throw new ArgumentNullException (nameof (instance)); } - pointees.Add (smi, new StructurePointerData (variableName, dataSize)); - } - - public StructurePointerData? GetPointerData (StructureMemberInfo smi) - { - if (pointees != null && pointees.TryGetValue (smi, out StructurePointerData ssd)) { - return ssd; + if (!info.Type.IsAssignableFrom (instance.GetType ())) { + throw new ArgumentException ($"must be an instance of, or derived from, the {info.Type} type, or `null` (was {instance})", nameof (instance)); } - return null; + this.info = info; + Obj = instance; } } + + /// + /// Represents a typed structure instance, derived from the class. The slightly weird + /// approach is because on one hand we need to operate on a heterogenous set of structures (in which generic types would + /// only get in the way), but on the other hand we need to be able to get the structure type (whose instance is in + /// and ) only by looking at the **type**. This is needed in situations when we have + /// an array of some structures that is empty - we wouldn't be able to gleam the structure type from any instance and we still + /// need to output a stronly typed LLVM IR declaration of the structure array. With this class, most of the code will use the + /// abstract type, and knowing we have only one non-abstract implementation of the class allows + /// us to use StructureInstance<T> in a cast, to get T via reflection. + /// + sealed class StructureInstance : StructureInstance + { + public T? Instance => (T)Obj; + + public StructureInstance (StructureInfo info, T instance) + : base (info, instance) + {} + } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/StructureMemberInfo.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/StructureMemberInfo.cs index c09d46910fe..1f313a34f31 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/StructureMemberInfo.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/StructureMemberInfo.cs @@ -3,7 +3,7 @@ namespace Xamarin.Android.Tasks.LLVMIR { - sealed class StructureMemberInfo + sealed class StructureMemberInfo { public string IRType { get; } public MemberInfo Info { get; } @@ -27,7 +27,7 @@ sealed class StructureMemberInfo public bool IsInlineArray { get; } public bool NeedsPadding { get; } - public StructureMemberInfo (MemberInfo mi, LlvmIrGenerator generator) + public StructureMemberInfo (MemberInfo mi, LlvmIrModule module) { Info = mi; @@ -38,18 +38,19 @@ public StructureMemberInfo (MemberInfo mi, LlvmIrGenerator generator) }; ulong size = 0; + bool isPointer = false; if (MemberType != typeof(string) && !MemberType.IsArray && (MemberType.IsStructure () || MemberType.IsClass)) { IRType = $"%struct.{MemberType.GetShortName ()}"; // TODO: figure out how to get structure size if it isn't a pointer } else { - IRType = generator.MapManagedTypeToIR (MemberType, out size); + IRType = LlvmIrGenerator.MapToIRType (MemberType, out size, out isPointer); } - IsNativePointer = IRType[IRType.Length - 1] == '*'; + IsNativePointer = isPointer; if (!IsNativePointer) { IsNativePointer = mi.IsNativePointer (); if (IsNativePointer) { - IRType += "*"; + IRType = LlvmIrGenerator.IRPointerType; } } @@ -60,14 +61,18 @@ public StructureMemberInfo (MemberInfo mi, LlvmIrGenerator generator) Alignment = 0; if (IsNativePointer) { - size = (ulong)generator.PointerSize; + size = 0; // Real size will be determined when code is generated and we know the target architecture } else if (mi.IsInlineArray ()) { + if (!MemberType.IsArray) { + throw new InvalidOperationException ($"Internal error: member {mi.Name} of structure {mi.DeclaringType.Name} is marked as inline array, but is not of an array type."); + } + IsInlineArray = true; IsNativeArray = true; NeedsPadding = mi.InlineArrayNeedsPadding (); int arrayElements = mi.GetInlineArraySize (); if (arrayElements < 0) { - arrayElements = GetArraySizeFromProvider (typeof(T).GetDataProvider (), mi.Name); + arrayElements = GetArraySizeFromProvider (MemberType.GetDataProvider (), mi.Name); } if (arrayElements < 0) { @@ -77,7 +82,7 @@ public StructureMemberInfo (MemberInfo mi, LlvmIrGenerator generator) IRType = $"[{arrayElements} x {IRType}]"; ArrayElements = (ulong)arrayElements; } else if (this.IsIRStruct ()) { - IStructureInfo si = generator.GetStructureInfo (MemberType); + StructureInfo si = module.GetStructureInfo (MemberType); size = si.Size; Alignment = (ulong)si.MaxFieldAlignment; } @@ -92,7 +97,7 @@ public StructureMemberInfo (MemberInfo mi, LlvmIrGenerator generator) } } - public object? GetValue (T instance) + public object? GetValue (object instance) { if (Info is FieldInfo fi) { return fi.GetValue (instance); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/StructureStringData.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/StructureStringData.cs deleted file mode 100644 index 8e23e81b1e6..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/StructureStringData.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace Xamarin.Android.Tasks.LLVMIR -{ - sealed class StructurePointerData - { - public string? VariableName { get; } - public ulong Size { get; } - - public StructurePointerData (string? name, ulong size) - { - VariableName = name; - Size = size; - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/TypeUtilities.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/TypeUtilities.cs index fd41c1790e8..b50b7290d90 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/TypeUtilities.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/TypeUtilities.cs @@ -1,8 +1,8 @@ using System; +using System.Collections; +using System.Collections.Generic; using System.Reflection; -using Xamarin.Android.Tasks; - namespace Xamarin.Android.Tasks.LLVMIR { static class TypeUtilities @@ -46,7 +46,7 @@ public static bool IsStructure (this Type type) type != typeof (object); } - public static bool IsIRStruct (this StructureMemberInfo smi) + public static bool IsIRStruct (this StructureMemberInfo smi) { Type type = smi.MemberType; @@ -73,5 +73,113 @@ public static bool IsNativeClass (this Type t) var attr = t.GetCustomAttribute (); return attr != null; } + + public static bool ImplementsInterface (this Type type, Type requiredIfaceType) + { + if (type == null || requiredIfaceType == null) { + return false; + } + + if (type == requiredIfaceType) { + return true; + } + + bool generic = requiredIfaceType.IsGenericType; + foreach (Type iface in type.GetInterfaces ()) { + if (iface == requiredIfaceType) { + return true; + } + + if (generic) { + if (!iface.IsGenericType) { + continue; + } + + if (iface.GetGenericTypeDefinition () == requiredIfaceType.GetGenericTypeDefinition ()) { + return true; + } + } + } + + return false; + } + + public static bool IsStructureInstance (this Type type, out Type? structureType) + { + structureType = null; + if (!type.IsGenericType) { + return false; + } + + if (type.GetGenericTypeDefinition () != typeof(StructureInstance<>)) { + return false; + } + + structureType = type.GetGenericArguments ()[0]; + return true; + } + + /// + /// Return element type of a single-dimensional (with one exception, see below) array. Parameter **MUST** + /// correspond to one of the following array types: T[], ICollection<T> or IDictionary<string, string>. The latter is + /// used to comfortably represent name:value arrays, which are output as single dimensional arrays in the native code. + /// + /// + /// Thrown when is not one of the array types listed above. + /// + public static Type GetArrayElementType (this Type type) + { + if (type.IsArray) { + return type.GetElementType (); + } + + if (!type.IsGenericType) { + throw WrongTypeException (); + } + + Type genericType = type.GetGenericTypeDefinition (); + if (genericType.ImplementsInterface (typeof(ICollection<>))) { + Type[] genericArgs = type.GetGenericArguments (); + return genericArgs[0]; + } + + if (!genericType.ImplementsInterface (typeof(IDictionary))) { + throw WrongTypeException (); + } + + return typeof(string); + + // Dictionary + Exception WrongTypeException () => new InvalidOperationException ($"Internal error: type '{type}' is not an array, ICollection or IDictionary"); + } + + /// + /// Determine whether type represents an array, in our understanding. That means the type has to be + /// a standard single-dimensional language array (i.e. T[]), implement ICollection<T> together with ICollection or, + /// as a special case for name:value pair collections, implement IDictionary<string, string> + /// + public static bool IsArray (this Type t) + { + if (t.IsPrimitive) { + return false; + } + + if (t == typeof(string)) { + return false; + } + + if (t.IsArray) { + if (t.GetArrayRank () > 1) { + throw new NotSupportedException ("Internal error: multi-dimensional arrays aren't supported"); + } + + return true; + } + + // TODO: cache results here + // IDictionary is a special case for name:value string arrays which we use for some constructs. + return (t.ImplementsInterface (typeof(ICollection<>)) && t.ImplementsInterface (typeof(ICollection))) || + t.ImplementsInterface (typeof(IDictionary)); + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/X64LlvmIrGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/X64LlvmIrGenerator.cs deleted file mode 100644 index 81aa5dfdb9a..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/X64LlvmIrGenerator.cs +++ /dev/null @@ -1,52 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; - -using Xamarin.Android.Tools; - -namespace Xamarin.Android.Tasks.LLVMIR -{ - class X64LlvmIrGenerator : LlvmIrGenerator - { - // See https://llvm.org/docs/LangRef.html#data-layout - // - // Value as used by Android NDK's clang++ - // - protected override string DataLayout => "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"; - public override int PointerSize => 8; - protected override string Triple => "x86_64-unknown-linux-android"; // NDK appends API level, we don't need that - - static readonly LlvmFunctionAttributeSet commonAttributes = new LlvmFunctionAttributeSet { - new FramePointerFunctionAttribute ("none"), - new TargetCpuFunctionAttribute ("x86-64"), - new TargetFeaturesFunctionAttribute ("+cx16,+cx8,+fxsr,+mmx,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87"), - new TuneCpuFunctionAttribute ("generic"), - }; - - public X64LlvmIrGenerator (AndroidTargetArch arch, StreamWriter output, string fileName) - : base (arch, output, fileName) - {} - - protected override int GetAggregateAlignment (int maxFieldAlignment, ulong dataSize) - { - // System V ABI for x86_64 mandates that any aggregates 16 bytes or more long will - // be aligned at at least 16 bytes - // - // See: https://refspecs.linuxbase.org/elf/x86_64-abi-0.99.pdf (Section '3.1.2 Data Representation', "Aggregates and Unions") - // - if (dataSize >= 16 && maxFieldAlignment < 16) { - return 16; - } - - return maxFieldAlignment; - } - - protected override void InitFunctionAttributes () - { - base.InitFunctionAttributes (); - - FunctionAttributes[FunctionAttributesXamarinAppInit].Add (commonAttributes); - FunctionAttributes[FunctionAttributesJniMethods].Add (commonAttributes); - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/X86LlvmIrGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/X86LlvmIrGenerator.cs deleted file mode 100644 index d779bf25bb2..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/X86LlvmIrGenerator.cs +++ /dev/null @@ -1,45 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; - -using Xamarin.Android.Tools; - -namespace Xamarin.Android.Tasks.LLVMIR -{ - class X86LlvmIrGenerator : LlvmIrGenerator - { - // See https://llvm.org/docs/LangRef.html#data-layout - // - // Value as used by Android NDK's clang++ - // - protected override string DataLayout => "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-f64:32:64-f80:32-n8:16:32-S128"; - public override int PointerSize => 4; - protected override string Triple => "i686-unknown-linux-android"; // NDK appends API level, we don't need that - - static readonly LlvmFunctionAttributeSet commonAttributes = new LlvmFunctionAttributeSet { - new FramePointerFunctionAttribute ("none"), - new TargetCpuFunctionAttribute ("i686"), - new TargetFeaturesFunctionAttribute ("+cx8,+mmx,+sse,+sse2,+sse3,+ssse3,+x87"), - new TuneCpuFunctionAttribute ("generic"), - new StackrealignFunctionAttribute (), - }; - - public X86LlvmIrGenerator (AndroidTargetArch arch, StreamWriter output, string fileName) - : base (arch, output, fileName) - {} - - protected override void AddModuleFlagsMetadata (List flagsFields) - { - base.AddModuleFlagsMetadata (flagsFields); - flagsFields.Add (MetadataManager.AddNumbered (LlvmIrModuleMergeBehavior.Error, "NumRegisterParameters", 0)); - } - - protected override void InitFunctionAttributes () - { - base.InitFunctionAttributes (); - - FunctionAttributes[FunctionAttributesXamarinAppInit].Add (commonAttributes); - FunctionAttributes[FunctionAttributesJniMethods].Add (commonAttributes); - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs index 229ba0397d7..45ef98dba31 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs @@ -14,8 +14,10 @@ namespace Xamarin.Android.Tasks { - class MarshalMethodsNativeAssemblyGenerator : LlvmIrComposer + partial class MarshalMethodsNativeAssemblyGenerator : LlvmIrComposer { + const string GetFunctionPointerVariableName = "get_function_pointer"; + // This is here only to generate strongly-typed IR internal sealed class MonoClass {} @@ -92,8 +94,8 @@ public MarshalMethodInfo (MarshalMethodEntry method, Type returnType, string nat } NativeSymbolName = nativeSymbolName; Parameters = new List { - new LlvmIrFunctionParameter (typeof (_JNIEnv), "env", isNativePointer: true), // JNIEnv *env - new LlvmIrFunctionParameter (typeof (_jclass), "klass", isNativePointer: true), // jclass klass + new LlvmIrFunctionParameter (typeof (_JNIEnv), "env"), // JNIEnv *env + new LlvmIrFunctionParameter (typeof (_jclass), "klass"), // jclass klass }; ClassCacheIndex = (uint)classCacheIndex; } @@ -106,7 +108,7 @@ public override string GetComment (object data, string fieldName) var klass = EnsureType (data); if (String.Compare ("token", fieldName, StringComparison.Ordinal) == 0) { - return $"token 0x{klass.token:x}; class name: {klass.ClassName}"; + return $" token 0x{klass.token:x}; class name: {klass.ClassName}"; } return String.Empty; @@ -133,7 +135,7 @@ public override string GetComment (object data, string fieldName) var methodName = EnsureType (data); if (String.Compare ("id", fieldName, StringComparison.Ordinal) == 0) { - return $"id 0x{methodName.id:x}; name: {methodName.name}"; + return $" id 0x{methodName.id:x}; name: {methodName.name}"; } return String.Empty; @@ -143,11 +145,63 @@ public override string GetComment (object data, string fieldName) [NativeAssemblerStructContextDataProvider (typeof(MarshalMethodNameDataProvider))] sealed class MarshalMethodName { + [NativeAssembler (Ignore = true)] + public ulong Id32; + + [NativeAssembler (Ignore = true)] + public ulong Id64; + [NativeAssembler (UsesDataProvider = true)] public ulong id; public string name; } + sealed class AssemblyCacheState + { + public Dictionary? AsmNameToIndexData32; + public Dictionary Hashes32; + public List Keys32; + public List Indices32; + + public Dictionary? AsmNameToIndexData64; + public Dictionary Hashes64; + public List Keys64; + public List Indices64; + } + + sealed class MarshalMethodsWriteState + { + public AssemblyCacheState AssemblyCacheState; + public LlvmIrFunctionAttributeSet AttributeSet; + public Dictionary UniqueAssemblyId; + public Dictionary UsedBackingFields; + public LlvmIrVariable GetFunctionPtrVariable; + public LlvmIrFunction GetFunctionPtrFunction; + } + + sealed class MarshalMethodAssemblyIndexValuePlaceholder : LlvmIrInstructionArgumentValuePlaceholder + { + MarshalMethodInfo mmi; + AssemblyCacheState acs; + + public MarshalMethodAssemblyIndexValuePlaceholder (MarshalMethodInfo mmi, AssemblyCacheState acs) + { + this.mmi = mmi; + this.acs = acs; + } + + public override object? GetValue (LlvmIrModuleTarget target) + { + // What a monstrosity... + string asmName = mmi.Method.NativeCallback.DeclaringType.Module.Assembly.Name.Name; + Dictionary asmNameToIndex = target.Is64Bit ? acs.AsmNameToIndexData64 : acs.AsmNameToIndexData32; + if (!asmNameToIndex.TryGetValue (asmName, out uint asmIndex)) { + throw new InvalidOperationException ($"Unable to translate assembly name '{asmName}' to its index"); + } + return asmIndex; + } + } + static readonly Dictionary jniSimpleTypeMap = new Dictionary { { 'Z', typeof(bool) }, { 'B', typeof(byte) }, @@ -176,29 +230,14 @@ sealed class MarshalMethodName IDictionary> marshalMethods; TaskLoggingHelper logger; - StructureInfo monoImage; - StructureInfo marshalMethodsClass; - StructureInfo marshalMethodName; - StructureInfo monoClass; - StructureInfo<_JNIEnv> _jniEnvSI; - StructureInfo<_jobject> _jobjectSI; - StructureInfo<_jclass> _jclassSI; - StructureInfo<_jstring> _jstringSI; - StructureInfo<_jthrowable> _jthrowableSI; - StructureInfo<_jarray> _jarraySI; - StructureInfo<_jobjectArray> _jobjectArraySI; - StructureInfo<_jbooleanArray> _jbooleanArraySI; - StructureInfo<_jbyteArray> _jbyteArraySI; - StructureInfo<_jcharArray> _jcharArraySI; - StructureInfo<_jshortArray> _jshortArraySI; - StructureInfo<_jintArray> _jintArraySI; - StructureInfo<_jlongArray> _jlongArraySI; - StructureInfo<_jfloatArray> _jfloatArraySI; - StructureInfo<_jdoubleArray> _jdoubleArraySI; + StructureInfo marshalMethodsManagedClassStructureInfo; + StructureInfo marshalMethodNameStructureInfo; List methods; List> classes = new List> (); + LlvmIrCallMarker defaultCallMarker; + readonly bool generateEmptyCode; /// @@ -209,6 +248,7 @@ public MarshalMethodsNativeAssemblyGenerator (int numberOfAssembliesInApk, IColl this.numberOfAssembliesInApk = numberOfAssembliesInApk; this.uniqueAssemblyNames = uniqueAssemblyNames ?? throw new ArgumentNullException (nameof (uniqueAssemblyNames)); generateEmptyCode = true; + defaultCallMarker = LlvmIrCallMarker.Tail; } /// @@ -222,9 +262,10 @@ public MarshalMethodsNativeAssemblyGenerator (int numberOfAssembliesInApk, IColl this.logger = logger ?? throw new ArgumentNullException (nameof (logger)); generateEmptyCode = false; + defaultCallMarker = LlvmIrCallMarker.None; } - public override void Init () + void Init () { if (generateEmptyCode || marshalMethods == null || marshalMethods.Count == 0) { return; @@ -354,7 +395,7 @@ void ProcessAndAddMethod (List allMethods, MarshalMethodEntry ClassName = klass, }; - classes.Add (new StructureInstance (mc)); + classes.Add (new StructureInstance (marshalMethodsManagedClassStructureInfo, mc)); } // Methods with `IsSpecial == true` are "synthetic" methods - they contain only the callback reference @@ -541,64 +582,296 @@ void AddParameter (Type type) } // Every parameter which isn't a primitive type becomes a pointer - parameters.Add (new LlvmIrFunctionParameter (type, implementedMethod.Parameters[parameters.Count].Name, isNativePointer: type.IsNativeClass ())); + parameters.Add (new LlvmIrFunctionParameter (type, implementedMethod.Parameters[parameters.Count].Name)); } } - protected override void InitGenerator (LlvmIrGenerator generator) + protected override void Construct (LlvmIrModule module) { - generator.InitCodeOutput (); + MapStructures (module); + + Init (); + // InitTracing (module); + AddAssemblyImageCache (module, out AssemblyCacheState acs); + + // class cache + module.AddGlobalVariable ("marshal_methods_number_of_classes", (uint)classes.Count, LlvmIrVariableOptions.GlobalConstant); + module.AddGlobalVariable ("marshal_methods_class_cache", classes, LlvmIrVariableOptions.GlobalWritable); + + // Marshal methods class names + var mm_class_names = new List (); + foreach (StructureInstance klass in classes) { + mm_class_names.Add (klass.Instance.ClassName); + } + module.AddGlobalVariable ("mm_class_names", mm_class_names, LlvmIrVariableOptions.GlobalConstant, comment: " Names of classes in which marshal methods reside"); + + AddMarshalMethodNames (module, acs); + (LlvmIrVariable getFunctionPtrVariable, LlvmIrFunction getFunctionPtrFunction) = AddXamarinAppInitFunction (module); + + AddMarshalMethods (module, acs, getFunctionPtrVariable, getFunctionPtrFunction); } - protected override void MapStructures (LlvmIrGenerator generator) + void MapStructures (LlvmIrModule module) { - monoImage = generator.MapStructure (); - monoClass = generator.MapStructure (); - marshalMethodsClass = generator.MapStructure (); - marshalMethodName = generator.MapStructure (); - _jniEnvSI = generator.MapStructure<_JNIEnv> (); - _jobjectSI = generator.MapStructure<_jobject> (); - _jclassSI = generator.MapStructure<_jclass> (); - _jstringSI = generator.MapStructure<_jstring> (); - _jthrowableSI = generator.MapStructure<_jthrowable> (); - _jarraySI = generator.MapStructure<_jarray> (); - _jobjectArraySI = generator.MapStructure<_jobjectArray> (); - _jbooleanArraySI = generator.MapStructure<_jbooleanArray> (); - _jbyteArraySI = generator.MapStructure<_jbyteArray> (); - _jcharArraySI = generator.MapStructure<_jcharArray> (); - _jshortArraySI = generator.MapStructure<_jshortArray> (); - _jintArraySI = generator.MapStructure<_jintArray> (); - _jlongArraySI = generator.MapStructure<_jlongArray> (); - _jfloatArraySI = generator.MapStructure<_jfloatArray> (); - _jdoubleArraySI = generator.MapStructure<_jdoubleArray> (); + marshalMethodsManagedClassStructureInfo = module.MapStructure (); + marshalMethodNameStructureInfo = module.MapStructure (); } - protected override void Write (LlvmIrGenerator generator) + void AddMarshalMethods (LlvmIrModule module, AssemblyCacheState acs, LlvmIrVariable getFunctionPtrVariable, LlvmIrFunction getFunctionPtrFunction) { - WriteAssemblyImageCache (generator, out Dictionary asmNameToIndex); - WriteClassCache (generator); - LlvmIrVariableReference get_function_pointer_ref = WriteXamarinAppInitFunction (generator); - WriteNativeMethods (generator, asmNameToIndex, get_function_pointer_ref); + if (generateEmptyCode || methods == null || methods.Count == 0) { + return; + } - var mm_class_names = new List (); - foreach (StructureInstance klass in classes) { - mm_class_names.Add (klass.Obj.ClassName); + // This will make all the backing fields to appear in a block without empty lines separating them. + module.Add ( + new LlvmIrGroupDelimiterVariable () { + Comment = " Marshal methods backing fields, pointers to native functions" + } + ); + + var writeState = new MarshalMethodsWriteState { + AssemblyCacheState = acs, + AttributeSet = MakeMarshalMethodAttributeSet (module), + UsedBackingFields = new Dictionary (StringComparer.Ordinal), + UniqueAssemblyId = new Dictionary (StringComparer.OrdinalIgnoreCase), + GetFunctionPtrVariable = getFunctionPtrVariable, + GetFunctionPtrFunction = getFunctionPtrFunction, + }; + foreach (MarshalMethodInfo mmi in methods) { + CecilMethodDefinition nativeCallback = mmi.Method.NativeCallback; + string asmName = nativeCallback.DeclaringType.Module.Assembly.Name.Name; + + if (!writeState.UniqueAssemblyId.TryGetValue (asmName, out ulong asmId)) { + asmId = (ulong)writeState.UniqueAssemblyId.Count; + writeState.UniqueAssemblyId.Add (asmName, asmId); + } + + AddMarshalMethod (module, mmi, asmId, writeState); + } + + module.Add (new LlvmIrGroupDelimiterVariable ()); + } + + void AddMarshalMethod (LlvmIrModule module, MarshalMethodInfo method, ulong asmId, MarshalMethodsWriteState writeState) + { + CecilMethodDefinition nativeCallback = method.Method.NativeCallback; + string backingFieldName = $"native_cb_{method.Method.JniMethodName}_{asmId}_{method.ClassCacheIndex}_{nativeCallback.MetadataToken.ToUInt32():x}"; + + if (!writeState.UsedBackingFields.TryGetValue (backingFieldName, out LlvmIrVariable backingField)) { + backingField = module.AddGlobalVariable (typeof(IntPtr), backingFieldName, null, LlvmIrVariableOptions.LocalWritableInsignificantAddr); + writeState.UsedBackingFields.Add (backingFieldName, backingField); } - generator.WriteArray (mm_class_names, "mm_class_names", "Names of classes in which marshal methods reside"); - var uniqueMethods = new Dictionary (); + var funcComment = new StringBuilder (" Method: "); + funcComment.AppendLine (nativeCallback.FullName); + funcComment.Append (" Assembly: "); + funcComment.AppendLine (nativeCallback.Module.Assembly.Name.FullName); + funcComment.Append (" Registered: "); + funcComment.AppendLine (method.Method.RegisteredMethod?.FullName ?? "none"); + + var func = new LlvmIrFunction (method.NativeSymbolName, method.ReturnType, method.Parameters, writeState.AttributeSet) { + Comment = funcComment.ToString (), + }; + + WriteBody (func.Body); + module.Add (func); + + void WriteBody (LlvmIrFunctionBody body) + { + LlvmIrLocalVariable cb1 = func.CreateLocalVariable (typeof(IntPtr), "cb1"); + body.Load (backingField, cb1, tbaa: module.TbaaAnyPointer); + + LlvmIrLocalVariable isNullResult = func.CreateLocalVariable (typeof(bool), "isNull"); + body.Icmp (LlvmIrIcmpCond.Equal, cb1, null, isNullResult); + + var loadCallbackLabel = new LlvmIrFunctionLabelItem ("loadCallback"); + var callbackLoadedLabel = new LlvmIrFunctionLabelItem ("callbackLoaded"); + body.Br (isNullResult, loadCallbackLabel, callbackLoadedLabel); + + // Callback variable was null + body.Add (loadCallbackLabel); + + LlvmIrLocalVariable getFuncPtrResult = func.CreateLocalVariable (typeof(IntPtr), "get_func_ptr"); + body.Load (writeState.GetFunctionPtrVariable, getFuncPtrResult, tbaa: module.TbaaAnyPointer); + + var placeholder = new MarshalMethodAssemblyIndexValuePlaceholder (method, writeState.AssemblyCacheState); + LlvmIrInstructions.Call call = body.Call ( + writeState.GetFunctionPtrFunction, + arguments: new List { placeholder, method.ClassCacheIndex, nativeCallback.MetadataToken.ToUInt32 (), backingField }, + funcPointer: getFuncPtrResult + ); + + LlvmIrLocalVariable cb2 = func.CreateLocalVariable (typeof(IntPtr), "cb2"); + body.Load (backingField, cb2, tbaa: module.TbaaAnyPointer); + body.Br (callbackLoadedLabel); + + // Callback variable has just been set or it wasn't null + body.Add (callbackLoadedLabel); + LlvmIrLocalVariable fn = func.CreateLocalVariable (typeof(IntPtr), "fn"); + + // Preceding blocks are ordered from the newest to the oldest, so we need to pass the variables referring to our callback in "reverse" order + body.Phi (fn, cb2, body.PrecedingBlock1, cb1, body.PrecedingBlock2); + + var nativeFunc = new LlvmIrFunction (method.NativeSymbolName, method.ReturnType, method.Parameters); + nativeFunc.Signature.ReturnAttributes.NoUndef = true; + + var arguments = new List (); + foreach (LlvmIrFunctionParameter parameter in nativeFunc.Signature.Parameters) { + arguments.Add (new LlvmIrLocalVariable (parameter.Type, parameter.Name)); + } + LlvmIrLocalVariable? result = nativeFunc.ReturnsValue ? func.CreateLocalVariable (nativeFunc.Signature.ReturnType) : null; + call = body.Call (nativeFunc, result, arguments, funcPointer: fn); + call.CallMarker = LlvmIrCallMarker.Tail; + + body.Ret (nativeFunc.Signature.ReturnType, result); + } + } + + LlvmIrFunctionAttributeSet MakeMarshalMethodAttributeSet (LlvmIrModule module) + { + var attrSet = new LlvmIrFunctionAttributeSet { + new MustprogressFunctionAttribute (), + new UwtableFunctionAttribute (), + new MinLegalVectorWidthFunctionAttribute (0), + new NoTrappingMathFunctionAttribute (true), + new StackProtectorBufferSizeFunctionAttribute (8), + }; + + return module.AddAttributeSet (attrSet); + } + + (LlvmIrVariable getFuncPtrVariable, LlvmIrFunction getFuncPtrFunction) AddXamarinAppInitFunction (LlvmIrModule module) + { + var getFunctionPtrParams = new List { + new (typeof(uint), "mono_image_index") { + NoUndef = true, + }, + new (typeof(uint), "class_index") { + NoUndef = true, + }, + new (typeof(uint), "method_token") { + NoUndef = true, + }, + new (typeof(IntPtr), "target_ptr") { + NoUndef = true, + NonNull = true, + Align = 0, // 0 means use natural pointer alignment + Dereferenceable = 0, // ditto 👆 + IsCplusPlusReference = true, + }, + }; + + var getFunctionPtrComment = new StringBuilder (" "); + getFunctionPtrComment.Append (GetFunctionPointerVariableName); + getFunctionPtrComment.Append (" ("); + for (int i = 0; i < getFunctionPtrParams.Count; i++) { + if (i > 0) { + getFunctionPtrComment.Append (", "); + } + LlvmIrFunctionParameter parameter = getFunctionPtrParams[i]; + getFunctionPtrComment.Append (LlvmIrGenerator.MapManagedTypeToNative (parameter.Type)); + if (parameter.IsCplusPlusReference.HasValue && parameter.IsCplusPlusReference.Value) { + getFunctionPtrComment.Append ('&'); + } + getFunctionPtrComment.Append (' '); + getFunctionPtrComment.Append (parameter.Name); + } + getFunctionPtrComment.Append (')'); + + LlvmIrFunction getFunctionPtrFunc = new LlvmIrFunction ( + name: GetFunctionPointerVariableName, + returnType: typeof(void), + parameters: getFunctionPtrParams + ); + + LlvmIrVariable getFunctionPtrVariable = module.AddGlobalVariable ( + typeof(IntPtr), + GetFunctionPointerVariableName, + null, + LlvmIrVariableOptions.LocalWritableInsignificantAddr, + getFunctionPtrComment.ToString () + ); + + var init_params = new List { + new (typeof(_JNIEnv), "env") { + NoCapture = true, + NoUndef = true, + ReadNone = true, + }, + new (typeof(IntPtr), "fn") { + NoUndef = true, + }, + }; + + var init_signature = new LlvmIrFunctionSignature ( + name: "xamarin_app_init", + returnType: typeof(void), + parameters: init_params + ); + + LlvmIrFunctionAttributeSet attrSet = MakeXamarinAppInitAttributeSet (module); + var xamarin_app_init = new LlvmIrFunction (init_signature, attrSet); + xamarin_app_init.Body.Add ( + new LlvmIrInstructions.Store (init_params[1], getFunctionPtrVariable) { + TBAA = module.TbaaAnyPointer, + } + ); + xamarin_app_init.Body.Add (new LlvmIrInstructions.Ret (typeof(void))); + + module.Add (xamarin_app_init); + + return (getFunctionPtrVariable, getFunctionPtrFunc); + } + + LlvmIrFunctionAttributeSet MakeXamarinAppInitAttributeSet (LlvmIrModule module) + { + var attrSet = new LlvmIrFunctionAttributeSet { + new MustprogressFunctionAttribute (), + new NofreeFunctionAttribute (), + new NorecurseFunctionAttribute (), + new NosyncFunctionAttribute (), + new NounwindFunctionAttribute (), + new WillreturnFunctionAttribute (), + // TODO: LLVM 16+ feature, enable when we switch to this version + // new MemoryFunctionAttribute { + // Default = MemoryAttributeAccessKind.Write, + // Argmem = MemoryAttributeAccessKind.None, + // InaccessibleMem = MemoryAttributeAccessKind.None, + // }, + new UwtableFunctionAttribute (), + new MinLegalVectorWidthFunctionAttribute (0), + new NoTrappingMathFunctionAttribute (true), + new StackProtectorBufferSizeFunctionAttribute (8), + }; + + return module.AddAttributeSet (attrSet); + } + + void AddMarshalMethodNames (LlvmIrModule module, AssemblyCacheState acs) + { + var uniqueMethods = new Dictionary (); + if (!generateEmptyCode && methods != null) { foreach (MarshalMethodInfo mmi in methods) { string asmName = Path.GetFileName (mmi.Method.NativeCallback.Module.Assembly.MainModule.FileName); - if (!asmNameToIndex.TryGetValue (asmName, out uint idx)) { - throw new InvalidOperationException ($"Internal error: failed to match assembly name '{asmName}' to cache array index"); + + if (!acs.AsmNameToIndexData32.TryGetValue (asmName, out uint idx32)) { + throw new InvalidOperationException ($"Internal error: failed to match assembly name '{asmName}' to 32-bit cache array index"); } - ulong id = ((ulong)idx << 32) | (ulong)mmi.Method.NativeCallback.MetadataToken.ToUInt32 (); - if (uniqueMethods.ContainsKey (id)) { + if (!acs.AsmNameToIndexData64.TryGetValue (asmName, out uint idx64)) { + throw new InvalidOperationException ($"Internal error: failed to match assembly name '{asmName}' to 64-bit cache array index"); + } + + ulong methodToken = (ulong)mmi.Method.NativeCallback.MetadataToken.ToUInt32 (); + ulong id32 = ((ulong)idx32 << 32) | methodToken; + if (uniqueMethods.ContainsKey (id32)) { continue; } - uniqueMethods.Add (id, mmi); + + ulong id64 = ((ulong)idx64 << 32) | methodToken; + uniqueMethods.Add (id32, (mmi, id32, id64)); } } @@ -607,25 +880,35 @@ protected override void Write (LlvmIrGenerator generator) var mm_method_names = new List> (); foreach (var kvp in uniqueMethods) { ulong id = kvp.Key; - MarshalMethodInfo mmi = kvp.Value; + (MarshalMethodInfo mmi, ulong id32, ulong id64) = kvp.Value; RenderMethodNameWithParams (mmi.Method.NativeCallback, methodName); name = new MarshalMethodName { + Id32 = id32, + Id64 = id64, + // Tokens are unique per assembly - id = id, + id = 0, name = methodName.ToString (), }; - mm_method_names.Add (new StructureInstance (name)); + mm_method_names.Add (new StructureInstance (marshalMethodNameStructureInfo, name)); } // Must terminate with an "invalid" entry name = new MarshalMethodName { + Id32 = 0, + Id64 = 0, + id = 0, name = String.Empty, }; - mm_method_names.Add (new StructureInstance (name)); + mm_method_names.Add (new StructureInstance (marshalMethodNameStructureInfo, name)); - generator.WriteStructureArray (marshalMethodName, mm_method_names, LlvmIrVariableOptions.GlobalConstant, "mm_method_names"); + var mm_method_names_variable = new LlvmIrGlobalVariable (mm_method_names, "mm_method_names", LlvmIrVariableOptions.GlobalConstant) { + BeforeWriteCallback = UpdateMarshalMethodNameIds, + BeforeWriteCallbackCallerState = acs, + }; + module.Add (mm_method_names_variable); void RenderMethodNameWithParams (CecilMethodDefinition md, StringBuilder buffer) { @@ -650,210 +933,150 @@ void RenderMethodNameWithParams (CecilMethodDefinition md, StringBuilder buffer) } } - void WriteNativeMethods (LlvmIrGenerator generator, Dictionary asmNameToIndex, LlvmIrVariableReference get_function_pointer_ref) + void UpdateMarshalMethodNameIds (LlvmIrVariable variable, LlvmIrModuleTarget target, object? callerState) { - if (generateEmptyCode || methods == null || methods.Count == 0) { - return; - } + var mm_method_names = (List>)variable.Value; + bool is64Bit = target.Is64Bit; - var usedBackingFields = new HashSet (StringComparer.Ordinal); - foreach (MarshalMethodInfo mmi in methods) { - CecilMethodDefinition nativeCallback = mmi.Method.NativeCallback; - string asmName = nativeCallback.DeclaringType.Module.Assembly.Name.Name; - if (!asmNameToIndex.TryGetValue (asmName, out uint asmIndex)) { - throw new InvalidOperationException ($"Unable to translate assembly name '{asmName}' to its index"); - } - mmi.AssemblyCacheIndex = asmIndex; - WriteMarshalMethod (generator, mmi, get_function_pointer_ref, usedBackingFields); + foreach (StructureInstance mmn in mm_method_names) { + mmn.Instance.id = is64Bit ? mmn.Instance.Id64 : mmn.Instance.Id32; } } - void WriteMarshalMethod (LlvmIrGenerator generator, MarshalMethodInfo method, LlvmIrVariableReference get_function_pointer_ref, HashSet usedBackingFields) + // TODO: this should probably be moved to a separate writer, since not only marshal methods use the cache + void AddAssemblyImageCache (LlvmIrModule module, out AssemblyCacheState acs) { - var backingFieldSignature = new LlvmNativeFunctionSignature ( - returnType: method.ReturnType, - parameters: method.Parameters - ) { - FieldValue = "null", + var assembly_image_cache = new LlvmIrGlobalVariable (typeof(List), "assembly_image_cache", LlvmIrVariableOptions.GlobalWritable) { + ZeroInitializeArray = true, + ArrayItemCount = (ulong)numberOfAssembliesInApk, }; + module.Add (assembly_image_cache); - CecilMethodDefinition nativeCallback = method.Method.NativeCallback; - string backingFieldName = $"native_cb_{method.Method.JniMethodName}_{method.AssemblyCacheIndex}_{method.ClassCacheIndex}_{nativeCallback.MetadataToken.ToUInt32():x}"; - var backingFieldRef = new LlvmIrVariableReference (backingFieldSignature, backingFieldName, isGlobal: true); + acs = new AssemblyCacheState { + AsmNameToIndexData32 = new Dictionary (StringComparer.Ordinal), + Indices32 = new List (), - if (!usedBackingFields.Contains (backingFieldName)) { - generator.WriteVariable (backingFieldName, backingFieldSignature, LlvmIrVariableOptions.LocalWritableInsignificantAddr); - usedBackingFields.Add (backingFieldName); - } - - var func = new LlvmIrFunction ( - name: method.NativeSymbolName, - returnType: method.ReturnType, - attributeSetID: LlvmIrGenerator.FunctionAttributesJniMethods, - parameters: method.Parameters - ); - - generator.WriteFunctionStart (func, $"Method: {nativeCallback.FullName}\nAssembly: {nativeCallback.Module.Assembly.Name}"); - - LlvmIrFunctionLocalVariable callbackVariable1 = generator.EmitLoadInstruction (func, backingFieldRef, "cb1"); - var callbackVariable1Ref = new LlvmIrVariableReference (callbackVariable1, isGlobal: false); - - LlvmIrFunctionLocalVariable isNullVariable = generator.EmitIcmpInstruction (func, LlvmIrIcmpCond.Equal, callbackVariable1Ref, expectedValue: "null", resultVariableName: "isNull"); - var isNullVariableRef = new LlvmIrVariableReference (isNullVariable, isGlobal: false); - - const string loadCallbackLabel = "loadCallback"; - const string callbackLoadedLabel = "callbackLoaded"; - - generator.EmitBrInstruction (func, isNullVariableRef, loadCallbackLabel, callbackLoadedLabel); - - generator.WriteEOL (); - generator.EmitLabel (func, loadCallbackLabel); - LlvmIrFunctionLocalVariable getFunctionPointerVariable = generator.EmitLoadInstruction (func, get_function_pointer_ref, "get_func_ptr"); - var getFunctionPtrRef = new LlvmIrVariableReference (getFunctionPointerVariable, isGlobal: false); - - generator.EmitCall ( - func, - getFunctionPtrRef, - new List { - new LlvmIrFunctionArgument (typeof(uint), method.AssemblyCacheIndex), - new LlvmIrFunctionArgument (typeof(uint), method.ClassCacheIndex), - new LlvmIrFunctionArgument (typeof(uint), nativeCallback.MetadataToken.ToUInt32 ()), - new LlvmIrFunctionArgument (typeof(LlvmIrVariableReference), backingFieldRef), - } - ); - - LlvmIrFunctionLocalVariable callbackVariable2 = generator.EmitLoadInstruction (func, backingFieldRef, "cb2"); - var callbackVariable2Ref = new LlvmIrVariableReference (callbackVariable2, isGlobal: false); - - generator.EmitBrInstruction (func, callbackLoadedLabel); - - generator.WriteEOL (); - generator.EmitLabel (func, callbackLoadedLabel); - - LlvmIrFunctionLocalVariable fnVariable = generator.EmitPhiInstruction ( - func, - backingFieldRef, - new List<(LlvmIrVariableReference variableRef, string label)> { - (callbackVariable1Ref, func.ImplicitFuncTopLabel), - (callbackVariable2Ref, loadCallbackLabel), - }, - resultVariableName: "fn" - ); - var fnVariableRef = new LlvmIrVariableReference (fnVariable, isGlobal: false); + AsmNameToIndexData64 = new Dictionary (StringComparer.Ordinal), + Indices64 = new List (), + }; - LlvmIrFunctionLocalVariable? result = generator.EmitCall ( - func, - fnVariableRef, - func.ParameterVariables.Select (pv => new LlvmIrFunctionArgument (pv)).ToList () - ); + acs.Hashes32 = new Dictionary (); + acs.Hashes64 = new Dictionary (); + uint index = 0; + + foreach (string name in uniqueAssemblyNames) { + // We must make sure we keep the possible culture prefix, which will be treated as "directory" path here + string clippedName = Path.Combine (Path.GetDirectoryName (name) ?? String.Empty, Path.GetFileNameWithoutExtension (name)); + ulong hashFull32 = GetXxHash (name, is64Bit: false); + ulong hashClipped32 = GetXxHash (clippedName, is64Bit: false); + ulong hashFull64 = GetXxHash (name, is64Bit: true); + ulong hashClipped64 = GetXxHash (clippedName, is64Bit: true); + + // + // If the number of name forms changes, xamarin-app.hh MUST be updated to set value of the + // `number_of_assembly_name_forms_in_image_cache` constant to the number of forms. + // + acs.Hashes32.Add ((uint)Convert.ChangeType (hashFull32, typeof(uint)), (name, index)); + acs.Hashes32.Add ((uint)Convert.ChangeType (hashClipped32, typeof(uint)), (clippedName, index)); + acs.Hashes64.Add (hashFull64, (name, index)); + acs.Hashes64.Add (hashClipped64, (clippedName, index)); + + index++; + } - if (result != null) { - generator.EmitReturnInstruction (func, result); + acs.Keys32 = acs.Hashes32.Keys.ToList (); + acs.Keys32.Sort (); + for (int i = 0; i < acs.Keys32.Count; i++) { + (string name, uint idx) = acs.Hashes32[acs.Keys32[i]]; + acs.Indices32.Add (idx); + acs.AsmNameToIndexData32.Add (name, idx); } - generator.WriteFunctionEnd (func); - } + acs.Keys64 = acs.Hashes64.Keys.ToList (); + acs.Keys64.Sort (); + for (int i = 0; i < acs.Keys64.Count; i++) { + (string name, uint idx) = acs.Hashes64[acs.Keys64[i]]; + acs.Indices64.Add (idx); + acs.AsmNameToIndexData64.Add (name, idx); + } - LlvmIrVariableReference WriteXamarinAppInitFunction (LlvmIrGenerator generator) - { - var get_function_pointer_sig = new LlvmNativeFunctionSignature ( - returnType: typeof(void), - parameters: new List { - new LlvmIrFunctionParameter (typeof(uint), "mono_image_index"), - new LlvmIrFunctionParameter (typeof(uint), "class_index"), - new LlvmIrFunctionParameter (typeof(uint), "method_token"), - new LlvmIrFunctionParameter (typeof(IntPtr), "target_ptr", isNativePointer: true, isCplusPlusReference: true) - } - ) { - FieldValue = "null", + var assembly_image_cache_hashes = new LlvmIrGlobalVariable (typeof(List), "assembly_image_cache_hashes", LlvmIrVariableOptions.GlobalConstant) { + Comment = " Each entry maps hash of an assembly name to an index into the `assembly_image_cache` array", + BeforeWriteCallback = UpdateAssemblyImageCacheHashes, + BeforeWriteCallbackCallerState = acs, + GetArrayItemCommentCallback = GetAssemblyImageCacheItemComment, + GetArrayItemCommentCallbackCallerState = acs, }; + module.Add (assembly_image_cache_hashes); - const string GetFunctionPointerFieldName = "get_function_pointer"; - generator.WriteVariable (GetFunctionPointerFieldName, get_function_pointer_sig, LlvmIrVariableOptions.LocalWritableInsignificantAddr); + var assembly_image_cache_indices = new LlvmIrGlobalVariable (typeof(List), "assembly_image_cache_indices", LlvmIrVariableOptions.GlobalConstant) { + WriteOptions = LlvmIrVariableWriteOptions.ArrayWriteIndexComments | LlvmIrVariableWriteOptions.ArrayFormatInRows, + BeforeWriteCallback = UpdateAssemblyImageCacheIndices, + BeforeWriteCallbackCallerState = acs, + }; + module.Add (assembly_image_cache_indices); + } - var fnParameter = new LlvmIrFunctionParameter (get_function_pointer_sig, "fn"); - var func = new LlvmIrFunction ( - name: "xamarin_app_init", - returnType: typeof (void), - attributeSetID: LlvmIrGenerator.FunctionAttributesXamarinAppInit, - parameters: new List { - fnParameter, - } - ); + void UpdateAssemblyImageCacheHashes (LlvmIrVariable variable, LlvmIrModuleTarget target, object? callerState) + { + AssemblyCacheState acs = EnsureAssemblyCacheState (callerState); + object value; + Type type; - generator.WriteFunctionStart (func); - generator.EmitStoreInstruction (func, fnParameter, new LlvmIrVariableReference (get_function_pointer_sig, GetFunctionPointerFieldName, isGlobal: true)); - generator.WriteFunctionEnd (func); + if (target.Is64Bit) { + value = acs.Keys64; + type = typeof(List); + } else { + value = acs.Keys32; + type = typeof(List); + } - return new LlvmIrVariableReference (get_function_pointer_sig, GetFunctionPointerFieldName, isGlobal: true); + LlvmIrGlobalVariable gv = EnsureGlobalVariable (variable); + gv.OverrideValueAndType (type, value); } - void WriteClassCache (LlvmIrGenerator generator) + string? GetAssemblyImageCacheItemComment (LlvmIrVariable v, LlvmIrModuleTarget target, ulong index, object? value, object? callerState) { - uint marshal_methods_number_of_classes = (uint)classes.Count; + AssemblyCacheState acs = EnsureAssemblyCacheState (callerState); + + string name; + uint i; + if (target.Is64Bit) { + var v64 = (ulong)value; + name = acs.Hashes64[v64].name; + i = acs.Hashes64[v64].index; + } else { + var v32 = (uint)value; + name = acs.Hashes32[v32].name; + i = acs.Hashes32[v32].index; + } - generator.WriteVariable (nameof (marshal_methods_number_of_classes), marshal_methods_number_of_classes); - generator.WriteStructureArray (marshalMethodsClass, classes, LlvmIrVariableOptions.GlobalWritable, "marshal_methods_class_cache"); + return $" {index}: {name} => 0x{value:x} => {i}"; } - // TODO: this should probably be moved to a separate writer, since not only marshal methods use the cache - void WriteAssemblyImageCache (LlvmIrGenerator generator, out Dictionary asmNameToIndex) + void UpdateAssemblyImageCacheIndices (LlvmIrVariable variable, LlvmIrModuleTarget target, object? callerState) { - bool is64Bit = generator.Is64Bit; - generator.WriteStructureArray (monoImage, (ulong)numberOfAssembliesInApk, "assembly_image_cache", isArrayOfPointers: true); + AssemblyCacheState acs = EnsureAssemblyCacheState (callerState); + object value; - var asmNameToIndexData = new Dictionary (StringComparer.Ordinal); - if (is64Bit) { - WriteHashes (); + if (target.Is64Bit) { + value = acs.Indices64; } else { - WriteHashes (); + value = acs.Indices32; } - asmNameToIndex = asmNameToIndexData; - - void WriteHashes () where T: struct - { - var hashes = new Dictionary (); - uint index = 0; - - foreach (string name in uniqueAssemblyNames) { - // We must make sure we keep the possible culture prefix, which will be treated as "directory" path here - string clippedName = Path.Combine (Path.GetDirectoryName (name) ?? String.Empty, Path.GetFileNameWithoutExtension (name)); - ulong hashFull = HashName (name, is64Bit); - ulong hashClipped = HashName (clippedName, is64Bit); - - // - // If the number of name forms changes, xamarin-app.hh MUST be updated to set value of the - // `number_of_assembly_name_forms_in_image_cache` constant to the number of forms. - // - hashes.Add ((T)Convert.ChangeType (hashFull, typeof(T)), (name, index)); - hashes.Add ((T)Convert.ChangeType (hashClipped, typeof(T)), (clippedName, index)); - - index++; - } - List keys = hashes.Keys.ToList (); - keys.Sort (); - - generator.WriteCommentLine ("Each entry maps hash of an assembly name to an index into the `assembly_image_cache` array"); - generator.WriteArray ( - keys, - LlvmIrVariableOptions.GlobalConstant, - "assembly_image_cache_hashes", - (int idx, T value) => $"{idx}: {hashes[value].name} => 0x{value:x} => {hashes[value].index}" - ); + LlvmIrGlobalVariable gv = EnsureGlobalVariable (variable); + gv.OverrideValueAndType (variable.Type, value); + } - var indices = new List (); - for (int i = 0; i < keys.Count; i++) { - (string name, uint idx) = hashes[keys[i]]; - indices.Add (idx); - asmNameToIndexData.Add (name, idx); - } - generator.WriteArray ( - indices, - LlvmIrVariableOptions.GlobalConstant, - "assembly_image_cache_indices" - ); + AssemblyCacheState EnsureAssemblyCacheState (object? callerState) + { + var acs = callerState as AssemblyCacheState; + if (acs == null) { + throw new InvalidOperationException ("Internal error: construction state expected but not found"); } + + return acs; } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs index f59656518fc..31a501c1d94 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs @@ -211,9 +211,8 @@ bool GenerateDebugFiles (bool skipJniAddNativeMethodRegistrationAttributeScan, L } GeneratedBinaryTypeMaps.Add (typeMapIndexPath); - var generator = new TypeMappingDebugNativeAssemblyGenerator (new ModuleDebugData ()); - generator.Init (); - GenerateNativeAssembly (generator, outputDirectory); + var composer = new TypeMappingDebugNativeAssemblyGenerator (new ModuleDebugData ()); + GenerateNativeAssembly (composer, composer.Construct (), outputDirectory); return true; } @@ -243,9 +242,8 @@ bool GenerateDebugNativeAssembly (bool skipJniAddNativeMethodRegistrationAttribu PrepareDebugMaps (data); - var generator = new TypeMappingDebugNativeAssemblyGenerator (data); - generator.Init (); - GenerateNativeAssembly (generator, outputDirectory); + var composer = new TypeMappingDebugNativeAssemblyGenerator (data); + GenerateNativeAssembly (composer, composer.Construct (), outputDirectory); return true; } @@ -414,9 +412,8 @@ bool GenerateRelease (bool skipJniAddNativeMethodRegistrationAttributeScan, List NativeTypeMappingData data; data = new NativeTypeMappingData (logger, modules); - var generator = new TypeMappingReleaseNativeAssemblyGenerator (data); - generator.Init (); - GenerateNativeAssembly (generator, outputDirectory); + var composer = new TypeMappingReleaseNativeAssemblyGenerator (data); + GenerateNativeAssembly (composer, composer.Construct (), outputDirectory); return true; } @@ -426,36 +423,22 @@ bool ShouldSkipInJavaToManaged (TypeDefinition td) return td.IsInterface || td.HasGenericParameters; } - void GenerateNativeAssembly (TypeMappingAssemblyGenerator generator, string baseFileName) + void GenerateNativeAssembly (LLVMIR.LlvmIrComposer composer, LLVMIR.LlvmIrModule typeMapModule, string baseFileName) { AndroidTargetArch arch; foreach (string abi in supportedAbis) { - switch (abi.Trim ()) { - case "armeabi-v7a": - arch = AndroidTargetArch.Arm; - break; - - case "arm64-v8a": - arch = AndroidTargetArch.Arm64; - break; - - case "x86": - arch = AndroidTargetArch.X86; - break; - - case "x86_64": - arch = AndroidTargetArch.X86_64; - break; - - default: - throw new InvalidOperationException ($"Unknown ABI {abi}"); - } + arch = GeneratePackageManagerJava.GetAndroidTargetArchForAbi (abi); string outputFile = $"{baseFileName}.{abi}.ll"; - using (var sw = MemoryStreamPool.Shared.CreateStreamWriter (outputEncoding)) { - generator.Write (arch, sw, outputFile); - sw.Flush (); - Files.CopyIfStreamChanged (sw.BaseStream, outputFile); + using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) { + try { + composer.Generate (typeMapModule, arch, sw, outputFile); + } catch { + throw; + } finally { + sw.Flush (); + Files.CopyIfStreamChanged (sw.BaseStream, outputFile); + } } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingDebugNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingDebugNativeAssemblyGenerator.cs index 3f1917d791b..2d9e3388475 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingDebugNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingDebugNativeAssemblyGenerator.cs @@ -5,7 +5,7 @@ namespace Xamarin.Android.Tasks { - class TypeMappingDebugNativeAssemblyGenerator : TypeMappingAssemblyGenerator + class TypeMappingDebugNativeAssemblyGenerator : LlvmIrComposer { const string JavaToManagedSymbol = "map_java_to_managed"; const string ManagedToJavaSymbol = "map_managed_to_java"; @@ -110,8 +110,8 @@ sealed class TypeMap readonly TypeMapGenerator.ModuleDebugData data; - StructureInfo typeMapEntryStructureInfo; - StructureInfo typeMapStructureInfo; + StructureInfo typeMapEntryStructureInfo; + StructureInfo typeMapStructureInfo; List> javaToManagedMap; List> managedToJavaMap; StructureInstance type_map; @@ -124,15 +124,17 @@ public TypeMappingDebugNativeAssemblyGenerator (TypeMapGenerator.ModuleDebugData managedToJavaMap = new List> (); } - public override void Init () + protected override void Construct (LlvmIrModule module) { + MapStructures (module); + if (data.ManagedToJavaMap != null && data.ManagedToJavaMap.Count > 0) { foreach (TypeMapGenerator.TypeMapDebugEntry entry in data.ManagedToJavaMap) { var m2j = new TypeMapEntry { from = entry.ManagedName, to = entry.JavaName, }; - managedToJavaMap.Add (new StructureInstance (m2j)); + managedToJavaMap.Add (new StructureInstance (typeMapEntryStructureInfo, m2j)); } } @@ -144,7 +146,7 @@ public override void Init () from = entry.JavaName, to = managedEntry.SkipInJavaToManaged ? null : managedEntry.ManagedName, }; - javaToManagedMap.Add (new StructureInstance (j2m)); + javaToManagedMap.Add (new StructureInstance (typeMapEntryStructureInfo, j2m)); } } @@ -154,26 +156,22 @@ public override void Init () entry_count = data.EntryCount, }; - type_map = new StructureInstance (map); - } - - protected override void MapStructures (LlvmIrGenerator generator) - { - typeMapEntryStructureInfo = generator.MapStructure (); - typeMapStructureInfo = generator.MapStructure (); - } + type_map = new StructureInstance (typeMapStructureInfo, map); + module.AddGlobalVariable (TypeMapSymbol, type_map, LlvmIrVariableOptions.GlobalConstant); - protected override void Write (LlvmIrGenerator generator) - { if (managedToJavaMap.Count > 0) { - generator.WriteStructureArray (typeMapEntryStructureInfo, managedToJavaMap, LlvmIrVariableOptions.LocalConstant, ManagedToJavaSymbol); + module.AddGlobalVariable (ManagedToJavaSymbol, managedToJavaMap, LlvmIrVariableOptions.LocalConstant); } if (javaToManagedMap.Count > 0) { - generator.WriteStructureArray (typeMapEntryStructureInfo, javaToManagedMap, LlvmIrVariableOptions.LocalConstant, JavaToManagedSymbol); + module.AddGlobalVariable (JavaToManagedSymbol, javaToManagedMap, LlvmIrVariableOptions.LocalConstant); } + } - generator.WriteStructure (typeMapStructureInfo, type_map, LlvmIrVariableOptions.GlobalConstant, TypeMapSymbol); + void MapStructures (LlvmIrModule module) + { + typeMapEntryStructureInfo = module.MapStructure (); + typeMapStructureInfo = module.MapStructure (); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs index bc407b478aa..a6e4fd465df 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs @@ -1,14 +1,14 @@ using System; +using System.Collections; using System.Collections.Generic; using System.IO.Hashing; -using System.Linq; using System.Text; using Xamarin.Android.Tasks.LLVMIR; namespace Xamarin.Android.Tasks { - class TypeMappingReleaseNativeAssemblyGenerator : TypeMappingAssemblyGenerator + partial class TypeMappingReleaseNativeAssemblyGenerator : LlvmIrComposer { sealed class TypeMapModuleContextDataProvider : NativeAssemblerStructContextDataProvider { @@ -17,11 +17,11 @@ public override string GetComment (object data, string fieldName) var map_module = EnsureType (data); if (String.Compare ("module_uuid", fieldName, StringComparison.Ordinal) == 0) { - return $"module_uuid: {map_module.MVID}"; + return $" module_uuid: {map_module.MVID}"; } if (String.Compare ("assembly_name", fieldName, StringComparison.Ordinal) == 0) { - return $"assembly_name: {map_module.assembly_name}"; + return $" assembly_name: {map_module.assembly_name}"; } return String.Empty; @@ -58,14 +58,6 @@ public override ulong GetBufferSize (object data, string fieldName) } } - sealed class JavaNameHashComparer : IComparer> - { - public int Compare (StructureInstance a, StructureInstance b) - { - return a.Obj.JavaNameHash.CompareTo (b.Obj.JavaNameHash); - } - } - // This is here only to generate strongly-typed IR internal sealed class MonoImage {} @@ -74,6 +66,9 @@ internal sealed class MonoImage // src/monodroid/jni/xamarin-app.hh TypeMapModuleEntry structure sealed class TypeMapModuleEntry { + [NativeAssembler (Ignore = true)] + public TypeMapJava JavaTypeMapEntry; + public uint type_token_id; public uint java_map_index; } @@ -125,7 +120,10 @@ sealed class TypeMapJava public string JavaName; [NativeAssembler (Ignore = true)] - public ulong JavaNameHash; + public uint JavaNameHash32; + + [NativeAssembler (Ignore = true)] + public ulong JavaNameHash64; public uint module_index; public uint type_token_id; @@ -144,62 +142,195 @@ public ModuleMapData (string symbolLabel, List> + { + public int Compare (StructureInstance a, StructureInstance b) + { + return a.Instance.JavaNameHash32.CompareTo (b.Instance.JavaNameHash32); + } + } + + sealed class JavaNameHash64Comparer : IComparer> + { + public int Compare (StructureInstance a, StructureInstance b) + { + return a.Instance.JavaNameHash64.CompareTo (b.Instance.JavaNameHash64); + } + } + + sealed class ConstructionState + { + public List> MapModules; + public Dictionary JavaTypesByName; + public List JavaNames; + public List> JavaMap; + public List AllModulesData; + } + readonly NativeTypeMappingData mappingData; - StructureInfo typeMapJavaStructureInfo; - StructureInfo typeMapModuleStructureInfo; - StructureInfo typeMapModuleEntryStructureInfo; - List> mapModules; - List> javaMap; - Dictionary javaTypesByName; - List javaNames; - JavaNameHashComparer javaNameHashComparer; + StructureInfo typeMapJavaStructureInfo; + StructureInfo typeMapModuleStructureInfo; + StructureInfo typeMapModuleEntryStructureInfo; + JavaNameHash32Comparer javaNameHash32Comparer; + JavaNameHash64Comparer javaNameHash64Comparer; ulong moduleCounter = 0; public TypeMappingReleaseNativeAssemblyGenerator (NativeTypeMappingData mappingData) { this.mappingData = mappingData ?? throw new ArgumentNullException (nameof (mappingData)); - mapModules = new List> (); - javaMap = new List> (); - javaTypesByName = new Dictionary (StringComparer.Ordinal); - javaNameHashComparer = new JavaNameHashComparer (); - javaNames = new List (); + javaNameHash32Comparer = new JavaNameHash32Comparer (); + javaNameHash64Comparer = new JavaNameHash64Comparer (); + } + + protected override void Construct (LlvmIrModule module) + { + MapStructures (module); + + var cs = new ConstructionState (); + cs.JavaTypesByName = new Dictionary (StringComparer.Ordinal); + cs.JavaNames = new List (); + InitJavaMap (cs); + InitMapModules (cs); + HashJavaNames (cs); + PrepareModules (cs); + + module.AddGlobalVariable ("map_module_count", mappingData.MapModuleCount); + module.AddGlobalVariable ("java_type_count", cs.JavaMap.Count); + + var map_modules = new LlvmIrGlobalVariable (cs.MapModules, "map_modules", LlvmIrVariableOptions.GlobalWritable) { + Comment = " Managed modules map", + }; + module.Add (map_modules); + + // Java hashes are output bafore Java type map **and** managed modules, because they will also sort the Java map for us. + // This is not strictly necessary, as we could do the sorting in the java map BeforeWriteCallback, but this way we save + // time sorting only once. + var map_java_hashes = new LlvmIrGlobalVariable (typeof(List), "map_java_hashes") { + Comment = " Java types name hashes", + BeforeWriteCallback = GenerateAndSortJavaHashes, + BeforeWriteCallbackCallerState = cs, + GetArrayItemCommentCallback = GetJavaHashesItemComment, + GetArrayItemCommentCallbackCallerState = cs, + }; + map_java_hashes.WriteOptions &= ~LlvmIrVariableWriteOptions.ArrayWriteIndexComments; + module.Add (map_java_hashes); + + foreach (ModuleMapData mmd in cs.AllModulesData) { + var mmdVar = new LlvmIrGlobalVariable (mmd.Entries, mmd.SymbolLabel, LlvmIrVariableOptions.LocalConstant) { + BeforeWriteCallback = UpdateJavaIndexes, + BeforeWriteCallbackCallerState = cs, + }; + module.Add (mmdVar); + } + + module.AddGlobalVariable ("map_java", cs.JavaMap, LlvmIrVariableOptions.GlobalConstant, " Java to managed map"); + module.AddGlobalVariable ("java_type_names", cs.JavaNames, LlvmIrVariableOptions.GlobalConstant, " Java type names"); + } + + void UpdateJavaIndexes (LlvmIrVariable variable, LlvmIrModuleTarget target, object? callerState) + { + ConstructionState cs = EnsureConstructionState (callerState); + LlvmIrGlobalVariable gv = EnsureGlobalVariable (variable); + IComparer> hashComparer = target.Is64Bit ? javaNameHash64Comparer : javaNameHash32Comparer; + + var entries = (List>)variable.Value; + foreach (StructureInstance entry in entries) { + entry.Instance.java_map_index = GetJavaEntryIndex (entry.Instance.JavaTypeMapEntry); + } + + uint GetJavaEntryIndex (TypeMapJava javaEntry) + { + var key = new StructureInstance (typeMapJavaStructureInfo, javaEntry); + int idx = cs.JavaMap.BinarySearch (key, hashComparer); + if (idx < 0) { + throw new InvalidOperationException ($"Could not map entry '{javaEntry.JavaName}' to array index"); + } + + return (uint)idx; + } + } + + string? GetJavaHashesItemComment (LlvmIrVariable v, LlvmIrModuleTarget target, ulong index, object? value, object? callerState) + { + var cs = callerState as ConstructionState; + if (cs == null) { + throw new InvalidOperationException ("Internal error: construction state expected but not found"); + } + + return $" {index}: 0x{value:x} => {cs.JavaMap[(int)index].Instance.JavaName}"; + } + + void GenerateAndSortJavaHashes (LlvmIrVariable variable, LlvmIrModuleTarget target, object? callerState) + { + ConstructionState cs = EnsureConstructionState (callerState); + LlvmIrGlobalVariable gv = EnsureGlobalVariable (variable); + Type listType; + IList hashes; + if (target.Is64Bit) { + listType = typeof(List); + cs.JavaMap.Sort ((StructureInstance a, StructureInstance b) => a.Instance.JavaNameHash64.CompareTo (b.Instance.JavaNameHash64)); + + var list = new List (); + foreach (StructureInstance si in cs.JavaMap) { + list.Add (si.Instance.JavaNameHash64); + } + hashes = list; + } else { + listType = typeof(List); + cs.JavaMap.Sort ((StructureInstance a, StructureInstance b) => a.Instance.JavaNameHash32.CompareTo (b.Instance.JavaNameHash32)); + + var list = new List (); + foreach (StructureInstance si in cs.JavaMap) { + list.Add (si.Instance.JavaNameHash32); + } + hashes = list; + } + + gv.OverrideValueAndType (listType, hashes); } - public override void Init () + ConstructionState EnsureConstructionState (object? callerState) { - InitMapModules (); - InitJavaMap (); + var cs = callerState as ConstructionState; + if (cs == null) { + throw new InvalidOperationException ("Internal error: construction state expected but not found"); + } + + return cs; } - void InitJavaMap () + void InitJavaMap (ConstructionState cs) { + cs.JavaMap = new List> (); TypeMapJava map_entry; foreach (TypeMapGenerator.TypeMapReleaseEntry entry in mappingData.JavaTypes) { - javaNames.Add (entry.JavaName); + cs.JavaNames.Add (entry.JavaName); map_entry = new TypeMapJava { module_index = (uint)entry.ModuleIndex, // UInt32.MaxValue, type_token_id = entry.SkipInJavaToManaged ? 0 : entry.Token, - java_name_index = (uint)(javaNames.Count - 1), + java_name_index = (uint)(cs.JavaNames.Count - 1), JavaName = entry.JavaName, }; - javaMap.Add (new StructureInstance (map_entry)); - javaTypesByName.Add (map_entry.JavaName, map_entry); + cs.JavaMap.Add (new StructureInstance (typeMapJavaStructureInfo, map_entry)); + cs.JavaTypesByName.Add (map_entry.JavaName, map_entry); } } - void InitMapModules () + void InitMapModules (ConstructionState cs) { + cs.MapModules = new List> (); foreach (TypeMapGenerator.ModuleReleaseData data in mappingData.Modules) { string mapName = $"module{moduleCounter++}_managed_to_java"; string duplicateMapName; - if (data.DuplicateTypes.Count == 0) + if (data.DuplicateTypes.Count == 0) { duplicateMapName = String.Empty; - else + } else { duplicateMapName = $"{mapName}_duplicates"; + } var map_module = new TypeMapModule { MVID = data.Mvid, @@ -214,85 +345,69 @@ void InitMapModules () java_name_width = 0, }; - mapModules.Add (new StructureInstance (map_module)); + cs.MapModules.Add (new StructureInstance (typeMapModuleStructureInfo, map_module)); } } - protected override void MapStructures (LlvmIrGenerator generator) + void MapStructures (LlvmIrModule module) { - generator.MapStructure (); - typeMapJavaStructureInfo = generator.MapStructure (); - typeMapModuleStructureInfo = generator.MapStructure (); - typeMapModuleEntryStructureInfo = generator.MapStructure (); + typeMapJavaStructureInfo = module.MapStructure (); + typeMapModuleStructureInfo = module.MapStructure (); + typeMapModuleEntryStructureInfo = module.MapStructure (); } - // Prepare module map entries by sorting them on the managed token, and then mapping each entry to its corresponding Java type map index. - // Requires that `javaMap` is sorted on the type name hash. - void PrepareMapModuleData (string moduleDataSymbolLabel, IEnumerable moduleEntries, List allModulesData) + void PrepareMapModuleData (string moduleDataSymbolLabel, IEnumerable moduleEntries, ConstructionState cs) { var mapModuleEntries = new List> (); foreach (TypeMapGenerator.TypeMapReleaseEntry entry in moduleEntries) { + if (!cs.JavaTypesByName.TryGetValue (entry.JavaName, out TypeMapJava javaType)) { + throw new InvalidOperationException ($"Internal error: Java type '{entry.JavaName}' not found in cache"); + } + var map_entry = new TypeMapModuleEntry { + JavaTypeMapEntry = javaType, type_token_id = entry.Token, - java_map_index = GetJavaEntryIndex (entry.JavaName), + java_map_index = UInt32.MaxValue, // will be set later, when the target is known }; - mapModuleEntries.Add (new StructureInstance (map_entry)); + mapModuleEntries.Add (new StructureInstance (typeMapModuleEntryStructureInfo, map_entry)); } - mapModuleEntries.Sort ((StructureInstance a, StructureInstance b) => a.Obj.type_token_id.CompareTo (b.Obj.type_token_id)); - allModulesData.Add (new ModuleMapData (moduleDataSymbolLabel, mapModuleEntries)); - - uint GetJavaEntryIndex (string javaTypeName) - { - if (!javaTypesByName.TryGetValue (javaTypeName, out TypeMapJava javaType)) { - throw new InvalidOperationException ($"INTERNAL ERROR: Java type '{javaTypeName}' not found in cache"); - } + mapModuleEntries.Sort ((StructureInstance a, StructureInstance b) => a.Instance.type_token_id.CompareTo (b.Instance.type_token_id)); + cs.AllModulesData.Add (new ModuleMapData (moduleDataSymbolLabel, mapModuleEntries)); + } - var key = new StructureInstance (javaType); - int idx = javaMap.BinarySearch (key, javaNameHashComparer); - if (idx < 0) { - throw new InvalidOperationException ($"Could not map entry '{javaTypeName}' to array index"); + void PrepareModules (ConstructionState cs) + { + cs.AllModulesData = new List (); + foreach (StructureInstance moduleInstance in cs.MapModules) { + TypeMapModule module = moduleInstance.Instance; + PrepareMapModuleData (module.MapSymbolName, module.Data.Types, cs); + if (module.Data.DuplicateTypes.Count > 0) { + PrepareMapModuleData (module.DuplicateMapSymbolName, module.Data.DuplicateTypes, cs); } - - return (uint)idx; } } - // Generate hashes for all Java type names, then sort javaMap on the name hash. This has to be done in the writing phase because hashes - // will depend on architecture (or, actually, on its bitness) and may differ between architectures (they will be the same for all architectures - // with the same bitness) - (List allMapModulesData, List javaMapHashes) PrepareMapsForWriting (LlvmIrGenerator generator) + void HashJavaNames (ConstructionState cs) { - bool is64Bit = generator.Is64Bit; + // We generate both 32-bit and 64-bit hashes at the construction time. Which set will be used depends on the target. + // Java map list will also be sorted when the target is known + var hashes32 = new HashSet (); + var hashes64 = new HashSet (); // Generate Java type name hashes... - for (int i = 0; i < javaMap.Count; i++) { - TypeMapJava entry = javaMap[i].Obj; - entry.JavaNameHash = HashName (entry.JavaName); - } + for (int i = 0; i < cs.JavaMap.Count; i++) { + TypeMapJava entry = cs.JavaMap[i].Instance; - // ...sort them... - javaMap.Sort ((StructureInstance a, StructureInstance b) => a.Obj.JavaNameHash.CompareTo (b.Obj.JavaNameHash)); - - var allMapModulesData = new List (); - - // ...and match managed types to Java... - foreach (StructureInstance moduleInstance in mapModules) { - TypeMapModule module = moduleInstance.Obj; - PrepareMapModuleData (module.MapSymbolName, module.Data.Types, allMapModulesData); - if (module.Data.DuplicateTypes.Count > 0) { - PrepareMapModuleData (module.DuplicateMapSymbolName, module.Data.DuplicateTypes, allMapModulesData); - } - } + // The cast is safe, xxHash will return a 32-bit value which (for convenience) was upcast to 64-bit + entry.JavaNameHash32 = (uint)HashName (entry.JavaName, is64Bit: false); + hashes32.Add (entry.JavaNameHash32); - var javaMapHashes = new HashSet (); - foreach (StructureInstance entry in javaMap) { - javaMapHashes.Add (entry.Obj.JavaNameHash); + entry.JavaNameHash64 = HashName (entry.JavaName, is64Bit: true); + hashes64.Add (entry.JavaNameHash64); } - return (allMapModulesData, javaMapHashes.ToList ()); - - ulong HashName (string name) + ulong HashName (string name, bool is64Bit) { if (name.Length == 0) { return UInt64.MaxValue; @@ -300,10 +415,10 @@ ulong HashName (string name) // Native code (EmbeddedAssemblies::typemap_java_to_managed in embedded-assemblies.cc) will operate on wchar_t cast to a byte array, we need to do // the same - return HashBytes (Encoding.Unicode.GetBytes (name)); + return HashBytes (Encoding.Unicode.GetBytes (name), is64Bit); } - ulong HashBytes (byte[] bytes) + ulong HashBytes (byte[] bytes, bool is64Bit) { if (is64Bit) { return XxHash64.HashToUInt64 (bytes); @@ -312,79 +427,5 @@ ulong HashBytes (byte[] bytes) return (ulong)XxHash32.HashToUInt32 (bytes); } } - - protected override void Write (LlvmIrGenerator generator) - { - generator.WriteVariable ("map_module_count", mappingData.MapModuleCount); - generator.WriteVariable ("java_type_count", javaMap.Count); // must include the padding item, if any - - (List allMapModulesData, List javaMapHashes) = PrepareMapsForWriting (generator); - WriteMapModules (generator, allMapModulesData); - WriteJavaMap (generator, javaMapHashes); - } - - void WriteJavaMap (LlvmIrGenerator generator, List javaMapHashes) - { - generator.WriteEOL (); - generator.WriteEOL ("Java to managed map"); - - generator.WriteStructureArray ( - typeMapJavaStructureInfo, - javaMap, - LlvmIrVariableOptions.GlobalConstant, - "map_java" - ); - - if (generator.Is64Bit) { - WriteHashes (javaMapHashes); - } else { - // A bit ugly, but simple. We know that hashes are really 32-bit, so we can cast without - // worrying. - var hashes = new List (javaMapHashes.Count); - foreach (ulong hash in javaMapHashes) { - hashes.Add ((uint)hash); - } - WriteHashes (hashes); - } - - generator.WriteArray (javaNames, "java_type_names"); - - void WriteHashes (List hashes) where T: struct - { - generator.WriteArray ( - hashes, - LlvmIrVariableOptions.GlobalConstant, - "map_java_hashes", - (int idx, T value) => $"{idx}: 0x{value:x} => {javaMap[idx].Obj.JavaName}" - ); - } - } - - void WriteMapModules (LlvmIrGenerator generator, List mapModulesData) - { - if (mapModules.Count == 0) { - return; - } - - generator.WriteEOL (); - generator.WriteEOL ("Map modules data"); - - foreach (ModuleMapData mmd in mapModulesData) { - generator.WriteStructureArray ( - typeMapModuleEntryStructureInfo, - mmd.Entries, - LlvmIrVariableOptions.LocalConstant, - mmd.SymbolLabel - ); - } - - generator.WriteEOL ("Map modules"); - generator.WriteStructureArray ( - typeMapModuleStructureInfo, - mapModules, - LlvmIrVariableOptions.GlobalWritable, - "map_modules" - ); - } } } From 12e36e76cc832a6bc90ad0d9024ac6913dd1ad05 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 22 Jun 2023 12:15:14 +0200 Subject: [PATCH 02/71] Fix native assembly reader LLVM IR we use now generates slightly different native assembler code --- .../Utilities/EnvironmentHelper.cs | 2 +- .../Utilities/NativeAssemblyParser.cs | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs index 5b79d0e8309..afd2c24fbab 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs @@ -402,7 +402,7 @@ static Dictionary ReadEnvironmentVariables (EnvironmentFile envF static string[] GetField (string llvmAssemblerFile, string nativeAssemblerFile, string line, ulong lineNumber) { string[] ret = line?.Trim ()?.Split ('\t'); - Assert.IsTrue (ret.Length >= 2, $"Invalid assembler field format in file '{nativeAssemblerFile}:{lineNumber}': '{line}'. File generated from '{llvmAssemblerFile}'"); + Assert.IsTrue (ret != null && ret.Length >= 2, $"Invalid assembler field format in file '{nativeAssemblerFile}:{lineNumber}': '{line}'. File generated from '{llvmAssemblerFile}'"); return ret; } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/NativeAssemblyParser.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/NativeAssemblyParser.cs index bfbf5fb709c..3a2aa982f0f 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/NativeAssemblyParser.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/NativeAssemblyParser.cs @@ -192,7 +192,7 @@ public SymbolMetadata (SymbolMetadataKind kind, string value = null) static readonly char[] splitOnWhitespace = new char[] { ' ', '\t' }; static readonly char[] splitOnComma = new char[] { ',' }; - static readonly Regex assemblerLabelRegex = new Regex ("^[_.a-zA-Z0-9]+:", RegexOptions.Compiled); + static readonly Regex assemblerLabelRegex = new Regex ("^[_.$a-zA-Z0-9]+:", RegexOptions.Compiled); Dictionary symbols = new Dictionary (StringComparer.Ordinal); Dictionary> symbolMetadata = new Dictionary> (StringComparer.Ordinal); @@ -238,8 +238,10 @@ void Load (string sourceFilePath) AssemblerSection currentSection = null; AssemblerSymbol currentSymbol = null; - string symbolName; + string symbolName = null; ulong lineNumber = 0; + bool addedNewSymbol = false; + foreach (string l in File.ReadLines (sourceFilePath, Encoding.UTF8)) { lineNumber++; @@ -253,6 +255,15 @@ void Load (string sourceFilePath) continue; } + if (addedNewSymbol) { + addedNewSymbol = false; + // Some forms of LLVM IR can generate two labels for a single symbol, depending on symbol visibility, attributes and llc parameters. + // The exported symbol name 'symbol:' may be followed by another one '.Lsymbol$local:', we need to detect this and ignore the new symbol. + if (assemblerLabelRegex.IsMatch (line) && String.Compare (line.Trim (), $".L{symbolName}$local:", StringComparison.Ordinal) == 0) { + continue; + } + } + if (StartsNewSection (parts, ref currentSection)) { currentSymbol = null; // Symbols cannot cross sections continue; @@ -265,6 +276,7 @@ void Load (string sourceFilePath) if (assemblerLabelRegex.IsMatch (line)) { symbolName = GetSymbolName (line); currentSymbol = AddNewSymbol (symbolName); + addedNewSymbol = true; continue; } From 5939b448f2629e1158dfb841e38c3a2144d5f96c Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 22 Jun 2023 22:59:06 +0200 Subject: [PATCH 03/71] A handful of updates --- .../LlvmIrGenerator/LlvmIrFunction.cs | 13 +++++ .../LlvmIrFunctionAttributeSet.cs | 3 +- .../LlvmIrGenerator/LlvmIrFunctionBody.cs | 27 ++++++++++ .../LlvmIrGenerator/LlvmIrGenerator.cs | 21 +++++++- .../LlvmIrGenerator/LlvmIrInstructions.cs | 50 ++++++++++++++++++- .../MarshalMethodsNativeAssemblyGenerator.cs | 8 +-- 6 files changed, 113 insertions(+), 9 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunction.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunction.cs index d65a8fef5a6..43fec28bf65 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunction.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunction.cs @@ -22,6 +22,7 @@ sealed class SavedParameterState : ILlvmIrSavedFunctionParameterState public bool? SignExt; public bool? ZeroExt; public bool? IsCplusPlusReference; + public bool IsVarArgs; public SavedParameterState (LlvmIrFunctionParameter owner, SavedParameterState? previousState = null) { @@ -39,6 +40,8 @@ public SavedParameterState (LlvmIrFunctionParameter owner, SavedParameterState? ReadNone = previousState.ReadNone; SignExt = previousState.SignExt; ZeroExt = previousState.ZeroExt; + IsCplusPlusReference = previousState.IsCplusPlusReference; + IsVarArgs = previousState.IsVarArgs; } } @@ -137,6 +140,14 @@ public bool? IsCplusPlusReference { set => state.IsCplusPlusReference = value; } + /// + /// Indicates that the argument is a C variable arguments placeholder (`...`) + /// + public bool IsVarArgs { + get => state.IsVarArgs; + set => state.IsVarArgs = value; + } + public LlvmIrFunctionParameter (Type type, string? name = null) : base (type, name) { @@ -193,6 +204,7 @@ public sealed class ReturnTypeAttributes public bool? NoUndef; public bool? SignExt; public bool? ZeroExt; + public bool? NonNull; public ReturnTypeAttributes () {} @@ -201,6 +213,7 @@ public ReturnTypeAttributes (ReturnTypeAttributes other) { InReg = other.InReg; NoUndef = other.NoUndef; + NonNull = other.NonNull; SignExt = other.SignExt; ZeroExt = other.ZeroExt; } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunctionAttributeSet.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunctionAttributeSet.cs index f8d91e83058..7c890e13417 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunctionAttributeSet.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunctionAttributeSet.cs @@ -9,7 +9,8 @@ namespace Xamarin.Android.Tasks.LLVMIR; class LlvmIrFunctionAttributeSet : IEnumerable, IEquatable { - public uint Number { get; set; } = 0; + public uint Number { get; set; } = 0; + public bool DoNotAddTargetSpecificAttributes { get; set; } HashSet attributes; Dictionary>? privateTargetSpecificAttributes; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunctionBody.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunctionBody.cs index 3336b5d0b7d..e268e98afb7 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunctionBody.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunctionBody.cs @@ -171,6 +171,13 @@ public LlvmIrFunctionBody (LlvmIrFunction func, LlvmIrFunction.FunctionState fun previousLabel = implicitStartBlock = new LlvmIrFunctionImplicitStartLabel (functionState.StartingBlockNumber); } + public LlvmIrInstructions.Alloca Alloca (LlvmIrVariable result) + { + var ret = new LlvmIrInstructions.Alloca (result); + Add (ret); + return ret; + } + public LlvmIrInstructions.Br Br (LlvmIrFunctionLabelItem label) { var ret = new LlvmIrInstructions.Br (label); @@ -208,6 +215,26 @@ public LlvmIrInstructions.Load Load (LlvmIrVariable source, LlvmIrVariable resul return ret; } + public LlvmIrInstructions.Store Store (LlvmIrVariable from, LlvmIrVariable to, LlvmIrMetadataItem? tbaa = null) + { + var ret = new LlvmIrInstructions.Store (from, to) { + TBAA = tbaa, + }; + + Add (ret); + return ret; + } + + public LlvmIrInstructions.Store Store (LlvmIrVariable to, LlvmIrMetadataItem? tbaa = null) + { + var ret = new LlvmIrInstructions.Store (to) { + TBAA = tbaa, + }; + + Add (ret); + return ret; + } + /// /// Creates the `phi` instruction form we use the most throughout marshal methods generator - one which refers to an if/else block and where /// **both** value:label pairs are **required**. Parameters and are nullable because, in theory, diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs index 202d04a48c3..bb81b03db0e 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs @@ -1012,6 +1012,10 @@ public static void WriteReturnAttributes (GeneratorWriteContext context, LlvmIrF context.Output.Write ("noundef "); } + if (AttributeIsSet (returnAttrs.NonNull)) { + context.Output.Write ("nonnull "); + } + if (AttributeIsSet (returnAttrs.SignExt)) { context.Output.Write ("signext "); } @@ -1033,15 +1037,28 @@ void WriteFunctionSignature (GeneratorWriteContext context, LlvmIrFunction func, context.Output.Write ('('); bool first = true; + bool varargsFound = false; + foreach (LlvmIrFunctionParameter parameter in func.Signature.Parameters) { + if (varargsFound) { + throw new InvalidOperationException ($"Internal error: function '{func.Signature.Name}' has extra parameters following the C varargs parameter. This is not allowed."); + } + if (!first) { context.Output.Write (", "); } else { first = false; } + if (parameter.IsVarArgs) { + context.Output.Write ("..."); + varargsFound = true; + continue; + } + context.Output.Write (MapToIRType (parameter.Type)); WriteParameterAttributes (context, parameter); + if (writeParameterNames) { if (String.IsNullOrEmpty (parameter.Name)) { throw new InvalidOperationException ($"Internal error: parameter must have a name"); @@ -1126,7 +1143,9 @@ void WriteAttributeSets (GeneratorWriteContext context) foreach (LlvmIrFunctionAttributeSet attrSet in context.Module.AttributeSets) { // Must not modify the original set, it is shared with other targets. var targetSet = new LlvmIrFunctionAttributeSet (attrSet); - target.AddTargetSpecificAttributes (targetSet); + if (!attrSet.DoNotAddTargetSpecificAttributes) { + target.AddTargetSpecificAttributes (targetSet); + } IList? privateTargetSet = attrSet.GetPrivateTargetAttributes (target.TargetArch); if (privateTargetSet != null) { diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrInstructions.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrInstructions.cs index 2efa5e45db8..33297922814 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrInstructions.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrInstructions.cs @@ -104,8 +104,45 @@ protected LlvmIrInstructionArgumentValuePlaceholder () public abstract object? GetValue (LlvmIrModuleTarget target); } +class LlvmIrInstructionPointerSizeArgumentPlaceholder : LlvmIrInstructionArgumentValuePlaceholder +{ + public override object? GetValue (LlvmIrModuleTarget target) + { + return target.NativePointerSize; + } +} + sealed class LlvmIrInstructions { + public class Alloca : LlvmIrInstruction + { + LlvmIrVariable result; + + public Alloca (LlvmIrVariable result) + : base ("alloca") + { + this.result = result; + } + + protected override void WriteValueAssignment (GeneratorWriteContext context) + { + if (result == null) { + return; + } + + context.Output.Write (result.Reference); + context.Output.Write (" = "); + } + + protected override void WriteBody (GeneratorWriteContext context) + { + string irType = LlvmIrGenerator.MapToIRType (result.Type, out ulong size, out bool isPointer); + + context.Output.Write (irType); + WriteAlignment (context, size, isPointer); + } + } + public class Br : LlvmIrInstruction { const string OpName = "br"; @@ -452,16 +489,27 @@ protected override void WriteBody (GeneratorWriteContext context) public class Store : LlvmIrInstruction { + const string Opcode = "store"; + object? from; LlvmIrVariable to; public Store (LlvmIrVariable from, LlvmIrVariable to) - : base ("store") + : base (Opcode) { this.from = from; this.to = to; } + /// + /// Stores `null` in the indicated variable + /// + public Store (LlvmIrVariable to) + : base (Opcode) + { + this.to = to; + } + protected override void WriteBody (GeneratorWriteContext context) { string irType = LlvmIrGenerator.MapToIRType (to.Type, out ulong size, out bool isPointer); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs index 45ef98dba31..7f719606af4 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs @@ -812,12 +812,8 @@ LlvmIrFunctionAttributeSet MakeMarshalMethodAttributeSet (LlvmIrModule module) LlvmIrFunctionAttributeSet attrSet = MakeXamarinAppInitAttributeSet (module); var xamarin_app_init = new LlvmIrFunction (init_signature, attrSet); - xamarin_app_init.Body.Add ( - new LlvmIrInstructions.Store (init_params[1], getFunctionPtrVariable) { - TBAA = module.TbaaAnyPointer, - } - ); - xamarin_app_init.Body.Add (new LlvmIrInstructions.Ret (typeof(void))); + xamarin_app_init.Body.Store (init_params[1], getFunctionPtrVariable, module.TbaaAnyPointer); + xamarin_app_init.Body.Ret (typeof(void)); module.Add (xamarin_app_init); From 042858d00ba06a6b57a49d9d9ef339ac89b614c9 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 23 Jun 2023 23:03:04 +0200 Subject: [PATCH 04/71] A handful of updates --- .../LlvmIrGenerator/LlvmIrFunction.cs | 10 ++ .../LlvmIrGenerator/LlvmIrFunctionBody.cs | 103 ++++++++++-------- .../LlvmIrGenerator/LlvmIrInstructions.cs | 53 ++++++++- .../Utilities/LlvmIrGenerator/LlvmIrModule.cs | 2 +- 4 files changed, 122 insertions(+), 46 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunction.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunction.cs index 43fec28bf65..aa156ecaef9 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunction.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunction.cs @@ -414,6 +414,7 @@ public SavedFunctionState (LlvmIrFunction owner, ILlvmIrSavedFunctionSignatureSt public LlvmIrFunctionBody Body { get; } public string? Comment { get; set; } public bool ReturnsValue => Signature.ReturnType != typeof(void); + public bool UsesVarArgs { get; } public LlvmIrFunction (LlvmIrFunctionSignature signature, LlvmIrFunctionAttributeSet? attributeSet = null) { @@ -422,6 +423,15 @@ public LlvmIrFunction (LlvmIrFunctionSignature signature, LlvmIrFunctionAttribut functionState = new FunctionState (); foreach (LlvmIrFunctionParameter parameter in signature.Parameters) { + if (UsesVarArgs) { + throw new InvalidOperationException ($"Internal error: function '{signature.Name}' uses variable arguments and it has at least one argument following the varargs (...) one. This is not allowed."); + } + + if (parameter.IsVarArgs) { + UsesVarArgs = true; + continue; + } + if (!String.IsNullOrEmpty (parameter.Name)) { continue; } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunctionBody.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunctionBody.cs index e268e98afb7..2ac5d515d29 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunctionBody.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunctionBody.cs @@ -171,6 +171,43 @@ public LlvmIrFunctionBody (LlvmIrFunction func, LlvmIrFunction.FunctionState fun previousLabel = implicitStartBlock = new LlvmIrFunctionImplicitStartLabel (functionState.StartingBlockNumber); } + public void Add (LlvmIrFunctionLabelItem label) + { + label.WillAddToBody (this, functionState); + if (definedLabels.Contains (label.Name)) { + throw new InvalidOperationException ($"Internal error: label with name '{label.Name}' already added to function '{ownerFunction.Signature.Name}' body"); + } + items.Add (label); + definedLabels.Add (label.Name); + + // Rotate preceding blocks + if (precedingBlock2 != null) { + precedingBlock2 = null; + } + + precedingBlock2 = precedingBlock1; + precedingBlock1 = previousLabel; + previousLabel = label; + + var comment = new StringBuilder (" preds = %"); + comment.Append (precedingBlock1.Name); + if (precedingBlock2 != null) { + comment.Append (", %"); + comment.Append (precedingBlock2.Name); + } + label.Comment = comment.ToString (); + } + + public void Add (LlvmIrFunctionBodyItem item) + { + items.Add (item); + } + + public void AddComment (string text) + { + Add (new LlvmIrFunctionBodyComment (text)); + } + public LlvmIrInstructions.Alloca Alloca (LlvmIrVariable result) { var ret = new LlvmIrInstructions.Alloca (result); @@ -199,38 +236,25 @@ public LlvmIrInstructions.Call Call (LlvmIrFunction function, LlvmIrVariable? re return ret; } - public LlvmIrInstructions.Icmp Icmp (LlvmIrIcmpCond cond, LlvmIrVariable op1, object? op2, LlvmIrVariable result) - { - var ret = new LlvmIrInstructions.Icmp (cond, op1, op2, result); - Add (ret); - return ret; - } - - public LlvmIrInstructions.Load Load (LlvmIrVariable source, LlvmIrVariable result, LlvmIrMetadataItem? tbaa = null) + public LlvmIrInstructions.Ext Ext (LlvmIrVariable source, Type targetType, LlvmIrVariable result) { - var ret = new LlvmIrInstructions.Load (source, result) { - TBAA = tbaa, - }; + var ret = new LlvmIrInstructions.Ext (source, targetType, result); Add (ret); return ret; } - public LlvmIrInstructions.Store Store (LlvmIrVariable from, LlvmIrVariable to, LlvmIrMetadataItem? tbaa = null) + public LlvmIrInstructions.Icmp Icmp (LlvmIrIcmpCond cond, LlvmIrVariable op1, object? op2, LlvmIrVariable result) { - var ret = new LlvmIrInstructions.Store (from, to) { - TBAA = tbaa, - }; - + var ret = new LlvmIrInstructions.Icmp (cond, op1, op2, result); Add (ret); return ret; } - public LlvmIrInstructions.Store Store (LlvmIrVariable to, LlvmIrMetadataItem? tbaa = null) + public LlvmIrInstructions.Load Load (LlvmIrVariable source, LlvmIrVariable result, LlvmIrMetadataItem? tbaa = null) { - var ret = new LlvmIrInstructions.Store (to) { + var ret = new LlvmIrInstructions.Load (source, result) { TBAA = tbaa, }; - Add (ret); return ret; } @@ -255,35 +279,26 @@ public LlvmIrInstructions.Ret Ret (Type retvalType, object? retval = null) return ret; } - public void Add (LlvmIrFunctionLabelItem label) + public LlvmIrInstructions.Store Store (LlvmIrVariable from, LlvmIrVariable to, LlvmIrMetadataItem? tbaa = null) { - label.WillAddToBody (this, functionState); - if (definedLabels.Contains (label.Name)) { - throw new InvalidOperationException ($"Internal error: label with name '{label.Name}' already added to function '{ownerFunction.Signature.Name}' body"); - } - items.Add (label); - definedLabels.Add (label.Name); - - // Rotate preceding blocks - if (precedingBlock2 != null) { - precedingBlock2 = null; - } - - precedingBlock2 = precedingBlock1; - precedingBlock1 = previousLabel; - previousLabel = label; + var ret = new LlvmIrInstructions.Store (from, to) { + TBAA = tbaa, + }; - var comment = new StringBuilder (" preds = %"); - comment.Append (precedingBlock1.Name); - if (precedingBlock2 != null) { - comment.Append (", %"); - comment.Append (precedingBlock2.Name); - } - label.Comment = comment.ToString (); + Add (ret); + return ret; } - public void Add (LlvmIrFunctionBodyItem item) + /// + /// Stores `null` in the indicated variable + /// + public LlvmIrInstructions.Store Store (LlvmIrVariable to, LlvmIrMetadataItem? tbaa = null) { - items.Add (item); + var ret = new LlvmIrInstructions.Store (to) { + TBAA = tbaa, + }; + + Add (ret); + return ret; } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrInstructions.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrInstructions.cs index 33297922814..1a670106c1a 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrInstructions.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrInstructions.cs @@ -230,7 +230,11 @@ public Call (LlvmIrFunction function, LlvmIrVariable? result = null, ICollection throw new ArgumentNullException ($"Internal error: function '{function.Signature.Name}' requires {argCount} arguments", nameof (arguments)); } - if (arguments.Count != argCount) { + if (function.UsesVarArgs) { + if (arguments.Count < argCount) { + throw new ArgumentException ($"Internal error: varargs function '{function.Signature.Name}' needs at least {argCount} fixed arguments, got {arguments.Count} instead"); + } + } else if (arguments.Count != argCount) { throw new ArgumentException ($"Internal error: function '{function.Signature.Name}' requires {argCount} arguments, but {arguments.Count} were provided", nameof (arguments)); } @@ -334,6 +338,53 @@ void WriteArgument (GeneratorWriteContext context, LlvmIrFunctionParameter param } } + public class Ext : LlvmIrInstruction + { + const string FpextOpCode = "fpext"; + const string SextOpCode = "sext"; + const string ZextOpCode = "zext"; + + LlvmIrVariable result; + LlvmIrVariable source; + Type targetType; + + public Ext (LlvmIrVariable source, Type targetType, LlvmIrVariable result) + : base (GetOpCode (targetType)) + { + this.source = source; + this.targetType = targetType; + this.result = result; + } + + protected override void WriteValueAssignment (GeneratorWriteContext context) + { + context.Output.Write (result.Reference); + context.Output.Write (" = "); + } + + protected override void WriteBody (GeneratorWriteContext context) + { + context.Output.Write (LlvmIrGenerator.MapToIRType (source.Type)); + context.Output.Write (' '); + context.Output.Write (source.Reference); + context.Output.Write (" to "); + context.Output.Write ( LlvmIrGenerator.MapToIRType (targetType)); + } + + static string GetOpCode (Type targetType) + { + if (targetType == typeof(double)) { + return FpextOpCode; + } else if (targetType == typeof(int)) { + return SextOpCode; + } else if (targetType == typeof(uint)) { + return ZextOpCode; + } else { + throw new InvalidOperationException ($"Unsupported target type for upcasting: {targetType}"); + } + } + } + public class Icmp : LlvmIrInstruction { LlvmIrIcmpCond cond; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModule.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModule.cs index f7094dbbd42..170f2c6a860 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModule.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModule.cs @@ -279,7 +279,7 @@ void RegisterString (LlvmIrGlobalVariable variable, string? stringGroupName = nu RegisterString ((string)variable.Value, stringGroupName, stringGroupComment, symbolSuffix); } - void RegisterString (string value, string? stringGroupName = null, string? stringGroupComment = null, string? symbolSuffix = null) + public void RegisterString (string value, string? stringGroupName = null, string? stringGroupComment = null, string? symbolSuffix = null) { if (stringManager == null) { stringManager = new LlvmIrStringManager (); From 3ed01e9fa395892a1eb77b6c5cf0aebe737b0447 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 26 Jun 2023 21:56:15 +0200 Subject: [PATCH 05/71] Add some more tracing capabilities --- .../LlvmIrGenerator/LlvmIrFunctionBody.cs | 8 ++ .../LlvmIrGenerator/LlvmIrInstructions.cs | 7 + .../Utilities/LlvmIrGenerator/LlvmIrModule.cs | 121 ++++++++++++++++++ 3 files changed, 136 insertions(+) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunctionBody.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunctionBody.cs index 2ac5d515d29..5c1b03990ae 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunctionBody.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrFunctionBody.cs @@ -301,4 +301,12 @@ public LlvmIrInstructions.Store Store (LlvmIrVariable to, LlvmIrMetadataItem? tb Add (ret); return ret; } + + public LlvmIrInstructions.Unreachable Unreachable () + { + var ret = new LlvmIrInstructions.Unreachable (); + + Add (ret); + return ret; + } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrInstructions.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrInstructions.cs index 1a670106c1a..d6885cf2734 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrInstructions.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrInstructions.cs @@ -575,4 +575,11 @@ protected override void WriteBody (GeneratorWriteContext context) WriteAlignment (context, size, isPointer); } } + + public class Unreachable : LlvmIrInstruction + { + public Unreachable () + : base ("unreachable") + {} + } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModule.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModule.cs index 170f2c6a860..3dd0c72290e 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModule.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModule.cs @@ -40,6 +40,9 @@ partial class LlvmIrModule List? globalVariables; + LlvmIrFunction? puts; + LlvmIrFunction? abort; + public LlvmIrModule () { metadataManager = new LlvmIrMetadataManager (); @@ -125,6 +128,119 @@ public void Add (LlvmIrFunction func) functions.Add (func, func); } + public LlvmIrInstructions.Call CreatePuts (string text, LlvmIrVariable result) + { + EnsurePuts (); + RegisterString (text); + return new LlvmIrInstructions.Call (puts, result, new List { text }); + } + + /// + /// Generate code to call the `puts(3)` C library function to print a simple string to standard output. + /// + public LlvmIrInstructions.Call AddPuts (LlvmIrFunction function, string text, LlvmIrVariable result) + { + EnsurePuts (); + RegisterString (text); + return function.Body.Call (puts, result, new List { text }); + } + + void EnsurePuts () + { + if (puts != null) { + return; + } + + var puts_params = new List { + new (typeof(string), "s"), + }; + + var puts_sig = new LlvmIrFunctionSignature ( + name: "puts", + returnType: typeof(int), + parameters: puts_params + ); + puts_sig.ReturnAttributes.NoUndef = true; + + puts = DeclareExternalFunction (puts_sig, MakePutsAttributeSet ()); + } + + LlvmIrFunctionAttributeSet MakePutsAttributeSet () + { + var ret = new LlvmIrFunctionAttributeSet { + new NofreeFunctionAttribute (), + new NounwindFunctionAttribute (), + }; + + ret.DoNotAddTargetSpecificAttributes = true; + return AddAttributeSet (ret); + } + + public LlvmIrInstructions.Call CreateAbort () + { + EnsureAbort (); + return new LlvmIrInstructions.Call (abort); + } + + public LlvmIrInstructions.Call AddAbort (LlvmIrFunction function) + { + EnsureAbort (); + LlvmIrInstructions.Call ret = function.Body.Call (abort); + function.Body.Unreachable (); + + return ret; + } + + void EnsureAbort () + { + if (abort != null) { + return; + } + + var abort_sig = new LlvmIrFunctionSignature (name: "abort", returnType: typeof(void)); + abort = DeclareExternalFunction (abort_sig, MakeAbortAttributeSet ()); + } + + LlvmIrFunctionAttributeSet MakeAbortAttributeSet () + { + var ret = new LlvmIrFunctionAttributeSet { + new NoreturnFunctionAttribute (), + new NounwindFunctionAttribute (), + new NoTrappingMathFunctionAttribute (true), + new StackProtectorBufferSizeFunctionAttribute (8), + }; + + return AddAttributeSet (ret); + } + + public void AddIfThenElse (LlvmIrFunction function, LlvmIrVariable result, LlvmIrIcmpCond condition, LlvmIrVariable conditionVariable, object? conditionComparand, ICollection codeIfThen, ICollection? codeIfElse = null) + { + function.Body.Icmp (condition, conditionVariable, conditionComparand, result); + + var labelIfThen = new LlvmIrFunctionLabelItem (); + LlvmIrFunctionLabelItem? labelIfElse = codeIfElse != null ? new LlvmIrFunctionLabelItem () : null; + var labelIfDone = new LlvmIrFunctionLabelItem (); + + function.Body.Br (result, labelIfThen, labelIfElse == null ? labelIfDone : labelIfElse); + function.Body.Add (labelIfThen); + + AddInstructions (codeIfThen); + + if (codeIfElse != null) { + function.Body.Add (labelIfElse); + AddInstructions (codeIfElse); + } + + function.Body.Add (labelIfDone); + + void AddInstructions (ICollection instructions) + { + foreach (LlvmIrInstruction ins in instructions) { + function.Body.Add (ins); + } + } + } + /// /// A shortcut way to add a global variable without first having to create an instance of first. This overload /// requires the parameter to not be null. @@ -509,6 +625,11 @@ public LlvmIrFunction DeclareExternalFunction (LlvmIrFunction func) return func; } + public LlvmIrFunction DeclareExternalFunction (LlvmIrFunctionSignature sig, LlvmIrFunctionAttributeSet? attrSet = null) + { + return DeclareExternalFunction (new LlvmIrFunction (sig, attrSet)); + } + /// /// Since LLVM IR is strongly typed, it requires each structure to be properly declared before it is /// used throughout the code. This method uses reflection to scan the managed type From c573767285461836eb68a1a8b9423f6492b81855 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 27 Jun 2023 23:09:34 +0200 Subject: [PATCH 06/71] Another fixlet --- .../LlvmIrGenerator/LlvmIrInstructions.cs | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrInstructions.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrInstructions.cs index d6885cf2734..8f3be296bf9 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrInstructions.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrInstructions.cs @@ -42,10 +42,6 @@ protected override void DoWrite (GeneratorWriteContext context, LlvmIrGenerator context.Output.Write (" #"); context.Output.Write (AttributeSet.Number.ToString (CultureInfo.InvariantCulture)); } - - if (!String.IsNullOrEmpty (Comment)) { - generator.WriteComment (context, Comment); - } } /// @@ -284,24 +280,44 @@ protected override void WriteBody (GeneratorWriteContext context) } context.Output.Write ('('); + bool isVararg = false; for (int i = 0; i < function.Signature.Parameters.Count; i++) { if (i > 0) { context.Output.Write (", "); } - WriteArgument (context, function.Signature.Parameters[i], i); + LlvmIrFunctionParameter parameter = function.Signature.Parameters[i]; + if (parameter.IsVarArgs) { + isVararg = true; + } + + WriteArgument (context, parameter, i, isVararg); } context.Output.Write (')'); } - void WriteArgument (GeneratorWriteContext context, LlvmIrFunctionParameter parameter, int index) + void WriteArgument (GeneratorWriteContext context, LlvmIrFunctionParameter parameter, int index, bool isVararg) { - context.Output.Write (LlvmIrGenerator.MapToIRType (parameter.Type)); + object? value = arguments[index]; + string irType; + + if (!isVararg) { + irType = LlvmIrGenerator.MapToIRType (parameter.Type); + } else { + if (value == null) { + // We have no way of verifying the vararg parameter type if value is null, so we'll assume it's a pointer. + // If our assumption is wrong, llc will fail and signal the error + irType = "ptr"; + } else { + irType = LlvmIrGenerator.MapToIRType (value.GetType ()); + } + } + + context.Output.Write (irType); LlvmIrGenerator.WriteParameterAttributes (context, parameter); context.Output.Write (' '); - object? value = arguments[index]; if (value is LlvmIrInstructionArgumentValuePlaceholder placeholder) { value = placeholder.GetValue (context.Target); } From 1d43520de282f3a3767eeca100be0533e68423d8 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 28 Jun 2023 19:32:57 +0200 Subject: [PATCH 07/71] A handful of additions and changes --- .../Tasks/LinkApplicationSharedLibraries.cs | 31 ++++++-- .../LlvmIrGenerator/LlvmIrInstructions.cs | 48 +++++++++--- .../MarshalMethodsNativeAssemblyGenerator.cs | 23 +++++- .../Utilities/MonoAndroidHelper.cs | 40 ++++++++++ src/monodroid/CMakeLists.txt | 74 ++++++++----------- src/monodroid/libstub/stub.cc | 14 ++++ 6 files changed, 167 insertions(+), 63 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs b/src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs index bbe49073e8c..cc39da6215c 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs @@ -29,6 +29,7 @@ sealed class InputFiles { public List ObjectFiles; public string OutputSharedLibrary; + public List ExtraLibraries; } [Required] @@ -112,22 +113,23 @@ void RunLinker (Config config) IEnumerable GetLinkerConfigs () { + string runtimeNativeLibsDir = Path.GetFullPath (Path.Combine (AndroidBinUtilsDirectory, "..", "..", "..", "lib")); + string runtimeNativeLibStubsDir = Path.GetFullPath (Path.Combine (runtimeNativeLibsDir, "..", "libstubs")); var abis = new Dictionary (StringComparer.Ordinal); ITaskItem[] dsos = ApplicationSharedLibraries; foreach (ITaskItem item in dsos) { string abi = item.GetMetadata ("abi"); - abis [abi] = GatherFilesForABI(item.ItemSpec, abi, ObjectFiles); + abis [abi] = GatherFilesForABI (item.ItemSpec, abi, ObjectFiles, runtimeNativeLibsDir, runtimeNativeLibStubsDir); } const string commonLinkerArgs = - "--unresolved-symbols=ignore-in-shared-libs " + + "--shared " + + "--allow-shlib-undefined " + "--export-dynamic " + "-soname libxamarin-app.so " + "-z relro " + "-z noexecstack " + "--enable-new-dtags " + - "--eh-frame-hdr " + - "-shared " + "--build-id " + "--warn-shared-textrel " + "--fatal-warnings"; @@ -177,6 +179,12 @@ IEnumerable GetLinkerConfigs () targetLinkerArgs.Add ("-o"); targetLinkerArgs.Add (QuoteFileName (inputs.OutputSharedLibrary)); + if (inputs.ExtraLibraries != null) { + foreach (string lib in inputs.ExtraLibraries) { + targetLinkerArgs.Add (lib); + } + } + string targetArgs = String.Join (" ", targetLinkerArgs); yield return new Config { LinkerPath = ld, @@ -186,11 +194,24 @@ IEnumerable GetLinkerConfigs () } } - InputFiles GatherFilesForABI (string runtimeSharedLibrary, string abi, ITaskItem[] objectFiles) + InputFiles GatherFilesForABI (string runtimeSharedLibrary, string abi, ITaskItem[] objectFiles, string runtimeNativeLibsDir, string runtimeNativeLibStubsDir) { + List extraLibraries = null; + string RID = MonoAndroidHelper.AbiToRid (abi); + AndroidTargetArch targetArch = MonoAndroidHelper.AbiToTargetArch (abi); + string libStubsPath = Path.Combine (runtimeNativeLibStubsDir, RID); + string runtimeLibsDir = Path.Combine (runtimeNativeLibsDir, RID); + + extraLibraries = new List { + $"-L \"{runtimeLibsDir}\"", + $"-L \"{libStubsPath}\"", + "-lc", + }; + return new InputFiles { OutputSharedLibrary = runtimeSharedLibrary, ObjectFiles = GetItemsForABI (abi, objectFiles), + ExtraLibraries = extraLibraries, }; } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrInstructions.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrInstructions.cs index 8f3be296bf9..5ce7e98bd9d 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrInstructions.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrInstructions.cs @@ -271,6 +271,21 @@ protected override void WriteBody (GeneratorWriteContext context) } context.Output.Write (LlvmIrGenerator.MapToIRType (function.Signature.ReturnType)); + + if (function.UsesVarArgs) { + context.Output.Write (" ("); + for (int j = 0; j < function.Signature.Parameters.Count; j++) { + if (j > 0) { + context.Output.Write (", "); + } + + LlvmIrFunctionParameter parameter = function.Signature.Parameters[j]; + string irType = parameter.IsVarArgs ? "..." : LlvmIrGenerator.MapToIRType (parameter.Type); + context.Output.Write (irType); + } + context.Output.Write (')'); + } + if (FuncPointer == null) { context.Output.Write (" @"); context.Output.Write (function.Signature.Name); @@ -281,7 +296,8 @@ protected override void WriteBody (GeneratorWriteContext context) context.Output.Write ('('); bool isVararg = false; - for (int i = 0; i < function.Signature.Parameters.Count; i++) { + int i; + for (i = 0; i < function.Signature.Parameters.Count; i++) { if (i > 0) { context.Output.Write (", "); } @@ -294,16 +310,28 @@ protected override void WriteBody (GeneratorWriteContext context) WriteArgument (context, parameter, i, isVararg); } + if (arguments != null) { + for (; i < arguments.Count; i++) { + context.Output.Write (", "); + WriteArgument (context, null, i, isVararg: true); + } + } + context.Output.Write (')'); } - void WriteArgument (GeneratorWriteContext context, LlvmIrFunctionParameter parameter, int index, bool isVararg) + void WriteArgument (GeneratorWriteContext context, LlvmIrFunctionParameter? parameter, int index, bool isVararg) { object? value = arguments[index]; - string irType; + if (value is LlvmIrInstructionArgumentValuePlaceholder placeholder) { + value = placeholder.GetValue (context.Target); + } + string irType; if (!isVararg) { irType = LlvmIrGenerator.MapToIRType (parameter.Type); + } else if (value is LlvmIrVariable v1) { + irType = LlvmIrGenerator.MapToIRType (v1.Type); } else { if (value == null) { // We have no way of verifying the vararg parameter type if value is null, so we'll assume it's a pointer. @@ -315,12 +343,10 @@ void WriteArgument (GeneratorWriteContext context, LlvmIrFunctionParameter param } context.Output.Write (irType); - LlvmIrGenerator.WriteParameterAttributes (context, parameter); - context.Output.Write (' '); - - if (value is LlvmIrInstructionArgumentValuePlaceholder placeholder) { - value = placeholder.GetValue (context.Target); + if (parameter != null) { + LlvmIrGenerator.WriteParameterAttributes (context, parameter); } + context.Output.Write (' '); if (value == null) { if (!parameter.Type.IsNativePointer ()) { @@ -331,12 +357,12 @@ void WriteArgument (GeneratorWriteContext context, LlvmIrFunctionParameter param return; } - if (value is LlvmIrVariable variable) { - context.Output.Write (variable.Reference); + if (value is LlvmIrVariable v2) { + context.Output.Write (v2.Reference); return; } - if (!parameter.Type.IsAssignableFrom (value.GetType ())) { + if (parameter != null && !parameter.Type.IsAssignableFrom (value.GetType ())) { throw new InvalidOperationException ($"Internal error: value type '{value.GetType ()}' for argument {index} to function '{function.Signature.Name}' is invalid. Expected '{parameter.Type}' or compatible"); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs index 7f719606af4..f87cf118971 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs @@ -262,7 +262,7 @@ public MarshalMethodsNativeAssemblyGenerator (int numberOfAssembliesInApk, IColl this.logger = logger ?? throw new ArgumentNullException (nameof (logger)); generateEmptyCode = false; - defaultCallMarker = LlvmIrCallMarker.None; + defaultCallMarker = LlvmIrCallMarker.Tail; } void Init () @@ -591,7 +591,6 @@ protected override void Construct (LlvmIrModule module) MapStructures (module); Init (); - // InitTracing (module); AddAssemblyImageCache (module, out AssemblyCacheState acs); // class cache @@ -669,6 +668,8 @@ void AddMarshalMethod (LlvmIrModule module, MarshalMethodInfo method, ulong asmI funcComment.AppendLine (nativeCallback.Module.Assembly.Name.FullName); funcComment.Append (" Registered: "); funcComment.AppendLine (method.Method.RegisteredMethod?.FullName ?? "none"); + funcComment.Append (" Implemented: "); + funcComment.AppendLine (method.Method.ImplementedMethod?.FullName ?? "none"); var func = new LlvmIrFunction (method.NativeSymbolName, method.ReturnType, method.Parameters, writeState.AttributeSet) { Comment = funcComment.ToString (), @@ -812,6 +813,24 @@ LlvmIrFunctionAttributeSet MakeMarshalMethodAttributeSet (LlvmIrModule module) LlvmIrFunctionAttributeSet attrSet = MakeXamarinAppInitAttributeSet (module); var xamarin_app_init = new LlvmIrFunction (init_signature, attrSet); + + // If `fn` is nullptr, print a message and abort... + // + // We must allocate result variables for both the null comparison and puts call here and with names, because + // labels and local unnamed variables must be numbered sequentially otherwise and the `AddIfThenElse` call will + // allocate up to 3 labels which would have been **defined** after these labels, but **used** before them - and + // thus the numbering sequence would be out of order and the .ll file wouldn't build. + var fnNullResult = xamarin_app_init.CreateLocalVariable (typeof(bool), "fnIsNull"); + LlvmIrVariable putsResult = xamarin_app_init.CreateLocalVariable (typeof(int), "putsResult"); + var ifThenInstructions = new List { + module.CreatePuts ("get_function_pointer MUST be specified\n", putsResult), + module.CreateAbort (), + new LlvmIrInstructions.Unreachable (), + }; + + module.AddIfThenElse (xamarin_app_init, fnNullResult, LlvmIrIcmpCond.Equal, init_params[1], null, ifThenInstructions); + + // ...otherwise store the pointer and return xamarin_app_init.Body.Store (init_params[1], getFunctionPtrVariable, module.TbaaAnyPointer); xamarin_app_init.Body.Ret (typeof(void)); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs index 8579f37a8c0..77fb0b8b356 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs @@ -536,6 +536,46 @@ public static string GetRelativePathForAndroidAsset (string assetsDirectory, ITa return path; } + public static string AbiToRid (string abi) + { + switch (abi) { + case "arm64-v8a": + return "android-arm64"; + + case "armeabi-v7a": + return "android-arm"; + + case "x86": + return "android-x86"; + + case "x86_64": + return "android-x64"; + + default: + throw new InvalidOperationException ($"Internal error: unsupported ABI '{abi}'"); + } + } + + public static AndroidTargetArch AbiToTargetArch (string abi) + { + switch (abi) { + case "arm64-v8a": + return AndroidTargetArch.Arm64; + + case "armeabi-v7a": + return AndroidTargetArch.Arm; + + case "x86": + return AndroidTargetArch.X86; + + case "x86_64": + return AndroidTargetArch.X86_64; + + default: + throw new InvalidOperationException ($"Internal error: unsupported ABI '{abi}'"); + } + } + public static string? CultureInvariantToString (object? obj) { if (obj == null) { diff --git a/src/monodroid/CMakeLists.txt b/src/monodroid/CMakeLists.txt index 3c4b7a69411..1e36816bed3 100644 --- a/src/monodroid/CMakeLists.txt +++ b/src/monodroid/CMakeLists.txt @@ -725,55 +725,39 @@ target_link_libraries( ) if(ANDROID AND ENABLE_NET AND (NOT ANALYZERS_ENABLED)) - add_library( - c - SHARED ${XAMARIN_STUB_LIB_SOURCES} - ) - - target_compile_definitions( - c - PRIVATE STUB_LIB_NAME=libc - ) - - target_compile_options( - c - PRIVATE -nostdlib -fno-exceptions -fno-rtti - ) - - target_link_options( - c - PRIVATE -nostdlib -fno-exceptions -fno-rtti - ) + macro(xa_add_stub_library _libname) + add_library( + ${_libname} + SHARED ${XAMARIN_STUB_LIB_SOURCES} + ) - set_target_properties( - c - PROPERTIES - LIBRARY_OUTPUT_DIRECTORY "${XA_LIBRARY_STUBS_OUTPUT_DIRECTORY}" - ) + string(TOUPPER ${_libname} _libname_uc) + target_compile_definitions( + ${_libname} + PRIVATE STUB_LIB_NAME=lib${_libname} IN_LIB${_libname_uc} + ) - add_library( - m - SHARED ${XAMARIN_STUB_LIB_SOURCES} - ) + target_compile_options( + ${_libname} + PRIVATE -nostdlib -fno-exceptions -fno-rtti + ) - target_compile_definitions( - m - PRIVATE STUB_LIB_NAME=libm - ) + target_link_options( + ${_libname} + PRIVATE -nostdlib -fno-exceptions -fno-rtti + ) - target_compile_options( - m - PRIVATE -nostdlib -fno-exceptions -fno-rtti - ) + set_target_properties( + ${_libname} + PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${XA_LIBRARY_STUBS_OUTPUT_DIRECTORY}" + ) + endmacro() - target_link_options( - m - PRIVATE -nostdlib -fno-exceptions -fno-rtti - ) + xa_add_stub_library(c) + xa_add_stub_library(m) - set_target_properties( - m - PROPERTIES - LIBRARY_OUTPUT_DIRECTORY "${XA_LIBRARY_STUBS_OUTPUT_DIRECTORY}" - ) + # These two are used by the marshal methods tracing library when linking libxamarin-app.so + xa_add_stub_library(log) + xa_add_stub_library(dl) endif() diff --git a/src/monodroid/libstub/stub.cc b/src/monodroid/libstub/stub.cc index be0ec6b7ec7..1b1c91385d6 100644 --- a/src/monodroid/libstub/stub.cc +++ b/src/monodroid/libstub/stub.cc @@ -6,3 +6,17 @@ void STUB_LIB_NAME () { // no-op } + +#if defined(IN_LIBC) +extern "C" { + [[gnu::weak]] int puts ([[maybe_unused]] const char *s) + { + return -1; + } + + [[gnu::weak]] void abort () + { + // no-op + } +} +#endif From 66f9566ebc6374c277b91a123b15fe72951cbc68 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 30 Jun 2023 22:13:53 +0200 Subject: [PATCH 08/71] Beginning to fix typemaps for per-abi assemblies --- .../Tasks/GenerateJavaStubs.cs | 260 ++++++++++++++---- .../Tasks/LinkerTests.cs | 54 +++- .../MarshalMethodsAssemblyRewriter.cs | 85 +++--- .../Utilities/MarshalMethodsClassifier.cs | 22 +- .../Utilities/XAJavaTypeScanner.cs | 173 ++++++++++++ 5 files changed, 478 insertions(+), 116 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 04286ff5d77..c780750ba0c 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.IO.MemoryMappedFiles; using System.Linq; using System.Reflection; using System.Text; @@ -145,15 +146,16 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) PackageNamingPolicy pnp; JavaNativeTypeManager.PackageNamingPolicy = Enum.TryParse (PackageNamingPolicy, out pnp) ? pnp : PackageNamingPolicyEnum.LowercaseCrc64; - Dictionary> marshalMethodsAssemblyPaths = null; + Dictionary>? abiSpecificAssembliesByPath = null; if (useMarshalMethods) { - marshalMethodsAssemblyPaths = new Dictionary> (StringComparer.Ordinal); + abiSpecificAssembliesByPath = new Dictionary> (StringComparer.Ordinal); } // Put every assembly we'll need in the resolver bool hasExportReference = false; bool haveMonoAndroid = false; var allTypemapAssemblies = new HashSet (StringComparer.OrdinalIgnoreCase); + var newAllTypemapAssemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); var userAssemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); foreach (var assembly in ResolvedAssemblies) { bool value; @@ -162,8 +164,20 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) continue; } - bool addAssembly = false; string fileName = Path.GetFileName (assembly.ItemSpec); + if (abiSpecificAssembliesByPath != null) { + string? abi = assembly.GetMetadata ("Abi"); + if (!String.IsNullOrEmpty (abi)) { + if (!abiSpecificAssembliesByPath.TryGetValue (fileName, out List? items)) { + items = new List (); + abiSpecificAssembliesByPath.Add (fileName, items); + } + + items.Add (assembly); + } + } + + bool addAssembly = false; if (!hasExportReference && String.Compare ("Mono.Android.Export.dll", fileName, StringComparison.OrdinalIgnoreCase) == 0) { hasExportReference = true; addAssembly = true; @@ -181,12 +195,12 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) if (addAssembly) { allTypemapAssemblies.Add (assembly.ItemSpec); + if (!newAllTypemapAssemblies.ContainsKey (assembly.ItemSpec)) { + newAllTypemapAssemblies.Add (assembly.ItemSpec, assembly); + } } res.Load (assembly.ItemSpec); - if (useMarshalMethods) { - StoreMarshalAssemblyPath (Path.GetFileNameWithoutExtension (assembly.ItemSpec), assembly); - } } // However we only want to look for JLO types in user code for Java stub code generation @@ -197,10 +211,13 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) } if (!allTypemapAssemblies.Contains (asm.ItemSpec)) allTypemapAssemblies.Add (asm.ItemSpec); + if (!newAllTypemapAssemblies.ContainsKey (asm.ItemSpec)) { + newAllTypemapAssemblies.Add (asm.ItemSpec, asm); + } + string name = Path.GetFileNameWithoutExtension (asm.ItemSpec); if (!userAssemblies.ContainsKey (name)) userAssemblies.Add (name, asm.ItemSpec); - StoreMarshalAssemblyPath (name, asm); } // Step 1 - Find all the JLO types @@ -208,9 +225,29 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) var scanner = new JavaTypeScanner (this.CreateTaskLogger (), cache) { ErrorOnCustomJavaObject = ErrorOnCustomJavaObject, }; - + var newCache = new TypeDefinitionCache (); + var newScanner = new XAJavaTypeScanner (Log, newCache) { + ErrorOnCustomJavaObject = ErrorOnCustomJavaObject, + }; + List newAllJavaTypes = newScanner.GetJavaTypes (newAllTypemapAssemblies.Values, res); List allJavaTypes = scanner.GetJavaTypes (allTypemapAssemblies, res); + Console.WriteLine ("All java types:"); + foreach (TypeDefinition td in allJavaTypes) { + Console.WriteLine ($" {td.GetPartialAssemblyQualifiedName (cache)} ({td.Module.FileName})"); + } + + Console.WriteLine (); + Console.WriteLine ("NEW all java types:"); + foreach (JavaType jt in newAllJavaTypes) { + Console.Write ($" {jt.Type.FullName}"); + if (jt.PerAbiTypes != null) { + Console.WriteLine (" (ABI-specific)"); + } else { + Console.WriteLine (); + } + } + var javaTypes = new List (); foreach (TypeDefinition td in allJavaTypes) { // Whem marshal methods are in use we do not want to skip non-user assemblies (such as Mono.Android) - we need to generate JCWs for them during @@ -237,27 +274,11 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) // in order to properly generate wrapper methods in the marshal methods assembly rewriter. // We don't care about those generated by us, since they won't contain the `XA_BROKEN_EXCEPTION_TRANSITIONS` variable we look for. var environmentParser = new EnvironmentFilesParser (); - var targetPaths = new List (); - if (!LinkingEnabled) { - targetPaths.Add (Path.GetDirectoryName (ResolvedAssemblies[0].ItemSpec)); - } else { - if (String.IsNullOrEmpty (IntermediateOutputDirectory)) { - throw new InvalidOperationException ($"Internal error: marshal methods require the `IntermediateOutputDirectory` property of the `GenerateJavaStubs` task to have a value"); - } + Dictionary assemblyPaths = AddMethodsFromAbiSpecificAssemblies (classifier, res, abiSpecificAssembliesByPath); - // If the property is set then, even if we have just one RID, the linked assemblies path will include the RID - if (!HaveMultipleRIDs && SupportedAbis.Length == 1) { - targetPaths.Add (Path.Combine (IntermediateOutputDirectory, "linked")); - } else { - foreach (string abi in SupportedAbis) { - targetPaths.Add (Path.Combine (IntermediateOutputDirectory, AbiToRid (abi), "linked")); - } - } - } - - var rewriter = new MarshalMethodsAssemblyRewriter (classifier.MarshalMethods, classifier.Assemblies, marshalMethodsAssemblyPaths, Log); - rewriter.Rewrite (res, targetPaths, environmentParser.AreBrokenExceptionTransitionsEnabled (Environments)); + var rewriter = new MarshalMethodsAssemblyRewriter (classifier.MarshalMethods, classifier.Assemblies, assemblyPaths, Log); + rewriter.Rewrite (res, environmentParser.AreBrokenExceptionTransitionsEnabled (Environments)); } // Step 3 - Generate type maps @@ -413,40 +434,35 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) Log.LogDebugMessage ($"Number of methods in the project that need marshal method wrappers: {classifier.WrappedMethodCount}"); } } + } - void StoreMarshalAssemblyPath (string name, ITaskItem asm) - { - if (!useMarshalMethods) { - return; - } - - // TODO: we need to keep paths to ALL the assemblies, we need to rewrite them for all RIDs eventually. Right now we rewrite them just for one RID - if (!marshalMethodsAssemblyPaths.TryGetValue (name, out HashSet assemblyPaths)) { - assemblyPaths = new HashSet (); - marshalMethodsAssemblyPaths.Add (name, assemblyPaths); - } - - assemblyPaths.Add (asm.ItemSpec); - } - - string AbiToRid (string abi) - { - switch (abi) { - case "arm64-v8a": - return "android-arm64"; + AssemblyDefinition LoadAssembly (string path, DirectoryAssemblyResolver? resolver = null) + { + string pdbPath = Path.ChangeExtension (path, ".pdb"); + var readerParameters = new ReaderParameters { + AssemblyResolver = resolver, + InMemory = false, + ReadingMode = ReadingMode.Immediate, + ReadSymbols = File.Exists (pdbPath), + ReadWrite = false, + }; - case "armeabi-v7a": - return "android-arm"; + MemoryMappedViewStream? viewStream = null; + try { + // Create stream because CreateFromFile(string, ...) uses FileShare.None which is too strict + using var fileStream = new FileStream (path, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, false); + using var mappedFile = MemoryMappedFile.CreateFromFile ( + fileStream, null, fileStream.Length, MemoryMappedFileAccess.Read, HandleInheritability.None, true); + viewStream = mappedFile.CreateViewStream (0, 0, MemoryMappedFileAccess.Read); - case "x86": - return "android-x86"; + AssemblyDefinition result = ModuleDefinition.ReadModule (viewStream, readerParameters).Assembly; - case "x86_64": - return "android-x64"; + // We transferred the ownership of the viewStream to the collection. + viewStream = null; - default: - throw new InvalidOperationException ($"Internal error: unsupported ABI '{abi}'"); - } + return result; + } finally { + viewStream?.Dispose (); } } @@ -573,5 +589,137 @@ void WriteTypeMappings (List types, TypeDefinitionCache cache) GeneratedBinaryTypeMaps = tmg.GeneratedBinaryTypeMaps.ToArray (); BuildEngine4.RegisterTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (ApplicationConfigTaskState.RegisterTaskObjectKey), appConfState, RegisteredTaskObjectLifetime.Build); } + + /// + /// + /// Classifier will see only unique assemblies, since that's what's processed by the JI type scanner - even though some assemblies may have + /// abi-specific features (e.g. inlined `IntPtr.Size` or processor-specific intrinsics), the **types** and **methods** will all be the same and, thus, + /// there's no point in scanning all of the additional copies of the same assembly. + /// + /// + /// This, however, doesn't work for the rewriter which needs to rewrite all of the copies so that they all have the same generated wrappers. In + /// order to do that, we need to go over the list of assemblies found by the classifier, see if they are abi-specific ones and then add all the + /// marshal methods from the abi-specific assembly copies, so that the rewriter can easily rewrite them all. + /// + /// + /// This method returns a dictionary matching `AssemblyDefinition` instances to the path on disk to the assembly file they were loaded from. It is necessary + /// because uses a stream to load the data, in order to avoid later sharing violation issues when writing the assemblies. Path + /// information is required by to be available for each + /// + /// + Dictionary AddMethodsFromAbiSpecificAssemblies (MarshalMethodsClassifier classifier, DirectoryAssemblyResolver resolver, Dictionary> abiSpecificAssemblies) + { + IDictionary> marshalMethods = classifier.MarshalMethods; + ICollection assemblies = classifier.Assemblies; + var newAssemblies = new List (); + var assemblyPaths = new Dictionary (); + + foreach (AssemblyDefinition asmdef in assemblies) { + string fileName = Path.GetFileName (asmdef.MainModule.FileName); + if (!abiSpecificAssemblies.TryGetValue (fileName, out List? abiAssemblyItems)) { + continue; + } + + List assemblyMarshalMethods = FindMarshalMethodsForAssembly (marshalMethods, asmdef);; + Log.LogDebugMessage ($"Assembly {fileName} is ABI-specific"); + foreach (ITaskItem abiAssemblyItem in abiAssemblyItems) { + if (String.Compare (abiAssemblyItem.ItemSpec, asmdef.MainModule.FileName, StringComparison.Ordinal) == 0) { + continue; + } + + Log.LogDebugMessage ($"Looking for matching mashal methods in {abiAssemblyItem.ItemSpec}"); + FindMatchingMethodsInAssembly (abiAssemblyItem, classifier, assemblyMarshalMethods, resolver, newAssemblies, assemblyPaths); + } + } + + if (newAssemblies.Count > 0) { + foreach (AssemblyDefinition asmdef in newAssemblies) { + assemblies.Add (asmdef); + } + } + + return assemblyPaths; + } + + List FindMarshalMethodsForAssembly (IDictionary> marshalMethods, AssemblyDefinition asm) + { + var seenNativeCallbacks = new HashSet (); + var assemblyMarshalMethods = new List (); + + foreach (var kvp in marshalMethods) { + foreach (MarshalMethodEntry method in kvp.Value) { + if (method.NativeCallback.Module.Assembly != asm) { + continue; + } + + // More than one overriden method can use the same native callback method, we're interested only in unique native + // callbacks, since that's what gets rewritten. + if (seenNativeCallbacks.Contains (method.NativeCallback)) { + continue; + } + + seenNativeCallbacks.Add (method.NativeCallback); + assemblyMarshalMethods.Add (method); + } + } + + return assemblyMarshalMethods; + } + + void FindMatchingMethodsInAssembly (ITaskItem assemblyItem, MarshalMethodsClassifier classifier, List assemblyMarshalMethods, DirectoryAssemblyResolver resolver, List newAssemblies, Dictionary assemblyPaths) + { + AssemblyDefinition asm = LoadAssembly (assemblyItem.ItemSpec, resolver); + newAssemblies.Add (asm); + assemblyPaths.Add (asm, assemblyItem.ItemSpec); + + foreach (MarshalMethodEntry methodEntry in assemblyMarshalMethods) { + TypeDefinition wantedType = methodEntry.NativeCallback.DeclaringType; + TypeDefinition? type = asm.MainModule.FindType (wantedType.FullName); + if (type == null) { + throw new InvalidOperationException ($"Internal error: type '{wantedType.FullName}' not found in assembly '{assemblyItem.ItemSpec}', a linker error?"); + } + + if (type.MetadataToken != wantedType.MetadataToken) { + throw new InvalidOperationException ($"Internal error: type '{type.FullName}' in assembly '{assemblyItem.ItemSpec}' has a different token ID than the original type"); + } + + FindMatchingMethodInType (methodEntry, type, classifier); + } + } + + void FindMatchingMethodInType (MarshalMethodEntry methodEntry, TypeDefinition type, MarshalMethodsClassifier classifier) + { + string callbackName = methodEntry.NativeCallback.FullName; + + foreach (MethodDefinition typeNativeCallbackMethod in type.Methods) { + if (String.Compare (typeNativeCallbackMethod.FullName, callbackName, StringComparison.Ordinal) != 0) { + continue; + } + + if (typeNativeCallbackMethod.Parameters.Count != methodEntry.NativeCallback.Parameters.Count) { + continue; + } + + if (typeNativeCallbackMethod.MetadataToken != methodEntry.NativeCallback.MetadataToken) { + throw new InvalidOperationException ($"Internal error: tokens don't match for '{typeNativeCallbackMethod.FullName}'"); + } + + bool allMatch = true; + for (int i = 0; i < typeNativeCallbackMethod.Parameters.Count; i++) { + if (String.Compare (typeNativeCallbackMethod.Parameters[i].ParameterType.FullName, methodEntry.NativeCallback.Parameters[i].ParameterType.FullName, StringComparison.Ordinal) != 0) { + allMatch = false; + break; + } + } + + if (!allMatch) { + continue; + } + + Log.LogDebugMessage ($"Found match for '{typeNativeCallbackMethod.FullName}' in {type.Module.FileName}"); + string methodKey = classifier.GetStoreMethodKey (methodEntry); + classifier.MarshalMethods[methodKey].Add (new MarshalMethodEntry (methodEntry, typeNativeCallbackMethod)); + } + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs index 823c2deec46..5a9cabe50a5 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs @@ -1,9 +1,12 @@ using System; using System.IO; using System.Linq; +using System.Text; using Java.Interop.Tools.Cecil; using Mono.Cecil; +using Mono.Cecil.Cil; using Mono.Linker; +using Mono.Tuner; using MonoDroid.Tuner; using NUnit.Framework; using Xamarin.ProjectTools; @@ -514,7 +517,7 @@ public void AndroidUseNegotiateAuthentication ([Values (true, false, null)] bool } [Test] - public void DoNotErrorOnPerArchJavaTypeDuplicates () + public void DoNotErrorOnPerArchJavaTypeDuplicates ([Values(true, false)] bool enableMarshalMethods) { if (!Builder.UseDotNet) Assert.Ignore ("Test only valid on .NET"); @@ -525,13 +528,24 @@ public void DoNotErrorOnPerArchJavaTypeDuplicates () lib.Sources.Add (new BuildItem.Source ("Library1.cs") { TextContent = () => @" namespace Lib1; -public class Library1 : Java.Lang.Object { +public class Library1 : Com.Example.Androidlib.MyRunner { private static bool Is64Bits = IntPtr.Size >= 8; public static bool Is64 () { return Is64Bits; } + + public override void Run () => Console.WriteLine (Is64Bits); }", + }); + lib.Sources.Add (new BuildItem ("AndroidJavaSource", "MyRunner.java") { + Encoding = new UTF8Encoding (encoderShouldEmitUTF8Identifier: false), + TextContent = () => @" +package com.example.androidlib; + +public abstract class MyRunner { + public abstract void run(); +}" }); var proj = new XamarinAndroidApplicationProject { IsRelease = true, ProjectName = "App1" }; proj.References.Add(new BuildItem.ProjectReference (Path.Combine ("..", "Lib1", "Lib1.csproj"), "Lib1")); @@ -539,12 +553,48 @@ public static bool Is64 () { "base.OnCreate (bundle);", "base.OnCreate (bundle);\n" + "if (Lib1.Library1.Is64 ()) Console.WriteLine (\"Hello World!\");"); + proj.SetProperty ("AndroidEnableMarshalMethods", enableMarshalMethods.ToString ()); using var lb = CreateDllBuilder (Path.Combine (path, "Lib1")); using var b = CreateApkBuilder (Path.Combine (path, "App1")); Assert.IsTrue (lb.Build (lib), "build should have succeeded."); Assert.IsTrue (b.Build (proj), "build should have succeeded."); + + var intermediate = Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath); + var dll = $"{lib.ProjectName}.dll"; + Assert64Bit ("android-arm", expected64: false); + Assert64Bit ("android-arm64", expected64: true); + Assert64Bit ("android-x86", expected64: false); + Assert64Bit ("android-x64", expected64: true); + + void Assert64Bit(string rid, bool expected64) + { + var assembly = AssemblyDefinition.ReadAssembly (Path.Combine (intermediate, rid, "linked", "shrunk", dll)); + var type = assembly.MainModule.FindType ("Lib1.Library1"); + Assert.NotNull (type, "Should find Lib1.Library1!"); + var cctor = type.GetTypeConstructor (); + Assert.NotNull (type, "Should find Lib1.Library1.cctor!"); + Assert.AreNotEqual (0, cctor.Body.Instructions.Count); + + /* + * IL snippet + * .method private hidebysig specialname rtspecialname static + * void .cctor () cil managed + * { + * // Is64Bits = 4 >= 8; + * IL_0000: ldc.i4 4 + * IL_0005: ldc.i4.8 + * ... + */ + var instruction = cctor.Body.Instructions [0]; + Assert.AreEqual (OpCodes.Ldc_I4, instruction.OpCode); + if (expected64) { + Assert.AreEqual (8, instruction.Operand, $"Expected 64-bit: {expected64}"); + } else { + Assert.AreEqual (4, instruction.Operand, $"Expected 64-bit: {expected64}"); + } + } } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs index 4131349d150..1182206250d 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs @@ -23,32 +23,24 @@ sealed class AssemblyImports IDictionary> methods; ICollection uniqueAssemblies; - IDictionary > assemblyPaths; + IDictionary assemblyPaths; TaskLoggingHelper log; - public MarshalMethodsAssemblyRewriter (IDictionary> methods, ICollection uniqueAssemblies, IDictionary > assemblyPaths, TaskLoggingHelper log) + public MarshalMethodsAssemblyRewriter (IDictionary> methods, ICollection uniqueAssemblies, IDictionary assemblyPaths, TaskLoggingHelper log) { + this.assemblyPaths = assemblyPaths; this.methods = methods ?? throw new ArgumentNullException (nameof (methods)); this.uniqueAssemblies = uniqueAssemblies ?? throw new ArgumentNullException (nameof (uniqueAssemblies)); - this.assemblyPaths = assemblyPaths ?? throw new ArgumentNullException (nameof (assemblyPaths)); this.log = log ?? throw new ArgumentNullException (nameof (log)); } // TODO: do away with broken exception transitions, there's no point in supporting them - public void Rewrite (DirectoryAssemblyResolver resolver, List targetAssemblyPaths, bool brokenExceptionTransitions) + public void Rewrite (DirectoryAssemblyResolver resolver, bool brokenExceptionTransitions) { if (resolver == null) { throw new ArgumentNullException (nameof (resolver)); } - if (targetAssemblyPaths == null) { - throw new ArgumentNullException (nameof (targetAssemblyPaths)); - } - - if (targetAssemblyPaths.Count == 0) { - throw new ArgumentException ("must contain at least one target path", nameof (targetAssemblyPaths)); - } - AssemblyDefinition? monoAndroidRuntime = resolver.Resolve ("Mono.Android.Runtime"); if (monoAndroidRuntime == null) { throw new InvalidOperationException ($"Internal error: unable to load the Mono.Android.Runtime assembly"); @@ -114,47 +106,34 @@ public void Rewrite (DirectoryAssemblyResolver resolver, List targetAsse } } - var newAssemblyPaths = new List (); foreach (AssemblyDefinition asm in uniqueAssemblies) { - foreach (string path in GetAssemblyPaths (asm)) { - var writerParams = new WriterParameters { - WriteSymbols = File.Exists (Path.ChangeExtension (path, ".pdb")), - }; - - string directory = Path.Combine (Path.GetDirectoryName (path), "new"); - Directory.CreateDirectory (directory); - string output = Path.Combine (directory, Path.GetFileName (path)); - log.LogDebugMessage ($"Writing new version of assembly: {output}"); - - // TODO: this should be used eventually, but it requires that all the types are reloaded from the assemblies before typemaps are generated - // since Cecil doesn't update the MVID in the already loaded types - //asm.MainModule.Mvid = Guid.NewGuid (); - asm.Write (output, writerParams); - newAssemblyPaths.Add (output); - } - } + string path = GetAssemblyPath (asm); + string pathPdb = Path.ChangeExtension (path, ".pdb"); + bool havePdb = File.Exists (pathPdb); - // Replace old versions of the assemblies only after we've finished rewriting without issues, otherwise leave the new - // versions around. - foreach (string path in newAssemblyPaths) { - string? pdb = null; - - string source = Path.ChangeExtension (path, ".pdb"); - if (File.Exists (source)) { - pdb = source; - } - - foreach (string targetPath in targetAssemblyPaths) { - string target = Path.Combine (targetPath, Path.GetFileName (path)); - CopyFile (path, target); + var writerParams = new WriterParameters { + WriteSymbols = havePdb, + }; - if (!String.IsNullOrEmpty (pdb)) { - CopyFile (pdb, Path.ChangeExtension (target, ".pdb")); + string directory = Path.Combine (Path.GetDirectoryName (path), "new"); + Directory.CreateDirectory (directory); + string output = Path.Combine (directory, Path.GetFileName (path)); + log.LogDebugMessage ($"Writing new version of assembly: {output}"); + + // TODO: this should be used eventually, but it requires that all the types are reloaded from the assemblies before typemaps are generated + // since Cecil doesn't update the MVID in the already loaded types + //asm.MainModule.Mvid = Guid.NewGuid (); + asm.Write (output, writerParams); + CopyFile (output, path); + RemoveFile (output); + + if (havePdb) { + string outputPdb = Path.ChangeExtension (output, ".pdb"); + if (File.Exists (outputPdb)) { + CopyFile (outputPdb, pathPdb); } + RemoveFile (pathPdb); } - - RemoveFile (path); - RemoveFile (pdb); } void CopyFile (string source, string target) @@ -452,13 +431,15 @@ TypeReference ReturnValid (Type typeToLookUp) } } - ICollection GetAssemblyPaths (AssemblyDefinition asm) + string GetAssemblyPath (AssemblyDefinition asm) { - if (!assemblyPaths.TryGetValue (asm.Name.Name, out HashSet paths)) { - throw new InvalidOperationException ($"Unable to determine file path for assembly '{asm.Name.Name}'"); + string filePath = asm.MainModule.FileName; + if (!String.IsNullOrEmpty (filePath)) { + return filePath; } - return paths; + // No checking on purpose - the assembly **must** be there if its MainModule.FileName property returns a null or empty string + return assemblyPaths[asm]; } MethodDefinition GetUnmanagedCallersOnlyAttributeConstructor (DirectoryAssemblyResolver resolver) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsClassifier.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsClassifier.cs index 3a7d4712b3f..3b241f42b0f 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsClassifier.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsClassifier.cs @@ -40,8 +40,8 @@ sealed class MarshalMethodEntry public bool IsSpecial { get; } public MarshalMethodEntry (TypeDefinition declaringType, MethodDefinition nativeCallback, MethodDefinition connector, MethodDefinition - registeredMethod, MethodDefinition implementedMethod, FieldDefinition callbackField, string jniTypeName, - string jniName, string jniSignature, bool needsBlittableWorkaround) + registeredMethod, MethodDefinition implementedMethod, FieldDefinition callbackField, string jniTypeName, + string jniName, string jniSignature, bool needsBlittableWorkaround) { DeclaringType = declaringType ?? throw new ArgumentNullException (nameof (declaringType)); nativeCallbackReal = nativeCallback ?? throw new ArgumentNullException (nameof (nativeCallback)); @@ -66,6 +66,12 @@ public MarshalMethodEntry (TypeDefinition declaringType, MethodDefinition native IsSpecial = true; } + public MarshalMethodEntry (MarshalMethodEntry other, MethodDefinition nativeCallback) + : this (other.DeclaringType, nativeCallback, other.Connector, other.RegisteredMethod, + other.ImplementedMethod, other.CallbackField, other.JniTypeName, other.JniMethodName, + other.JniMethodSignature, other.NeedsBlittableWorkaround) + {} + string EnsureNonEmpty (string s, string argName) { if (String.IsNullOrEmpty (s)) { @@ -499,7 +505,6 @@ bool IsStandardHandler (TypeDefinition topType, ConnectorInfo connector, MethodD // method.CallbackField?.DeclaringType.Fields == 'null' StoreMethod ( - registeredMethod, new MarshalMethodEntry ( topType, nativeCallbackMethod, @@ -683,10 +688,16 @@ FieldDefinition FindField (TypeDefinition type, string fieldName, bool lookForIn return FindField (tdCache.Resolve (type.BaseType), fieldName, lookForInherited); } - void StoreMethod (MethodDefinition registeredMethod, MarshalMethodEntry entry) + public string GetStoreMethodKey (MarshalMethodEntry methodEntry) { + MethodDefinition registeredMethod = methodEntry.RegisteredMethod; string typeName = registeredMethod.DeclaringType.FullName.Replace ('/', '+'); - string key = $"{typeName}, {registeredMethod.DeclaringType.GetPartialAssemblyName (tdCache)}\t{registeredMethod.Name}"; + return $"{typeName}, {registeredMethod.DeclaringType.GetPartialAssemblyName (tdCache)}\t{registeredMethod.Name}"; + } + + void StoreMethod (MarshalMethodEntry entry) + { + string key = GetStoreMethodKey (entry); // Several classes can override the same method, we need to generate the marshal method only once, at the same time // keeping track of overloads @@ -706,7 +717,6 @@ void StoreAssembly (AssemblyDefinition asm) if (assemblies.Contains (asm)) { return; } - assemblies.Add (asm); } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs new file mode 100644 index 00000000000..7f23b838160 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs @@ -0,0 +1,173 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Diagnostics; +using System.IO; +using System.IO.MemoryMappedFiles; +using System.Linq; +using System.Text; + +using Java.Interop.Tools.Cecil; +using Java.Interop.Tools.Diagnostics; +using Java.Interop.Tools.TypeNameMappings; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Mono.Cecil; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +class JavaType +{ + public readonly TypeDefinition Type; + public IDictionary? PerAbiTypes { get; } + + public JavaType (TypeDefinition type, IDictionary? perAbiTypes) + { + Type = type; + if (perAbiTypes != null) { + PerAbiTypes = new ReadOnlyDictionary (perAbiTypes); + } + } +} + +class XAJavaTypeScanner +{ + sealed class TypeData + { + public readonly TypeDefinition FirstType; + public readonly Dictionary PerAbi; + + public bool IsAbiSpecific => !PerAbi.ContainsKey (AndroidTargetArch.None); + + public TypeData (TypeDefinition firstType) + { + FirstType = firstType; + PerAbi = new Dictionary (); + } + } + + public bool ErrorOnCustomJavaObject { get; set; } + + TaskLoggingHelper log; + TypeDefinitionCache cache; + + public XAJavaTypeScanner (TaskLoggingHelper log, TypeDefinitionCache cache) + { + this.log = log; + this.cache = cache; + } + + public List GetJavaTypes (ICollection inputAssemblies, DirectoryAssemblyResolver resolver) + { + var types = new Dictionary (StringComparer.Ordinal); + foreach (ITaskItem asmItem in inputAssemblies) { + AndroidTargetArch arch = GetTargetArch (asmItem); + AssemblyDefinition asmdef = LoadAssembly (asmItem.ItemSpec, resolver); + + foreach (ModuleDefinition md in asmdef.Modules) { + foreach (TypeDefinition td in md.Types) { + AddJavaType (td, types, arch); + } + } + } + + var ret = new List (); + foreach (var kvp in types) { + ret.Add (new JavaType (kvp.Value.FirstType, kvp.Value.IsAbiSpecific ? kvp.Value.PerAbi : null)); + } + + return ret; + } + + void AddJavaType (TypeDefinition type, Dictionary types, AndroidTargetArch arch) + { + if (type.IsSubclassOf ("Java.Lang.Object", cache) || type.IsSubclassOf ("Java.Lang.Throwable", cache) || (type.IsInterface && type.ImplementsInterface ("Java.Interop.IJavaPeerable", cache))) { + // For subclasses of e.g. Android.App.Activity. + string typeName = type.GetPartialAssemblyQualifiedName (cache); + if (!types.TryGetValue (typeName, out TypeData typeData)) { + typeData = new TypeData (type); + types.Add (typeName, typeData); + } + + if (typeData.PerAbi.ContainsKey (AndroidTargetArch.None)) { + if (arch == AndroidTargetArch.None) { + throw new InvalidOperationException ($"Duplicate type '{type.FullName}' in assembly {type.Module.FileName}"); + } + + throw new InvalidOperationException ($"Previously added type '{type.FullName}' was in ABI-agnostic assembly, new one comes from ABI {arch} assembly"); + } + + if (typeData.PerAbi.ContainsKey (arch)) { + throw new InvalidOperationException ($"Duplicate type '{type.FullName}' in assembly {type.Module.FileName}, for ABI {arch}"); + } + + typeData.PerAbi.Add (arch, type); + } else if (type.IsClass && !type.IsSubclassOf ("System.Exception", cache) && type.ImplementsInterface ("Android.Runtime.IJavaObject", cache)) { + string message = $"XA4212: Type `{type.FullName}` implements `Android.Runtime.IJavaObject` but does not inherit `Java.Lang.Object` or `Java.Lang.Throwable`. This is not supported."; + + if (ErrorOnCustomJavaObject) { + log.LogError (message); + } else { + log.LogWarning (message); + } + return; + } + + if (!type.HasNestedTypes) { + return; + } + + foreach (TypeDefinition nested in type.NestedTypes) { + AddJavaType (nested, types, arch); + } + } + + AndroidTargetArch GetTargetArch (ITaskItem asmItem) + { + string? abi = asmItem.GetMetadata ("Abi"); + if (String.IsNullOrEmpty (abi)) { + return AndroidTargetArch.None; + } + + return abi switch { + "armeabi-v7a" => AndroidTargetArch.Arm, + "arm64-v8a" => AndroidTargetArch.Arm64, + "x86" => AndroidTargetArch.X86, + "x86_64" => AndroidTargetArch.X86_64, + _ => throw new NotSupportedException ($"Unsupported ABI '{abi}' for assembly {asmItem.ItemSpec}") + }; + } + + AssemblyDefinition LoadAssembly (string path, DirectoryAssemblyResolver resolver) + { + string pdbPath = Path.ChangeExtension (path, ".pdb"); + var readerParameters = new ReaderParameters { + AssemblyResolver = resolver, + InMemory = true, + ReadingMode = ReadingMode.Immediate, + ReadSymbols = File.Exists (pdbPath), + ReadWrite = false, + }; + + return AssemblyDefinition.ReadAssembly (path, readerParameters); + + // MemoryMappedViewStream? viewStream = null; + // try { + // // Create stream because CreateFromFile(string, ...) uses FileShare.None which is too strict + // using var fileStream = new FileStream (path, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, false); + // using var mappedFile = MemoryMappedFile.CreateFromFile ( + // fileStream, null, fileStream.Length, MemoryMappedFileAccess.Read, HandleInheritability.None, true); + // viewStream = mappedFile.CreateViewStream (0, 0, MemoryMappedFileAccess.Read); + + // AssemblyDefinition result = ModuleDefinition.ReadModule (viewStream, readerParameters).Assembly; + + // // We transferred the ownership of the viewStream to the collection. + // viewStream = null; + + // return result; + // } finally { + // viewStream?.Dispose (); + // } + } +} From e393968baf9a3dc880294decebc946834e81fb71 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 3 Jul 2023 21:40:08 +0200 Subject: [PATCH 09/71] Typemaps generated properly, testing and marshal methods tomorrow --- .../Tasks/GenerateJavaStubs.cs | 96 ++++--- .../Utilities/ManifestDocument.cs | 16 +- .../Utilities/TypeMapGenerator.cs | 251 +++++++++++------- ...peMappingReleaseNativeAssemblyGenerator.cs | 129 +++++---- .../Utilities/XAJavaTypeScanner.cs | 11 +- 5 files changed, 310 insertions(+), 193 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index c780750ba0c..ddcd10dfb90 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -221,51 +221,65 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) } // Step 1 - Find all the JLO types - var cache = new TypeDefinitionCache (); - var scanner = new JavaTypeScanner (this.CreateTaskLogger (), cache) { - ErrorOnCustomJavaObject = ErrorOnCustomJavaObject, - }; + // var cache = new TypeDefinitionCache (); + // var scanner = new JavaTypeScanner (this.CreateTaskLogger (), cache) { + // ErrorOnCustomJavaObject = ErrorOnCustomJavaObject, + // }; var newCache = new TypeDefinitionCache (); var newScanner = new XAJavaTypeScanner (Log, newCache) { ErrorOnCustomJavaObject = ErrorOnCustomJavaObject, }; List newAllJavaTypes = newScanner.GetJavaTypes (newAllTypemapAssemblies.Values, res); - List allJavaTypes = scanner.GetJavaTypes (allTypemapAssemblies, res); + //List allJavaTypes = scanner.GetJavaTypes (allTypemapAssemblies, res); - Console.WriteLine ("All java types:"); - foreach (TypeDefinition td in allJavaTypes) { - Console.WriteLine ($" {td.GetPartialAssemblyQualifiedName (cache)} ({td.Module.FileName})"); - } + // Console.WriteLine ("All java types:"); + // foreach (TypeDefinition td in allJavaTypes) { + // Console.WriteLine ($" {td.GetPartialAssemblyQualifiedName (cache)} ({td.Module.FileName})"); + // } Console.WriteLine (); Console.WriteLine ("NEW all java types:"); foreach (JavaType jt in newAllJavaTypes) { - Console.Write ($" {jt.Type.FullName}"); - if (jt.PerAbiTypes != null) { - Console.WriteLine (" (ABI-specific)"); + if (jt.IsABiSpecific) { + Console.WriteLine ($" {jt.Type.GetPartialAssemblyQualifiedName (newCache)} ((ABI-specific))"); + foreach (var kvp in jt.PerAbiTypes) { + TypeDefinition td = kvp.Value; + Console.WriteLine ($" [{kvp.Key}] {td.Module.FileName} (Type token: 0x{td.MetadataToken.ToUInt32 ():x}; MVID: {td.Module.Mvid}])"); + } } else { - Console.WriteLine (); + Console.WriteLine ($" {jt.Type.GetPartialAssemblyQualifiedName (newCache)} ({jt.Type.Module.FileName})"); } } - var javaTypes = new List (); - foreach (TypeDefinition td in allJavaTypes) { + // var javaTypes = new List (); + // foreach (TypeDefinition td in allJavaTypes) { + // // Whem marshal methods are in use we do not want to skip non-user assemblies (such as Mono.Android) - we need to generate JCWs for them during + // // application build, unlike in Debug configuration or when marshal methods are disabled, in which case we use JCWs generated during Xamarin.Android + // // build and stored in a jar file. + // if ((!useMarshalMethods && !userAssemblies.ContainsKey (td.Module.Assembly.Name.Name)) || JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (td, cache)) { + // continue; + // } + // javaTypes.Add (td); + // } + + var newJavaTypes = new List (); + foreach (JavaType jt in newAllJavaTypes) { // Whem marshal methods are in use we do not want to skip non-user assemblies (such as Mono.Android) - we need to generate JCWs for them during // application build, unlike in Debug configuration or when marshal methods are disabled, in which case we use JCWs generated during Xamarin.Android // build and stored in a jar file. - if ((!useMarshalMethods && !userAssemblies.ContainsKey (td.Module.Assembly.Name.Name)) || JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (td, cache)) { + if ((!useMarshalMethods && !userAssemblies.ContainsKey (jt.Type.Module.Assembly.Name.Name)) || JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (jt.Type, newCache)) { continue; } - javaTypes.Add (td); + newJavaTypes.Add (jt); } MarshalMethodsClassifier classifier = null; if (useMarshalMethods) { - classifier = new MarshalMethodsClassifier (cache, res, Log); + classifier = new MarshalMethodsClassifier (newCache, res, Log); } // Step 2 - Generate Java stub code - var success = CreateJavaSources (javaTypes, cache, classifier, useMarshalMethods); + var success = CreateJavaSources (newJavaTypes, newCache, classifier, useMarshalMethods); if (!success) return; @@ -283,21 +297,22 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) // Step 3 - Generate type maps // Type mappings need to use all the assemblies, always. - WriteTypeMappings (allJavaTypes, cache); + WriteTypeMappings (newAllJavaTypes, newCache); // We need to save a map of .NET type -> ACW type for resource file fixups - var managed = new Dictionary (javaTypes.Count, StringComparer.Ordinal); - var java = new Dictionary (javaTypes.Count, StringComparer.Ordinal); + var managed = new Dictionary (newJavaTypes.Count, StringComparer.Ordinal); + var java = new Dictionary (newJavaTypes.Count, StringComparer.Ordinal); var managedConflicts = new Dictionary> (0, StringComparer.Ordinal); var javaConflicts = new Dictionary> (0, StringComparer.Ordinal); using (var acw_map = MemoryStreamPool.Shared.CreateStreamWriter ()) { - foreach (TypeDefinition type in javaTypes) { + foreach (JavaType jt in newJavaTypes) { + TypeDefinition type = jt.Type; string managedKey = type.FullName.Replace ('/', '.'); - string javaKey = JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.'); + string javaKey = JavaNativeTypeManager.ToJniName (type, newCache).Replace ('/', '.'); - acw_map.Write (type.GetPartialAssemblyQualifiedName (cache)); + acw_map.Write (type.GetPartialAssemblyQualifiedName (newCache)); acw_map.Write (';'); acw_map.Write (javaKey); acw_map.WriteLine (); @@ -307,16 +322,16 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) if (managed.TryGetValue (managedKey, out conflict)) { if (!conflict.Module.Name.Equals (type.Module.Name)) { if (!managedConflicts.TryGetValue (managedKey, out var list)) - managedConflicts.Add (managedKey, list = new List { conflict.GetPartialAssemblyName (cache) }); - list.Add (type.GetPartialAssemblyName (cache)); + managedConflicts.Add (managedKey, list = new List { conflict.GetPartialAssemblyName (newCache) }); + list.Add (type.GetPartialAssemblyName (newCache)); } hasConflict = true; } if (java.TryGetValue (javaKey, out conflict)) { if (!conflict.Module.Name.Equals (type.Module.Name)) { if (!javaConflicts.TryGetValue (javaKey, out var list)) - javaConflicts.Add (javaKey, list = new List { conflict.GetAssemblyQualifiedName (cache) }); - list.Add (type.GetAssemblyQualifiedName (cache)); + javaConflicts.Add (javaKey, list = new List { conflict.GetAssemblyQualifiedName (newCache) }); + list.Add (type.GetAssemblyQualifiedName (newCache)); success = false; } hasConflict = true; @@ -330,7 +345,7 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) acw_map.Write (javaKey); acw_map.WriteLine (); - acw_map.Write (JavaNativeTypeManager.ToCompatJniName (type, cache).Replace ('/', '.')); + acw_map.Write (JavaNativeTypeManager.ToCompatJniName (type, newCache).Replace ('/', '.')); acw_map.Write (';'); acw_map.Write (javaKey); acw_map.WriteLine (); @@ -382,7 +397,7 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) manifest.ForceExtractNativeLibs = true; } - var additionalProviders = manifest.Merge (Log, cache, allJavaTypes, ApplicationJavaClass, EmbedAssemblies, BundledWearApplicationName, MergedManifestDocuments); + var additionalProviders = manifest.Merge (Log, newCache, newAllJavaTypes, ApplicationJavaClass, EmbedAssemblies, BundledWearApplicationName, MergedManifestDocuments); // Only write the new manifest if it actually changed if (manifest.SaveIfChanged (Log, MergedAndroidManifestOutput)) { @@ -402,15 +417,16 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) // Create additional application java sources. StringWriter regCallsWriter = new StringWriter (); regCallsWriter.WriteLine ("\t\t// Application and Instrumentation ACWs must be registered first."); - foreach (var type in javaTypes) { - if (JavaNativeTypeManager.IsApplication (type, cache) || JavaNativeTypeManager.IsInstrumentation (type, cache)) { + foreach (JavaType jt in newJavaTypes) { + TypeDefinition type = jt.Type; + if (JavaNativeTypeManager.IsApplication (type, newCache) || JavaNativeTypeManager.IsInstrumentation (type, newCache)) { if (classifier != null && !classifier.FoundDynamicallyRegisteredMethods (type)) { continue; } - string javaKey = JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.'); + string javaKey = JavaNativeTypeManager.ToJniName (type, newCache).Replace ('/', '.'); regCallsWriter.WriteLine ("\t\tmono.android.Runtime.register (\"{0}\", {1}.class, {1}.__md_methods);", - type.GetAssemblyQualifiedName (cache), javaKey); + type.GetAssemblyQualifiedName (newCache), javaKey); } } regCallsWriter.Close (); @@ -466,7 +482,7 @@ AssemblyDefinition LoadAssembly (string path, DirectoryAssemblyResolver? resolve } } - bool CreateJavaSources (IEnumerable javaTypes, TypeDefinitionCache cache, MarshalMethodsClassifier classifier, bool useMarshalMethods) + bool CreateJavaSources (IEnumerable newJavaTypes, TypeDefinitionCache cache, MarshalMethodsClassifier classifier, bool useMarshalMethods) { if (useMarshalMethods && classifier == null) { throw new ArgumentNullException (nameof (classifier)); @@ -478,7 +494,8 @@ bool CreateJavaSources (IEnumerable javaTypes, TypeDefinitionCac bool generateOnCreateOverrides = int.Parse (AndroidSdkPlatform) <= 10; bool ok = true; - foreach (var t in javaTypes) { + foreach (JavaType jt in newJavaTypes) { + TypeDefinition t = jt.Type; // JCW generator doesn't care about ABI-specific types or token ids if (t.IsInterface) { // Interfaces are in typemap but they shouldn't have JCW generated for them continue; @@ -581,11 +598,12 @@ void SaveResource (string resource, string filename, string destDir, Func types, TypeDefinitionCache cache) + void WriteTypeMappings (List types, TypeDefinitionCache cache) { var tmg = new TypeMapGenerator ((string message) => Log.LogDebugMessage (message), SupportedAbis); - if (!tmg.Generate (Debug, SkipJniAddNativeMethodRegistrationAttributeScan, types, cache, TypemapOutputDirectory, GenerateNativeAssembly, out ApplicationConfigTaskState appConfState)) + if (!tmg.Generate (Debug, SkipJniAddNativeMethodRegistrationAttributeScan, types, cache, TypemapOutputDirectory, GenerateNativeAssembly, out ApplicationConfigTaskState appConfState)) { throw new XamarinAndroidException (4308, Properties.Resources.XA4308); + } GeneratedBinaryTypeMaps = tmg.GeneratedBinaryTypeMaps.ToArray (); BuildEngine4.RegisterTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (ApplicationConfigTaskState.RegisterTaskObjectKey), appConfState, RegisteredTaskObjectLifetime.Build); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs index 05de2bd0c86..8fe24c3b4ff 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs @@ -255,7 +255,7 @@ void ReorderActivityAliases (TaskLoggingHelper log, XElement app) } } - public IList Merge (TaskLoggingHelper log, TypeDefinitionCache cache, List subclasses, string applicationClass, bool embed, string bundledWearApplicationName, IEnumerable mergedManifestDocuments) + public IList Merge (TaskLoggingHelper log, TypeDefinitionCache cache, List subclasses, string applicationClass, bool embed, string bundledWearApplicationName, IEnumerable mergedManifestDocuments) { var manifest = doc.Root; @@ -330,7 +330,8 @@ public IList Merge (TaskLoggingHelper log, TypeDefinitionCache cache, Li throw new InvalidOperationException (string.Format ("The targetSdkVersion ({0}) is not a valid API level", targetSdkVersion)); int targetSdkVersionValue = tryTargetSdkVersion.Value; - foreach (var t in subclasses) { + foreach (JavaType jt in subclasses) { + TypeDefinition t = jt.Type; if (t.IsAbstract) continue; @@ -567,7 +568,7 @@ Func GetGenerator (T return null; } - XElement CreateApplicationElement (XElement manifest, string applicationClass, List subclasses, TypeDefinitionCache cache) + XElement CreateApplicationElement (XElement manifest, string applicationClass, List subclasses, TypeDefinitionCache cache) { var application = manifest.Descendants ("application").FirstOrDefault (); @@ -591,7 +592,8 @@ XElement CreateApplicationElement (XElement manifest, string applicationClass, L List typeAttr = new List (); List typeUsesLibraryAttr = new List (); List typeUsesConfigurationAttr = new List (); - foreach (var t in subclasses) { + foreach (JavaType jt in subclasses) { + TypeDefinition t = jt.Type; ApplicationAttribute aa = ApplicationAttribute.FromCustomAttributeProvider (t); if (aa == null) continue; @@ -923,7 +925,7 @@ void AddSupportsGLTextures (XElement application, TypeDefinitionCache cache) } } - void AddInstrumentations (XElement manifest, IList subclasses, int targetSdkVersion, TypeDefinitionCache cache) + void AddInstrumentations (XElement manifest, IList subclasses, int targetSdkVersion, TypeDefinitionCache cache) { var assemblyAttrs = Assemblies.SelectMany (path => InstrumentationAttribute.FromCustomAttributeProvider (Resolver.GetAssembly (path))); @@ -936,12 +938,14 @@ void AddInstrumentations (XElement manifest, IList subclasses, i manifest.Add (ia.ToElement (PackageName, cache)); } - foreach (var type in subclasses) + foreach (JavaType jt in subclasses) { + TypeDefinition type = jt.Type; if (type.IsSubclassOf ("Android.App.Instrumentation", cache)) { var xe = InstrumentationFromTypeDefinition (type, JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.'), cache); if (xe != null) manifest.Add (xe); } + } } public bool SaveIfChanged (TaskLoggingHelper log, string filename) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs index f59656518fc..1b020083282 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs @@ -1,6 +1,7 @@ using System; using System.IO; using System.Collections.Generic; +using System.Collections.ObjectModel; using System.Linq; using System.Text; @@ -89,6 +90,49 @@ internal sealed class ModuleDebugData public byte[] ModuleNameBytes; } + sealed class ReleaseGenerationState + { + int assemblyId = 0; + + public readonly Dictionary KnownAssemblies; + public readonly Dictionary MvidCache; + public readonly IDictionary> TempModules; + + // Just a convenient way to access one of the temp modules dictionaries, to be used when dealing with ABI-agnostic + // types in ProcessReleaseType. + public readonly Dictionary TempModulesAbiAgnostic; + + public ReleaseGenerationState (string[] supportedAbis) + { + KnownAssemblies = new Dictionary (StringComparer.Ordinal); + MvidCache = new Dictionary (); + + var tempModules = new Dictionary> (); + foreach (string abi in supportedAbis) { + var dict = new Dictionary (); + if (TempModulesAbiAgnostic == null) { + TempModulesAbiAgnostic = dict; + } + tempModules.Add (AbiToArch (abi), dict); + } + + TempModules = new ReadOnlyDictionary> (tempModules); + } + + public void AddKnownAssembly (TypeDefinition td) + { + string assemblyName = GetAssemblyName (td); + + if (KnownAssemblies.ContainsKey (assemblyName)) { + return; + } + + KnownAssemblies.Add (assemblyName, ++assemblyId); + } + + public string GetAssemblyName (TypeDefinition td) => td.Module.Assembly.FullName; + } + Action logger; Encoding outputEncoding; byte[] moduleMagicString; @@ -124,7 +168,7 @@ void UpdateApplicationConfig (TypeDefinition javaType, ApplicationConfigTaskStat } } - public bool Generate (bool debugBuild, bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, bool generateNativeAssembly, out ApplicationConfigTaskState appConfState) + public bool Generate (bool debugBuild, bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, bool generateNativeAssembly, out ApplicationConfigTaskState appConfState) { if (String.IsNullOrEmpty (outputDirectory)) throw new ArgumentException ("must not be null or empty", nameof (outputDirectory)); @@ -145,21 +189,23 @@ public bool Generate (bool debugBuild, bool skipJniAddNativeMethodRegistrationAt return GenerateRelease (skipJniAddNativeMethodRegistrationAttributeScan, javaTypes, cache, typemapsOutputDirectory, appConfState); } - bool GenerateDebug (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, bool generateNativeAssembly, ApplicationConfigTaskState appConfState) + bool GenerateDebug (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, bool generateNativeAssembly, ApplicationConfigTaskState appConfState) { - if (generateNativeAssembly) + if (generateNativeAssembly) { return GenerateDebugNativeAssembly (skipJniAddNativeMethodRegistrationAttributeScan, javaTypes, cache, outputDirectory, appConfState); + } return GenerateDebugFiles (skipJniAddNativeMethodRegistrationAttributeScan, javaTypes, cache, outputDirectory, appConfState); } - bool GenerateDebugFiles (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, ApplicationConfigTaskState appConfState) + bool GenerateDebugFiles (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, ApplicationConfigTaskState appConfState) { var modules = new Dictionary (StringComparer.Ordinal); int maxModuleFileNameWidth = 0; int maxModuleNameWidth = 0; var javaDuplicates = new Dictionary> (StringComparer.Ordinal); - foreach (TypeDefinition td in javaTypes) { + foreach (JavaType jt in javaTypes) { + TypeDefinition td = jt.Type; UpdateApplicationConfig (td, appConfState); string moduleName = td.Module.Assembly.Name.Name; ModuleDebugData module; @@ -218,13 +264,14 @@ bool GenerateDebugFiles (bool skipJniAddNativeMethodRegistrationAttributeScan, L return true; } - bool GenerateDebugNativeAssembly (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, ApplicationConfigTaskState appConfState) + bool GenerateDebugNativeAssembly (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, ApplicationConfigTaskState appConfState) { var javaToManaged = new List (); var managedToJava = new List (); var javaDuplicates = new Dictionary> (StringComparer.Ordinal); - foreach (TypeDefinition td in javaTypes) { + foreach (JavaType jt in javaTypes) { + TypeDefinition td = jt.Type; UpdateApplicationConfig (td, appConfState); TypeMapDebugEntry entry = GetDebugEntry (td, cache); @@ -330,91 +377,118 @@ string GetManagedTypeName (TypeDefinition td) return $"{managedTypeName}, {td.Module.Assembly.Name.Name}"; } - bool GenerateRelease (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, ApplicationConfigTaskState appConfState) + void ProcessReleaseType (ReleaseGenerationState state, TypeDefinition td, AndroidTargetArch typeArch, ApplicationConfigTaskState appConfState, TypeDefinitionCache cache) { - int assemblyId = 0; - var knownAssemblies = new Dictionary (StringComparer.Ordinal); - var tempModules = new Dictionary (); - Dictionary moduleCounter = null; - var mvidCache = new Dictionary (); + UpdateApplicationConfig (td, appConfState); - foreach (TypeDefinition td in javaTypes) { - UpdateApplicationConfig (td, appConfState); + state.AddKnownAssembly (td); - string assemblyName = td.Module.Assembly.FullName; + // We must NOT use Guid here! The reason is that Guid sort order is different than its corresponding + // byte array representation and on the runtime we need the latter in order to be able to binary search + // through the module array. + byte[] moduleUUID; + if (!state.MvidCache.TryGetValue (td.Module.Mvid, out moduleUUID)) { + moduleUUID = td.Module.Mvid.ToByteArray (); + state.MvidCache.Add (td.Module.Mvid, moduleUUID); + } - if (!knownAssemblies.ContainsKey (assemblyName)) { - assemblyId++; - knownAssemblies.Add (assemblyName, assemblyId); + bool abiAgnosticType = typeArch == AndroidTargetArch.None; + Dictionary tempModules; + if (abiAgnosticType) { + tempModules = state.TempModulesAbiAgnostic; + } else { + // It will throw if `typeArch` isn't in the dictionary. This is intentional, since we must have no TypeDefinition entries for architectures not + // mentioned in `supportedAbis`. + try { + tempModules = state.TempModules[typeArch]; + } catch (KeyNotFoundException ex) { + throw new InvalidOperationException ($"Internal error: cannot process type specific to architecture '{typeArch}', since that architecture isn't mentioned in the set of supported ABIs", ex); } + } - // We must NOT use Guid here! The reason is that Guid sort order is different than its corresponding - // byte array representation and on the runtime we need the latter in order to be able to binary search - // through the module array. - byte[] moduleUUID; - if (!mvidCache.TryGetValue (td.Module.Mvid, out moduleUUID)) { - moduleUUID = td.Module.Mvid.ToByteArray (); - mvidCache.Add (td.Module.Mvid, moduleUUID); - } + if (!tempModules.TryGetValue (moduleUUID, out ModuleReleaseData moduleData)) { + moduleData = new ModuleReleaseData { + Mvid = td.Module.Mvid, + MvidBytes = moduleUUID, + Assembly = td.Module.Assembly, + AssemblyName = td.Module.Assembly.Name.Name, + TypesScratch = new Dictionary (StringComparer.Ordinal), + DuplicateTypes = new List (), + }; - ModuleReleaseData moduleData; - if (!tempModules.TryGetValue (moduleUUID, out moduleData)) { - if (moduleCounter == null) - moduleCounter = new Dictionary (); - - moduleData = new ModuleReleaseData { - Mvid = td.Module.Mvid, - MvidBytes = moduleUUID, - Assembly = td.Module.Assembly, - AssemblyName = td.Module.Assembly.Name.Name, - TypesScratch = new Dictionary (StringComparer.Ordinal), - DuplicateTypes = new List (), - }; + if (abiAgnosticType) { + // ABI-agnostic types must be added to all the ABIs + foreach (var kvp in state.TempModules) { + kvp.Value.Add (moduleUUID, moduleData); + } + } else { + // ABI-specific types are added only to their respective tempModules tempModules.Add (moduleUUID, moduleData); } + } - string javaName = Java.Interop.Tools.TypeNameMappings.JavaNativeTypeManager.ToJniName (td, cache); - // We will ignore generic types and interfaces when generating the Java to Managed map, but we must not - // omit them from the table we output - we need the same number of entries in both java-to-managed and - // managed-to-java tables. `SkipInJavaToManaged` set to `true` will cause the native assembly generator - // to output `0` as the token id for the type, thus effectively causing the runtime unable to match such - // a Java type name to a managed type. This fixes https://github.com/xamarin/xamarin-android/issues/4660 - var entry = new TypeMapReleaseEntry { - JavaName = javaName, - ManagedTypeName = td.FullName, - Token = td.MetadataToken.ToUInt32 (), - AssemblyNameIndex = knownAssemblies [assemblyName], - SkipInJavaToManaged = ShouldSkipInJavaToManaged (td), - }; + string javaName = Java.Interop.Tools.TypeNameMappings.JavaNativeTypeManager.ToJniName (td, cache); + // We will ignore generic types and interfaces when generating the Java to Managed map, but we must not + // omit them from the table we output - we need the same number of entries in both java-to-managed and + // managed-to-java tables. `SkipInJavaToManaged` set to `true` will cause the native assembly generator + // to output `0` as the token id for the type, thus effectively causing the runtime unable to match such + // a Java type name to a managed type. This fixes https://github.com/xamarin/xamarin-android/issues/4660 + var entry = new TypeMapReleaseEntry { + JavaName = javaName, + ManagedTypeName = td.FullName, + Token = td.MetadataToken.ToUInt32 (), + AssemblyNameIndex = state.KnownAssemblies [state.GetAssemblyName (td)], + SkipInJavaToManaged = ShouldSkipInJavaToManaged (td), + }; - if (moduleData.TypesScratch.ContainsKey (entry.JavaName)) { - // This is disabled because it costs a lot of time (around 150ms per standard XF Integration app - // build) and has no value for the end user. The message is left here because it may be useful to us - // in our devloop at some point. - //logger ($"Warning: duplicate Java type name '{entry.JavaName}' in assembly '{moduleData.AssemblyName}' (new token: {entry.Token})."); - moduleData.DuplicateTypes.Add (entry); - } else - moduleData.TypesScratch.Add (entry.JavaName, entry); + if (moduleData.TypesScratch.ContainsKey (entry.JavaName)) { + // This is disabled because it costs a lot of time (around 150ms per standard XF Integration app + // build) and has no value for the end user. The message is left here because it may be useful to us + // in our devloop at some point. + //logger ($"Warning: duplicate Java type name '{entry.JavaName}' in assembly '{moduleData.AssemblyName}' (new token: {entry.Token})."); + moduleData.DuplicateTypes.Add (entry); + } else { + moduleData.TypesScratch.Add (entry.JavaName, entry); } + } - var modules = tempModules.Values.ToArray (); - Array.Sort (modules, new ModuleUUIDArrayComparer ()); + bool GenerateRelease (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, ApplicationConfigTaskState appConfState) + { + var state = new ReleaseGenerationState (supportedAbis); - foreach (ModuleReleaseData module in modules) { - if (module.TypesScratch.Count == 0) { - module.Types = Array.Empty (); + foreach (JavaType jt in javaTypes) { + if (!jt.IsABiSpecific) { + ProcessReleaseType (state, jt.Type, AndroidTargetArch.None, appConfState, cache); continue; } - // No need to sort here, the LLVM IR generator will compute hashes and sort - // the array on write. - module.Types = module.TypesScratch.Values.ToArray (); + foreach (var kvp in jt.PerAbiTypes) { + ProcessReleaseType (state, kvp.Value, kvp.Key, appConfState, cache); + } } - NativeTypeMappingData data; - data = new NativeTypeMappingData (logger, modules); + var mappingData = new Dictionary (); + foreach (var kvp in state.TempModules) { + AndroidTargetArch arch = kvp.Key; + Dictionary tempModules = kvp.Value; + var modules = tempModules.Values.ToArray (); + Array.Sort (modules, new ModuleUUIDArrayComparer ()); + + foreach (ModuleReleaseData module in modules) { + if (module.TypesScratch.Count == 0) { + module.Types = Array.Empty (); + continue; + } + + // No need to sort here, the LLVM IR generator will compute hashes and sort + // the array on write. + module.Types = module.TypesScratch.Values.ToArray (); + } + + mappingData.Add (arch, new NativeTypeMappingData (logger, modules)); + } - var generator = new TypeMappingReleaseNativeAssemblyGenerator (data); + var generator = new TypeMappingReleaseNativeAssemblyGenerator (mappingData); generator.Init (); GenerateNativeAssembly (generator, outputDirectory); @@ -430,27 +504,7 @@ void GenerateNativeAssembly (TypeMappingAssemblyGenerator generator, string base { AndroidTargetArch arch; foreach (string abi in supportedAbis) { - switch (abi.Trim ()) { - case "armeabi-v7a": - arch = AndroidTargetArch.Arm; - break; - - case "arm64-v8a": - arch = AndroidTargetArch.Arm64; - break; - - case "x86": - arch = AndroidTargetArch.X86; - break; - - case "x86_64": - arch = AndroidTargetArch.X86_64; - break; - - default: - throw new InvalidOperationException ($"Unknown ABI {abi}"); - } - + arch = AbiToArch (abi); string outputFile = $"{baseFileName}.{abi}.ll"; using (var sw = MemoryStreamPool.Shared.CreateStreamWriter (outputEncoding)) { generator.Write (arch, sw, outputFile); @@ -460,6 +514,17 @@ void GenerateNativeAssembly (TypeMappingAssemblyGenerator generator, string base } } + static AndroidTargetArch AbiToArch (string abi) + { + return abi switch { + "armeabi-v7a" => AndroidTargetArch.Arm, + "arm64-v8a" => AndroidTargetArch.Arm64, + "x86_64" => AndroidTargetArch.X86_64, + "x86" => AndroidTargetArch.X86, + _ => throw new InvalidOperationException ($"Unknown ABI {abi}") + }; + } + // Binary index file format, all data is little-endian: // // [Magic string] # XATI diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs index bc407b478aa..f92af2e902c 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs @@ -5,6 +5,7 @@ using System.Text; using Xamarin.Android.Tasks.LLVMIR; +using Xamarin.Android.Tools; namespace Xamarin.Android.Tasks { @@ -144,56 +145,80 @@ public ModuleMapData (string symbolLabel, List> MapModules; + public readonly List> JavaMap; + public readonly Dictionary JavaTypesByName; + public readonly List JavaNames; + public readonly NativeTypeMappingData MappingData; + public ulong ModuleCounter = 0; + + public ArchGenerationState (NativeTypeMappingData mappingData) + { + MapModules = new List> (); + JavaMap = new List> (); + JavaTypesByName = new Dictionary (StringComparer.Ordinal); + JavaNames = new List (); + MappingData = mappingData; + } + } + StructureInfo typeMapJavaStructureInfo; StructureInfo typeMapModuleStructureInfo; StructureInfo typeMapModuleEntryStructureInfo; - List> mapModules; - List> javaMap; - Dictionary javaTypesByName; - List javaNames; - JavaNameHashComparer javaNameHashComparer; + Dictionary archState; - ulong moduleCounter = 0; + JavaNameHashComparer javaNameHashComparer; - public TypeMappingReleaseNativeAssemblyGenerator (NativeTypeMappingData mappingData) + public TypeMappingReleaseNativeAssemblyGenerator (Dictionary mappingData) { - this.mappingData = mappingData ?? throw new ArgumentNullException (nameof (mappingData)); - mapModules = new List> (); - javaMap = new List> (); - javaTypesByName = new Dictionary (StringComparer.Ordinal); + if (mappingData == null) { + throw new ArgumentNullException (nameof (mappingData)); + } + javaNameHashComparer = new JavaNameHashComparer (); - javaNames = new List (); + archState = new Dictionary (mappingData.Count); + + foreach (var kvp in mappingData) { + if (kvp.Value == null) { + throw new ArgumentException ("must not contain null values", nameof (mappingData)); + } + + archState.Add (kvp.Key, new ArchGenerationState (kvp.Value)); + } } public override void Init () { - InitMapModules (); - InitJavaMap (); + foreach (var kvp in archState) { + InitMapModules (kvp.Value); + InitJavaMap (kvp.Value); + } } - void InitJavaMap () + void InitJavaMap (ArchGenerationState state) { TypeMapJava map_entry; - foreach (TypeMapGenerator.TypeMapReleaseEntry entry in mappingData.JavaTypes) { - javaNames.Add (entry.JavaName); + foreach (TypeMapGenerator.TypeMapReleaseEntry entry in state.MappingData.JavaTypes) { + state.JavaNames.Add (entry.JavaName); map_entry = new TypeMapJava { module_index = (uint)entry.ModuleIndex, // UInt32.MaxValue, type_token_id = entry.SkipInJavaToManaged ? 0 : entry.Token, - java_name_index = (uint)(javaNames.Count - 1), + java_name_index = (uint)(state.JavaNames.Count - 1), JavaName = entry.JavaName, }; - javaMap.Add (new StructureInstance (map_entry)); - javaTypesByName.Add (map_entry.JavaName, map_entry); + state.JavaMap.Add (new StructureInstance (map_entry)); + state.JavaTypesByName.Add (map_entry.JavaName, map_entry); } } - void InitMapModules () + void InitMapModules (ArchGenerationState state) { - foreach (TypeMapGenerator.ModuleReleaseData data in mappingData.Modules) { - string mapName = $"module{moduleCounter++}_managed_to_java"; + foreach (TypeMapGenerator.ModuleReleaseData data in state.MappingData.Modules) { + string mapName = $"module{state.ModuleCounter++}_managed_to_java"; string duplicateMapName; if (data.DuplicateTypes.Count == 0) @@ -214,7 +239,7 @@ void InitMapModules () java_name_width = 0, }; - mapModules.Add (new StructureInstance (map_module)); + state.MapModules.Add (new StructureInstance (map_module)); } } @@ -228,7 +253,7 @@ protected override void MapStructures (LlvmIrGenerator generator) // Prepare module map entries by sorting them on the managed token, and then mapping each entry to its corresponding Java type map index. // Requires that `javaMap` is sorted on the type name hash. - void PrepareMapModuleData (string moduleDataSymbolLabel, IEnumerable moduleEntries, List allModulesData) + void PrepareMapModuleData (ArchGenerationState state, string moduleDataSymbolLabel, IEnumerable moduleEntries, List allModulesData) { var mapModuleEntries = new List> (); foreach (TypeMapGenerator.TypeMapReleaseEntry entry in moduleEntries) { @@ -244,12 +269,12 @@ void PrepareMapModuleData (string moduleDataSymbolLabel, IEnumerable (javaType); - int idx = javaMap.BinarySearch (key, javaNameHashComparer); + int idx = state.JavaMap.BinarySearch (key, javaNameHashComparer); if (idx < 0) { throw new InvalidOperationException ($"Could not map entry '{javaTypeName}' to array index"); } @@ -261,32 +286,32 @@ uint GetJavaEntryIndex (string javaTypeName) // Generate hashes for all Java type names, then sort javaMap on the name hash. This has to be done in the writing phase because hashes // will depend on architecture (or, actually, on its bitness) and may differ between architectures (they will be the same for all architectures // with the same bitness) - (List allMapModulesData, List javaMapHashes) PrepareMapsForWriting (LlvmIrGenerator generator) + (List allMapModulesData, List javaMapHashes) PrepareMapsForWriting (ArchGenerationState state, LlvmIrGenerator generator) { bool is64Bit = generator.Is64Bit; // Generate Java type name hashes... - for (int i = 0; i < javaMap.Count; i++) { - TypeMapJava entry = javaMap[i].Obj; + for (int i = 0; i < state.JavaMap.Count; i++) { + TypeMapJava entry = state.JavaMap[i].Obj; entry.JavaNameHash = HashName (entry.JavaName); } // ...sort them... - javaMap.Sort ((StructureInstance a, StructureInstance b) => a.Obj.JavaNameHash.CompareTo (b.Obj.JavaNameHash)); + state.JavaMap.Sort ((StructureInstance a, StructureInstance b) => a.Obj.JavaNameHash.CompareTo (b.Obj.JavaNameHash)); var allMapModulesData = new List (); // ...and match managed types to Java... - foreach (StructureInstance moduleInstance in mapModules) { + foreach (StructureInstance moduleInstance in state.MapModules) { TypeMapModule module = moduleInstance.Obj; - PrepareMapModuleData (module.MapSymbolName, module.Data.Types, allMapModulesData); + PrepareMapModuleData (state, module.MapSymbolName, module.Data.Types, allMapModulesData); if (module.Data.DuplicateTypes.Count > 0) { - PrepareMapModuleData (module.DuplicateMapSymbolName, module.Data.DuplicateTypes, allMapModulesData); + PrepareMapModuleData (state, module.DuplicateMapSymbolName, module.Data.DuplicateTypes, allMapModulesData); } } var javaMapHashes = new HashSet (); - foreach (StructureInstance entry in javaMap) { + foreach (StructureInstance entry in state.JavaMap) { javaMapHashes.Add (entry.Obj.JavaNameHash); } @@ -315,22 +340,30 @@ ulong HashBytes (byte[] bytes) protected override void Write (LlvmIrGenerator generator) { - generator.WriteVariable ("map_module_count", mappingData.MapModuleCount); - generator.WriteVariable ("java_type_count", javaMap.Count); // must include the padding item, if any + ArchGenerationState state; + + try { + state = archState[generator.TargetArch]; + } catch (KeyNotFoundException ex) { + throw new InvalidOperationException ($"Internal error: architecture {generator.TargetArch} has not been prepared for writing.", ex); + } + + generator.WriteVariable ("map_module_count", state.MappingData.MapModuleCount); + generator.WriteVariable ("java_type_count", state.JavaMap.Count); // must include the padding item, if any - (List allMapModulesData, List javaMapHashes) = PrepareMapsForWriting (generator); - WriteMapModules (generator, allMapModulesData); - WriteJavaMap (generator, javaMapHashes); + (List allMapModulesData, List javaMapHashes) = PrepareMapsForWriting (state, generator); + WriteMapModules (state, generator, allMapModulesData); + WriteJavaMap (state, generator, javaMapHashes); } - void WriteJavaMap (LlvmIrGenerator generator, List javaMapHashes) + void WriteJavaMap (ArchGenerationState state, LlvmIrGenerator generator, List javaMapHashes) { generator.WriteEOL (); generator.WriteEOL ("Java to managed map"); generator.WriteStructureArray ( typeMapJavaStructureInfo, - javaMap, + state.JavaMap, LlvmIrVariableOptions.GlobalConstant, "map_java" ); @@ -347,7 +380,7 @@ void WriteJavaMap (LlvmIrGenerator generator, List javaMapHashes) WriteHashes (hashes); } - generator.WriteArray (javaNames, "java_type_names"); + generator.WriteArray (state.JavaNames, "java_type_names"); void WriteHashes (List hashes) where T: struct { @@ -355,14 +388,14 @@ void WriteHashes (List hashes) where T: struct hashes, LlvmIrVariableOptions.GlobalConstant, "map_java_hashes", - (int idx, T value) => $"{idx}: 0x{value:x} => {javaMap[idx].Obj.JavaName}" + (int idx, T value) => $"{idx}: 0x{value:x} => {state.JavaMap[idx].Obj.JavaName}" ); } } - void WriteMapModules (LlvmIrGenerator generator, List mapModulesData) + void WriteMapModules (ArchGenerationState state, LlvmIrGenerator generator, List mapModulesData) { - if (mapModules.Count == 0) { + if (state.MapModules.Count == 0) { return; } @@ -381,7 +414,7 @@ void WriteMapModules (LlvmIrGenerator generator, List mapModulesD generator.WriteEOL ("Map modules"); generator.WriteStructureArray ( typeMapModuleStructureInfo, - mapModules, + state.MapModules, LlvmIrVariableOptions.GlobalWritable, "map_modules" ); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs index 7f23b838160..a9d476e9622 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs @@ -1,15 +1,10 @@ using System; using System.Collections.Generic; using System.Collections.ObjectModel; -using System.Diagnostics; using System.IO; -using System.IO.MemoryMappedFiles; -using System.Linq; -using System.Text; +//using System.IO.MemoryMappedFiles; using Java.Interop.Tools.Cecil; -using Java.Interop.Tools.Diagnostics; -using Java.Interop.Tools.TypeNameMappings; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using Mono.Cecil; @@ -20,13 +15,15 @@ namespace Xamarin.Android.Tasks; class JavaType { public readonly TypeDefinition Type; - public IDictionary? PerAbiTypes { get; } + public readonly IDictionary? PerAbiTypes; + public bool IsABiSpecific { get; } public JavaType (TypeDefinition type, IDictionary? perAbiTypes) { Type = type; if (perAbiTypes != null) { PerAbiTypes = new ReadOnlyDictionary (perAbiTypes); + IsABiSpecific = perAbiTypes.Count > 1 || (perAbiTypes.Count == 1 && !perAbiTypes.ContainsKey (AndroidTargetArch.None)); } } } From a37cd9a3ade3b753f5806dd808ad3799161876e2 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 4 Jul 2023 22:55:59 +0200 Subject: [PATCH 10/71] Something isn't right... Apps segfault at startup when AOT is enabled, and yet typemaps appear to be indentical... Investigation TBC tomorrow --- src/monodroid/jni/monodroid-glue.cc | 41 +++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/src/monodroid/jni/monodroid-glue.cc b/src/monodroid/jni/monodroid-glue.cc index e7409351411..cf51d05ca4b 100644 --- a/src/monodroid/jni/monodroid-glue.cc +++ b/src/monodroid/jni/monodroid-glue.cc @@ -830,20 +830,25 @@ MonodroidRuntime::cleanup_runtime_config (MonovmRuntimeConfigArguments *args, [[ MonoDomain* MonodroidRuntime::create_domain (JNIEnv *env, jstring_array_wrapper &runtimeApks, bool is_root_domain, bool have_split_apks) { + log_info (LOG_DEFAULT, "#grendel Start: %s", __PRETTY_FUNCTION__); size_t user_assemblies_count = 0; - + log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); gather_bundled_assemblies (runtimeApks, &user_assemblies_count, have_split_apks); - + log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); #if defined (NET) size_t blob_time_index; if (XA_UNLIKELY (FastTiming::enabled ())) { blob_time_index = internal_timing->start_event (TimingEventKind::RuntimeConfigBlob); } + log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); if (embeddedAssemblies.have_runtime_config_blob ()) { + log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); runtime_config_args.kind = 1; embeddedAssemblies.get_runtime_config_blob (runtime_config_args.runtimeconfig.data.data, runtime_config_args.runtimeconfig.data.data_len); + log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); monovm_runtimeconfig_initialize (&runtime_config_args, cleanup_runtime_config, nullptr); + log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); } if (XA_UNLIKELY (FastTiming::enabled ())) { @@ -869,7 +874,9 @@ MonodroidRuntime::create_domain (JNIEnv *env, jstring_array_wrapper &runtimeApks #if !defined (NET) if (is_root_domain) { #endif // ndef NET + log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); domain = mono_jit_init_version (const_cast ("RootDomain"), const_cast ("mobile")); + log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); #if !defined (NET) } else { MonoDomain* root_domain = mono_get_root_domain (); @@ -910,6 +917,7 @@ MonodroidRuntime::create_domain (JNIEnv *env, jstring_array_wrapper &runtimeApks } } + log_info (LOG_DEFAULT, "#grendel Leave: %s [%u]", __PRETTY_FUNCTION__, __LINE__); return domain; } @@ -985,6 +993,7 @@ MonodroidRuntime::init_android_runtime ( #endif // ndef NET JNIEnv *env, jclass runtimeClass, jobject loader) { + log_info (LOG_DEFAULT, "#grendel Start: %s", __PRETTY_FUNCTION__); constexpr char icall_typemap_java_to_managed[] = "Java.Interop.TypeManager::monodroid_typemap_java_to_managed"; constexpr char icall_typemap_managed_to_java[] = "Android.Runtime.JNIEnv::monodroid_typemap_managed_to_java"; @@ -994,16 +1003,22 @@ MonodroidRuntime::init_android_runtime ( using j2mFn = MonoReflectionType* (*)(MonoString *java_type); using m2jFn = const char* (*)(MonoReflectionType *type, const uint8_t *mvid); + log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); mono_add_internal_call (icall_typemap_java_to_managed, reinterpret_cast(static_cast(EmbeddedAssemblies::typemap_java_to_managed))); + log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); mono_add_internal_call (icall_typemap_managed_to_java, reinterpret_cast(static_cast(EmbeddedAssemblies::typemap_managed_to_java))); + log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); #else mono_add_internal_call (icall_typemap_java_to_managed, reinterpret_cast(typemap_java_to_managed)); mono_add_internal_call (icall_typemap_managed_to_java, reinterpret_cast(typemap_managed_to_java)); #endif // def RELEASE && def ANDROID #if defined (NET) + log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); mono_add_internal_call ("Android.Runtime.RuntimeNativeMethods::monodroid_debugger_unhandled_exception", reinterpret_cast (monodroid_debugger_unhandled_exception)); + log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); mono_add_internal_call ("Android.Runtime.RuntimeNativeMethods::monodroid_unhandled_exception", reinterpret_cast(monodroid_unhandled_exception)); + log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); #endif // def NET struct JnienvInitializeArgs init = {}; @@ -1121,7 +1136,9 @@ MonodroidRuntime::init_android_runtime ( } #if defined (NET) && defined (ANDROID) + log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); auto initialize = reinterpret_cast (mono_method_get_unmanaged_callers_only_ftnptr (method, &error)); + log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); if (initialize == nullptr) { log_fatal (LOG_DEFAULT, "Failed to get pointer to Initialize. Mono error: %s", mono_error_get_message (&error)); } @@ -1131,7 +1148,9 @@ MonodroidRuntime::init_android_runtime ( "Failed to obtain unmanaged-callers-only pointer to the Android.Runtime.JNIEnvInit.Initialize method. %s", mono_error_get_message (&error) ); + log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); initialize (&init); + log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); #else // def NET && def ANDROID void *args [] = { &init, @@ -1143,6 +1162,7 @@ MonodroidRuntime::init_android_runtime ( if (XA_UNLIKELY (FastTiming::enabled ())) { internal_timing->end_event (native_to_managed_index); } + log_info (LOG_DEFAULT, "#grendel Leave: %s [%u]", __PRETTY_FUNCTION__, __LINE__); } #if defined (NET) @@ -1917,7 +1937,9 @@ MonodroidRuntime::create_and_initialize_domain (JNIEnv* env, jclass runtimeClass [[maybe_unused]] jstring_array_wrapper &assembliesPaths, jobject loader, bool is_root_domain, bool force_preload_assemblies, bool have_split_apks) { + log_info (LOG_DEFAULT, "#grendel Start: %s", __PRETTY_FUNCTION__); MonoDomain* domain = create_domain (env, runtimeApks, is_root_domain, have_split_apks); + log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); #if defined (ANDROID) // Asserting this on desktop apparently breaks a Designer test abort_unless (domain != nullptr, "Failed to create AppDomain"); @@ -1931,9 +1953,12 @@ MonodroidRuntime::create_and_initialize_domain (JNIEnv* env, jclass runtimeClass } #if defined (NET) + log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); default_alc = mono_alc_get_default_gchandle (); + log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); abort_unless (default_alc != nullptr, "Default AssemblyLoadContext not found"); + log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); embeddedAssemblies.install_preload_hooks_for_alc (); log_debug (LOG_ASSEMBLY, "ALC hooks installed"); #endif // def NET @@ -1945,14 +1970,18 @@ MonodroidRuntime::create_and_initialize_domain (JNIEnv* env, jclass runtimeClass bool preload = (androidSystem.is_assembly_preload_enabled () || (is_running_on_desktop && force_preload_assemblies)); #if defined (NET) + log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); load_assemblies (default_alc, preload, assemblies); + log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); init_android_runtime (env, runtimeClass, loader); + log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); #else // def NET load_assemblies (domain, preload, assemblies); init_android_runtime (domain, env, runtimeClass, loader); #endif // ndef NET osBridge.add_monodroid_domain (domain); + log_info (LOG_DEFAULT, "#grendel Leave: %s [%u]", __PRETTY_FUNCTION__, __LINE__); return domain; } @@ -2117,6 +2146,8 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl init_logging_categories (mono_log_mask_raw, mono_log_level_raw); + log_info (LOG_DEFAULT, "#grendel Start: %s", __PRETTY_FUNCTION__); + std::unique_ptr mono_log_mask (mono_log_mask_raw); std::unique_ptr mono_log_level (mono_log_level_raw); @@ -2309,7 +2340,9 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl mono_runtime_init_index = internal_timing->start_event (TimingEventKind::MonoRuntimeInit); } + log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); mono_runtime_init (runtime_args); + log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); if (XA_UNLIKELY (FastTiming::enabled ())) { internal_timing->end_event (mono_runtime_init_index); @@ -2318,7 +2351,9 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl jstring_array_wrapper assemblies (env, assembliesJava); jstring_array_wrapper assembliesPaths (env); /* the first assembly is used to initialize the AppDomain name */ + log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); create_and_initialize_domain (env, klass, runtimeApks, assemblies, nullptr, assembliesPaths, loader, /*is_root_domain:*/ true, /*force_preload_assemblies:*/ false, haveSplitApks); + log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); #if defined (ANDROID) && !defined (NET) // Mono from mono/mono has a bug which requires us to install the handlers after `mono_init_jit_version` is called @@ -2358,6 +2393,8 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl xamarin_app_init (get_function_pointer_at_runtime); #endif // def RELEASE && def ANDROID && def NET startup_in_progress = false; + + log_info (LOG_DEFAULT, "#grendel Leave: %s [%u]", __PRETTY_FUNCTION__, __LINE__); } #if !defined (NET) From a18c3529550e5ac124339993d4654755b8af6e0c Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 5 Jul 2023 12:51:42 +0200 Subject: [PATCH 11/71] New typemap scanner, disable marshal methods by default --- .../Tasks/GenerateJavaStubs.cs | 133 +++++++----------- .../MarshalMethodsAssemblyRewriter.cs | 16 ++- .../Utilities/XAJavaTypeScanner.cs | 24 +--- .../Xamarin.Android.Common.targets | 2 +- src/monodroid/jni/monodroid-glue.cc | 41 +----- 5 files changed, 73 insertions(+), 143 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index ddcd10dfb90..425ff8330cd 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -154,8 +154,7 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) // Put every assembly we'll need in the resolver bool hasExportReference = false; bool haveMonoAndroid = false; - var allTypemapAssemblies = new HashSet (StringComparer.OrdinalIgnoreCase); - var newAllTypemapAssemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); + var allTypemapAssemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); var userAssemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); foreach (var assembly in ResolvedAssemblies) { bool value; @@ -164,20 +163,8 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) continue; } - string fileName = Path.GetFileName (assembly.ItemSpec); - if (abiSpecificAssembliesByPath != null) { - string? abi = assembly.GetMetadata ("Abi"); - if (!String.IsNullOrEmpty (abi)) { - if (!abiSpecificAssembliesByPath.TryGetValue (fileName, out List? items)) { - items = new List (); - abiSpecificAssembliesByPath.Add (fileName, items); - } - - items.Add (assembly); - } - } - bool addAssembly = false; + string fileName = Path.GetFileName (assembly.ItemSpec); if (!hasExportReference && String.Compare ("Mono.Android.Export.dll", fileName, StringComparison.OrdinalIgnoreCase) == 0) { hasExportReference = true; addAssembly = true; @@ -194,9 +181,9 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) } if (addAssembly) { - allTypemapAssemblies.Add (assembly.ItemSpec); - if (!newAllTypemapAssemblies.ContainsKey (assembly.ItemSpec)) { - newAllTypemapAssemblies.Add (assembly.ItemSpec, assembly); + MaybeAddAbiSpecifcAssembly (assembly, fileName); + if (!allTypemapAssemblies.ContainsKey (assembly.ItemSpec)) { + allTypemapAssemblies.Add (assembly.ItemSpec, assembly); } } @@ -209,10 +196,9 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) Log.LogDebugMessage ($"Skipping Java Stub Generation for {asm.ItemSpec}"); continue; } - if (!allTypemapAssemblies.Contains (asm.ItemSpec)) - allTypemapAssemblies.Add (asm.ItemSpec); - if (!newAllTypemapAssemblies.ContainsKey (asm.ItemSpec)) { - newAllTypemapAssemblies.Add (asm.ItemSpec, asm); + MaybeAddAbiSpecifcAssembly (asm, Path.GetFileName (asm.ItemSpec)); + if (!allTypemapAssemblies.ContainsKey (asm.ItemSpec)) { + allTypemapAssemblies.Add (asm.ItemSpec, asm); } string name = Path.GetFileNameWithoutExtension (asm.ItemSpec); @@ -221,65 +207,31 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) } // Step 1 - Find all the JLO types - // var cache = new TypeDefinitionCache (); - // var scanner = new JavaTypeScanner (this.CreateTaskLogger (), cache) { - // ErrorOnCustomJavaObject = ErrorOnCustomJavaObject, - // }; - var newCache = new TypeDefinitionCache (); - var newScanner = new XAJavaTypeScanner (Log, newCache) { - ErrorOnCustomJavaObject = ErrorOnCustomJavaObject, + var cache = new TypeDefinitionCache (); + var scanner = new XAJavaTypeScanner (Log, cache) { + ErrorOnCustomJavaObject = ErrorOnCustomJavaObject, }; - List newAllJavaTypes = newScanner.GetJavaTypes (newAllTypemapAssemblies.Values, res); - //List allJavaTypes = scanner.GetJavaTypes (allTypemapAssemblies, res); - - // Console.WriteLine ("All java types:"); - // foreach (TypeDefinition td in allJavaTypes) { - // Console.WriteLine ($" {td.GetPartialAssemblyQualifiedName (cache)} ({td.Module.FileName})"); - // } - - Console.WriteLine (); - Console.WriteLine ("NEW all java types:"); - foreach (JavaType jt in newAllJavaTypes) { - if (jt.IsABiSpecific) { - Console.WriteLine ($" {jt.Type.GetPartialAssemblyQualifiedName (newCache)} ((ABI-specific))"); - foreach (var kvp in jt.PerAbiTypes) { - TypeDefinition td = kvp.Value; - Console.WriteLine ($" [{kvp.Key}] {td.Module.FileName} (Type token: 0x{td.MetadataToken.ToUInt32 ():x}; MVID: {td.Module.Mvid}])"); - } - } else { - Console.WriteLine ($" {jt.Type.GetPartialAssemblyQualifiedName (newCache)} ({jt.Type.Module.FileName})"); - } - } - // var javaTypes = new List (); - // foreach (TypeDefinition td in allJavaTypes) { - // // Whem marshal methods are in use we do not want to skip non-user assemblies (such as Mono.Android) - we need to generate JCWs for them during - // // application build, unlike in Debug configuration or when marshal methods are disabled, in which case we use JCWs generated during Xamarin.Android - // // build and stored in a jar file. - // if ((!useMarshalMethods && !userAssemblies.ContainsKey (td.Module.Assembly.Name.Name)) || JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (td, cache)) { - // continue; - // } - // javaTypes.Add (td); - // } + List allJavaTypes = scanner.GetJavaTypes (allTypemapAssemblies.Values, res); - var newJavaTypes = new List (); - foreach (JavaType jt in newAllJavaTypes) { + var javaTypes = new List (); + foreach (JavaType jt in allJavaTypes) { // Whem marshal methods are in use we do not want to skip non-user assemblies (such as Mono.Android) - we need to generate JCWs for them during // application build, unlike in Debug configuration or when marshal methods are disabled, in which case we use JCWs generated during Xamarin.Android // build and stored in a jar file. - if ((!useMarshalMethods && !userAssemblies.ContainsKey (jt.Type.Module.Assembly.Name.Name)) || JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (jt.Type, newCache)) { + if ((!useMarshalMethods && !userAssemblies.ContainsKey (jt.Type.Module.Assembly.Name.Name)) || JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (jt.Type, cache)) { continue; } - newJavaTypes.Add (jt); + javaTypes.Add (jt); } MarshalMethodsClassifier classifier = null; if (useMarshalMethods) { - classifier = new MarshalMethodsClassifier (newCache, res, Log); + classifier = new MarshalMethodsClassifier (cache, res, Log); } // Step 2 - Generate Java stub code - var success = CreateJavaSources (newJavaTypes, newCache, classifier, useMarshalMethods); + var success = CreateJavaSources (javaTypes, cache, classifier, useMarshalMethods); if (!success) return; @@ -297,22 +249,22 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) // Step 3 - Generate type maps // Type mappings need to use all the assemblies, always. - WriteTypeMappings (newAllJavaTypes, newCache); + WriteTypeMappings (allJavaTypes, cache); // We need to save a map of .NET type -> ACW type for resource file fixups - var managed = new Dictionary (newJavaTypes.Count, StringComparer.Ordinal); - var java = new Dictionary (newJavaTypes.Count, StringComparer.Ordinal); + var managed = new Dictionary (javaTypes.Count, StringComparer.Ordinal); + var java = new Dictionary (javaTypes.Count, StringComparer.Ordinal); var managedConflicts = new Dictionary> (0, StringComparer.Ordinal); var javaConflicts = new Dictionary> (0, StringComparer.Ordinal); using (var acw_map = MemoryStreamPool.Shared.CreateStreamWriter ()) { - foreach (JavaType jt in newJavaTypes) { + foreach (JavaType jt in javaTypes) { TypeDefinition type = jt.Type; string managedKey = type.FullName.Replace ('/', '.'); - string javaKey = JavaNativeTypeManager.ToJniName (type, newCache).Replace ('/', '.'); + string javaKey = JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.'); - acw_map.Write (type.GetPartialAssemblyQualifiedName (newCache)); + acw_map.Write (type.GetPartialAssemblyQualifiedName (cache)); acw_map.Write (';'); acw_map.Write (javaKey); acw_map.WriteLine (); @@ -322,16 +274,16 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) if (managed.TryGetValue (managedKey, out conflict)) { if (!conflict.Module.Name.Equals (type.Module.Name)) { if (!managedConflicts.TryGetValue (managedKey, out var list)) - managedConflicts.Add (managedKey, list = new List { conflict.GetPartialAssemblyName (newCache) }); - list.Add (type.GetPartialAssemblyName (newCache)); + managedConflicts.Add (managedKey, list = new List { conflict.GetPartialAssemblyName (cache) }); + list.Add (type.GetPartialAssemblyName (cache)); } hasConflict = true; } if (java.TryGetValue (javaKey, out conflict)) { if (!conflict.Module.Name.Equals (type.Module.Name)) { if (!javaConflicts.TryGetValue (javaKey, out var list)) - javaConflicts.Add (javaKey, list = new List { conflict.GetAssemblyQualifiedName (newCache) }); - list.Add (type.GetAssemblyQualifiedName (newCache)); + javaConflicts.Add (javaKey, list = new List { conflict.GetAssemblyQualifiedName (cache) }); + list.Add (type.GetAssemblyQualifiedName (cache)); success = false; } hasConflict = true; @@ -345,7 +297,7 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) acw_map.Write (javaKey); acw_map.WriteLine (); - acw_map.Write (JavaNativeTypeManager.ToCompatJniName (type, newCache).Replace ('/', '.')); + acw_map.Write (JavaNativeTypeManager.ToCompatJniName (type, cache).Replace ('/', '.')); acw_map.Write (';'); acw_map.Write (javaKey); acw_map.WriteLine (); @@ -397,7 +349,7 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) manifest.ForceExtractNativeLibs = true; } - var additionalProviders = manifest.Merge (Log, newCache, newAllJavaTypes, ApplicationJavaClass, EmbedAssemblies, BundledWearApplicationName, MergedManifestDocuments); + var additionalProviders = manifest.Merge (Log, cache, allJavaTypes, ApplicationJavaClass, EmbedAssemblies, BundledWearApplicationName, MergedManifestDocuments); // Only write the new manifest if it actually changed if (manifest.SaveIfChanged (Log, MergedAndroidManifestOutput)) { @@ -417,16 +369,16 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) // Create additional application java sources. StringWriter regCallsWriter = new StringWriter (); regCallsWriter.WriteLine ("\t\t// Application and Instrumentation ACWs must be registered first."); - foreach (JavaType jt in newJavaTypes) { + foreach (JavaType jt in javaTypes) { TypeDefinition type = jt.Type; - if (JavaNativeTypeManager.IsApplication (type, newCache) || JavaNativeTypeManager.IsInstrumentation (type, newCache)) { + if (JavaNativeTypeManager.IsApplication (type, cache) || JavaNativeTypeManager.IsInstrumentation (type, cache)) { if (classifier != null && !classifier.FoundDynamicallyRegisteredMethods (type)) { continue; } - string javaKey = JavaNativeTypeManager.ToJniName (type, newCache).Replace ('/', '.'); + string javaKey = JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.'); regCallsWriter.WriteLine ("\t\tmono.android.Runtime.register (\"{0}\", {1}.class, {1}.__md_methods);", - type.GetAssemblyQualifiedName (newCache), javaKey); + type.GetAssemblyQualifiedName (cache), javaKey); } } regCallsWriter.Close (); @@ -450,6 +402,23 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) Log.LogDebugMessage ($"Number of methods in the project that need marshal method wrappers: {classifier.WrappedMethodCount}"); } } + + void MaybeAddAbiSpecifcAssembly (ITaskItem assembly, string fileName) + { + if (abiSpecificAssembliesByPath == null) { + return; + } + + string? abi = assembly.GetMetadata ("Abi"); + if (!String.IsNullOrEmpty (abi)) { + if (!abiSpecificAssembliesByPath.TryGetValue (fileName, out List? items)) { + items = new List (); + abiSpecificAssembliesByPath.Add (fileName, items); + } + + items.Add (assembly); + } + } } AssemblyDefinition LoadAssembly (string path, DirectoryAssemblyResolver? resolver = null) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs index 1182206250d..22529434ca4 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs @@ -118,12 +118,13 @@ public void Rewrite (DirectoryAssemblyResolver resolver, bool brokenExceptionTra string directory = Path.Combine (Path.GetDirectoryName (path), "new"); Directory.CreateDirectory (directory); string output = Path.Combine (directory, Path.GetFileName (path)); - log.LogDebugMessage ($"Writing new version of assembly: {output}"); + log.LogDebugMessage ($"Writing new version of '{path}' assembly: {output}"); // TODO: this should be used eventually, but it requires that all the types are reloaded from the assemblies before typemaps are generated // since Cecil doesn't update the MVID in the already loaded types //asm.MainModule.Mvid = Guid.NewGuid (); asm.Write (output, writerParams); + CopyFile (output, path); RemoveFile (output); @@ -132,14 +133,25 @@ public void Rewrite (DirectoryAssemblyResolver resolver, bool brokenExceptionTra if (File.Exists (outputPdb)) { CopyFile (outputPdb, pathPdb); } - RemoveFile (pathPdb); + RemoveFile (outputPdb); } } void CopyFile (string source, string target) { log.LogDebugMessage ($"Copying rewritten assembly: {source} -> {target}"); + + string targetBackup = "${target}.bak"; + if (File.Exists (target)) { + // Try to avoid sharing violations by first renaming the target + File.Move (target, targetBackup); + } + File.Copy (source, target, true); + + if (File.Exists (targetBackup)) { + File.Delete (targetBackup); + } } void RemoveFile (string? path) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs index a9d476e9622..c238a394019 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs @@ -147,24 +147,10 @@ AssemblyDefinition LoadAssembly (string path, DirectoryAssemblyResolver resolver ReadWrite = false, }; - return AssemblyDefinition.ReadAssembly (path, readerParameters); - - // MemoryMappedViewStream? viewStream = null; - // try { - // // Create stream because CreateFromFile(string, ...) uses FileShare.None which is too strict - // using var fileStream = new FileStream (path, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, false); - // using var mappedFile = MemoryMappedFile.CreateFromFile ( - // fileStream, null, fileStream.Length, MemoryMappedFileAccess.Read, HandleInheritability.None, true); - // viewStream = mappedFile.CreateViewStream (0, 0, MemoryMappedFileAccess.Read); - - // AssemblyDefinition result = ModuleDefinition.ReadModule (viewStream, readerParameters).Assembly; - - // // We transferred the ownership of the viewStream to the collection. - // viewStream = null; - - // return result; - // } finally { - // viewStream?.Dispose (); - // } + try { + return AssemblyDefinition.ReadAssembly (path, readerParameters); + } catch (Exception ex) { + throw new InvalidOperationException ($"Failed to load assembly: {path}", ex); + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 9c5fedf7980..28c0a5b456d 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -335,7 +335,7 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. false true True - True + False False <_AndroidUseMarshalMethods Condition=" '$(AndroidIncludeDebugSymbols)' == 'True' ">False <_AndroidUseMarshalMethods Condition=" '$(AndroidIncludeDebugSymbols)' != 'True' ">$(AndroidEnableMarshalMethods) diff --git a/src/monodroid/jni/monodroid-glue.cc b/src/monodroid/jni/monodroid-glue.cc index cf51d05ca4b..e7409351411 100644 --- a/src/monodroid/jni/monodroid-glue.cc +++ b/src/monodroid/jni/monodroid-glue.cc @@ -830,25 +830,20 @@ MonodroidRuntime::cleanup_runtime_config (MonovmRuntimeConfigArguments *args, [[ MonoDomain* MonodroidRuntime::create_domain (JNIEnv *env, jstring_array_wrapper &runtimeApks, bool is_root_domain, bool have_split_apks) { - log_info (LOG_DEFAULT, "#grendel Start: %s", __PRETTY_FUNCTION__); size_t user_assemblies_count = 0; - log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); + gather_bundled_assemblies (runtimeApks, &user_assemblies_count, have_split_apks); - log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); + #if defined (NET) size_t blob_time_index; if (XA_UNLIKELY (FastTiming::enabled ())) { blob_time_index = internal_timing->start_event (TimingEventKind::RuntimeConfigBlob); } - log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); if (embeddedAssemblies.have_runtime_config_blob ()) { - log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); runtime_config_args.kind = 1; embeddedAssemblies.get_runtime_config_blob (runtime_config_args.runtimeconfig.data.data, runtime_config_args.runtimeconfig.data.data_len); - log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); monovm_runtimeconfig_initialize (&runtime_config_args, cleanup_runtime_config, nullptr); - log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); } if (XA_UNLIKELY (FastTiming::enabled ())) { @@ -874,9 +869,7 @@ MonodroidRuntime::create_domain (JNIEnv *env, jstring_array_wrapper &runtimeApks #if !defined (NET) if (is_root_domain) { #endif // ndef NET - log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); domain = mono_jit_init_version (const_cast ("RootDomain"), const_cast ("mobile")); - log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); #if !defined (NET) } else { MonoDomain* root_domain = mono_get_root_domain (); @@ -917,7 +910,6 @@ MonodroidRuntime::create_domain (JNIEnv *env, jstring_array_wrapper &runtimeApks } } - log_info (LOG_DEFAULT, "#grendel Leave: %s [%u]", __PRETTY_FUNCTION__, __LINE__); return domain; } @@ -993,7 +985,6 @@ MonodroidRuntime::init_android_runtime ( #endif // ndef NET JNIEnv *env, jclass runtimeClass, jobject loader) { - log_info (LOG_DEFAULT, "#grendel Start: %s", __PRETTY_FUNCTION__); constexpr char icall_typemap_java_to_managed[] = "Java.Interop.TypeManager::monodroid_typemap_java_to_managed"; constexpr char icall_typemap_managed_to_java[] = "Android.Runtime.JNIEnv::monodroid_typemap_managed_to_java"; @@ -1003,22 +994,16 @@ MonodroidRuntime::init_android_runtime ( using j2mFn = MonoReflectionType* (*)(MonoString *java_type); using m2jFn = const char* (*)(MonoReflectionType *type, const uint8_t *mvid); - log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); mono_add_internal_call (icall_typemap_java_to_managed, reinterpret_cast(static_cast(EmbeddedAssemblies::typemap_java_to_managed))); - log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); mono_add_internal_call (icall_typemap_managed_to_java, reinterpret_cast(static_cast(EmbeddedAssemblies::typemap_managed_to_java))); - log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); #else mono_add_internal_call (icall_typemap_java_to_managed, reinterpret_cast(typemap_java_to_managed)); mono_add_internal_call (icall_typemap_managed_to_java, reinterpret_cast(typemap_managed_to_java)); #endif // def RELEASE && def ANDROID #if defined (NET) - log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); mono_add_internal_call ("Android.Runtime.RuntimeNativeMethods::monodroid_debugger_unhandled_exception", reinterpret_cast (monodroid_debugger_unhandled_exception)); - log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); mono_add_internal_call ("Android.Runtime.RuntimeNativeMethods::monodroid_unhandled_exception", reinterpret_cast(monodroid_unhandled_exception)); - log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); #endif // def NET struct JnienvInitializeArgs init = {}; @@ -1136,9 +1121,7 @@ MonodroidRuntime::init_android_runtime ( } #if defined (NET) && defined (ANDROID) - log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); auto initialize = reinterpret_cast (mono_method_get_unmanaged_callers_only_ftnptr (method, &error)); - log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); if (initialize == nullptr) { log_fatal (LOG_DEFAULT, "Failed to get pointer to Initialize. Mono error: %s", mono_error_get_message (&error)); } @@ -1148,9 +1131,7 @@ MonodroidRuntime::init_android_runtime ( "Failed to obtain unmanaged-callers-only pointer to the Android.Runtime.JNIEnvInit.Initialize method. %s", mono_error_get_message (&error) ); - log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); initialize (&init); - log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); #else // def NET && def ANDROID void *args [] = { &init, @@ -1162,7 +1143,6 @@ MonodroidRuntime::init_android_runtime ( if (XA_UNLIKELY (FastTiming::enabled ())) { internal_timing->end_event (native_to_managed_index); } - log_info (LOG_DEFAULT, "#grendel Leave: %s [%u]", __PRETTY_FUNCTION__, __LINE__); } #if defined (NET) @@ -1937,9 +1917,7 @@ MonodroidRuntime::create_and_initialize_domain (JNIEnv* env, jclass runtimeClass [[maybe_unused]] jstring_array_wrapper &assembliesPaths, jobject loader, bool is_root_domain, bool force_preload_assemblies, bool have_split_apks) { - log_info (LOG_DEFAULT, "#grendel Start: %s", __PRETTY_FUNCTION__); MonoDomain* domain = create_domain (env, runtimeApks, is_root_domain, have_split_apks); - log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); #if defined (ANDROID) // Asserting this on desktop apparently breaks a Designer test abort_unless (domain != nullptr, "Failed to create AppDomain"); @@ -1953,12 +1931,9 @@ MonodroidRuntime::create_and_initialize_domain (JNIEnv* env, jclass runtimeClass } #if defined (NET) - log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); default_alc = mono_alc_get_default_gchandle (); - log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); abort_unless (default_alc != nullptr, "Default AssemblyLoadContext not found"); - log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); embeddedAssemblies.install_preload_hooks_for_alc (); log_debug (LOG_ASSEMBLY, "ALC hooks installed"); #endif // def NET @@ -1970,18 +1945,14 @@ MonodroidRuntime::create_and_initialize_domain (JNIEnv* env, jclass runtimeClass bool preload = (androidSystem.is_assembly_preload_enabled () || (is_running_on_desktop && force_preload_assemblies)); #if defined (NET) - log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); load_assemblies (default_alc, preload, assemblies); - log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); init_android_runtime (env, runtimeClass, loader); - log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); #else // def NET load_assemblies (domain, preload, assemblies); init_android_runtime (domain, env, runtimeClass, loader); #endif // ndef NET osBridge.add_monodroid_domain (domain); - log_info (LOG_DEFAULT, "#grendel Leave: %s [%u]", __PRETTY_FUNCTION__, __LINE__); return domain; } @@ -2146,8 +2117,6 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl init_logging_categories (mono_log_mask_raw, mono_log_level_raw); - log_info (LOG_DEFAULT, "#grendel Start: %s", __PRETTY_FUNCTION__); - std::unique_ptr mono_log_mask (mono_log_mask_raw); std::unique_ptr mono_log_level (mono_log_level_raw); @@ -2340,9 +2309,7 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl mono_runtime_init_index = internal_timing->start_event (TimingEventKind::MonoRuntimeInit); } - log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); mono_runtime_init (runtime_args); - log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); if (XA_UNLIKELY (FastTiming::enabled ())) { internal_timing->end_event (mono_runtime_init_index); @@ -2351,9 +2318,7 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl jstring_array_wrapper assemblies (env, assembliesJava); jstring_array_wrapper assembliesPaths (env); /* the first assembly is used to initialize the AppDomain name */ - log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); create_and_initialize_domain (env, klass, runtimeApks, assemblies, nullptr, assembliesPaths, loader, /*is_root_domain:*/ true, /*force_preload_assemblies:*/ false, haveSplitApks); - log_info (LOG_DEFAULT, "#grendel %s [%u]", __PRETTY_FUNCTION__, __LINE__); #if defined (ANDROID) && !defined (NET) // Mono from mono/mono has a bug which requires us to install the handlers after `mono_init_jit_version` is called @@ -2393,8 +2358,6 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl xamarin_app_init (get_function_pointer_at_runtime); #endif // def RELEASE && def ANDROID && def NET startup_in_progress = false; - - log_info (LOG_DEFAULT, "#grendel Leave: %s [%u]", __PRETTY_FUNCTION__, __LINE__); } #if !defined (NET) From 1528dd9d352338bfb21de1bd609b17abdf94b82d Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 6 Jul 2023 11:58:17 +0200 Subject: [PATCH 12/71] Fix test failures --- .../Xamarin.Android.Build.Tests/BuildTest.cs | 4 +- .../Xamarin.Android.Build.Tests/BuildTest2.cs | 2 +- .../BuildReleaseArm64SimpleDotNet.apkdesc | 34 +++--- .../BuildReleaseArm64XFormsDotNet.apkdesc | 108 +++++++++--------- .../MarshalMethodsAssemblyRewriter.cs | 2 +- 5 files changed, 75 insertions(+), 75 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs index b17f7beafbb..e748cc8155f 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs @@ -1326,10 +1326,10 @@ public void Dispose () using (var builder = CreateApkBuilder (Path.Combine ("temp", TestContext.CurrentContext.Test.Name))) { builder.ThrowOnBuildFailure = false; Assert.IsFalse (builder.Build (proj), "Build should have failed with XA4212."); - StringAssertEx.Contains ($"error XA4", builder.LastBuildOutput, "Error should be XA4212"); + StringAssertEx.Contains ($"error : XA4", builder.LastBuildOutput, "Error should be XA4212"); StringAssertEx.Contains ($"Type `UnnamedProject.MyBadJavaObject` implements `Android.Runtime.IJavaObject`", builder.LastBuildOutput, "Error should mention MyBadJavaObject"); Assert.IsTrue (builder.Build (proj, parameters: new [] { "AndroidErrorOnCustomJavaObject=False" }), "Build should have succeeded."); - StringAssertEx.Contains ($"warning XA4", builder.LastBuildOutput, "warning XA4212"); + StringAssertEx.Contains ($"warning : XA4", builder.LastBuildOutput, "warning XA4212"); } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs index 33700618d72..219fa75914f 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs @@ -26,7 +26,7 @@ public partial class BuildTest2 : BaseTest new object[] { /* isClassic */ false, /* isRelease */ true, - /* marshalMethodsEnabled */ true, + /* marshalMethodsEnabled */ false, }, new object[] { /* isClassic */ false, diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc index 74cdceacd9e..4c814f7c391 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc @@ -2,64 +2,64 @@ "Comment": null, "Entries": { "AndroidManifest.xml": { - "Size": 3032 + "Size": 3036 }, "assemblies/_Microsoft.Android.Resource.Designer.dll": { "Size": 1024 }, "assemblies/Java.Interop.dll": { - "Size": 58703 + "Size": 58990 }, "assemblies/Mono.Android.dll": { - "Size": 86588 + "Size": 88074 }, "assemblies/Mono.Android.Runtime.dll": { - "Size": 5798 + "Size": 5819 }, "assemblies/rc.bin": { "Size": 1235 }, "assemblies/System.Console.dll": { - "Size": 6442 + "Size": 6448 }, "assemblies/System.Linq.dll": { - "Size": 9123 + "Size": 9135 }, "assemblies/System.Private.CoreLib.dll": { - "Size": 536436 + "Size": 537441 }, "assemblies/System.Runtime.dll": { - "Size": 2623 + "Size": 2629 }, "assemblies/System.Runtime.InteropServices.dll": { - "Size": 3752 + "Size": 3768 }, "assemblies/UnnamedProject.dll": { - "Size": 3349 + "Size": 3222 }, "classes.dex": { - "Size": 19748 + "Size": 377064 }, "lib/arm64-v8a/libmono-component-marshal-ilgen.so": { - "Size": 93552 + "Size": 97392 }, "lib/arm64-v8a/libmonodroid.so": { - "Size": 380832 + "Size": 380704 }, "lib/arm64-v8a/libmonosgen-2.0.so": { - "Size": 3160360 + "Size": 3177168 }, "lib/arm64-v8a/libSystem.IO.Compression.Native.so": { "Size": 723560 }, "lib/arm64-v8a/libSystem.Native.so": { - "Size": 94392 + "Size": 94424 }, "lib/arm64-v8a/libSystem.Security.Cryptography.Native.Android.so": { "Size": 154904 }, "lib/arm64-v8a/libxamarin-app.so": { - "Size": 16624 + "Size": 11080 }, "META-INF/BNDLTOOL.RSA": { "Size": 1213 @@ -95,5 +95,5 @@ "Size": 1904 } }, - "PackageSize": 2685258 + "PackageSize": 2771274 } \ No newline at end of file diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc index 426961a5f03..69c4b5976c5 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc @@ -2,148 +2,148 @@ "Comment": null, "Entries": { "AndroidManifest.xml": { - "Size": 3568 + "Size": 3572 }, "assemblies/_Microsoft.Android.Resource.Designer.dll": { "Size": 2102 }, "assemblies/FormsViewGroup.dll": { - "Size": 7313 + "Size": 7112 }, "assemblies/Java.Interop.dll": { - "Size": 66911 + "Size": 66908 }, "assemblies/Mono.Android.dll": { - "Size": 469830 + "Size": 469884 }, "assemblies/Mono.Android.Runtime.dll": { - "Size": 5818 + "Size": 5819 }, "assemblies/mscorlib.dll": { - "Size": 3866 + "Size": 3865 }, "assemblies/netstandard.dll": { - "Size": 5578 + "Size": 5581 }, "assemblies/rc.bin": { "Size": 1235 }, "assemblies/System.Collections.Concurrent.dll": { - "Size": 11561 + "Size": 11557 }, "assemblies/System.Collections.dll": { - "Size": 15445 + "Size": 15444 }, "assemblies/System.Collections.NonGeneric.dll": { "Size": 7501 }, "assemblies/System.ComponentModel.dll": { - "Size": 1974 + "Size": 1976 }, "assemblies/System.ComponentModel.Primitives.dll": { - "Size": 2596 + "Size": 2598 }, "assemblies/System.ComponentModel.TypeConverter.dll": { - "Size": 6083 + "Size": 6085 }, "assemblies/System.Console.dll": { - "Size": 6612 + "Size": 6614 }, "assemblies/System.Core.dll": { - "Size": 1992 + "Size": 1991 }, "assemblies/System.Diagnostics.TraceSource.dll": { - "Size": 6589 + "Size": 6590 }, "assemblies/System.dll": { - "Size": 2347 + "Size": 2348 }, "assemblies/System.Drawing.dll": { - "Size": 1938 + "Size": 1940 }, "assemblies/System.Drawing.Primitives.dll": { - "Size": 12004 + "Size": 12010 }, "assemblies/System.IO.Compression.Brotli.dll": { - "Size": 11221 + "Size": 11223 }, "assemblies/System.IO.Compression.dll": { - "Size": 15897 + "Size": 15904 }, "assemblies/System.IO.IsolatedStorage.dll": { - "Size": 9913 + "Size": 9912 }, "assemblies/System.Linq.dll": { - "Size": 19490 + "Size": 19495 }, "assemblies/System.Linq.Expressions.dll": { - "Size": 164335 + "Size": 164340 }, "assemblies/System.Net.Http.dll": { - "Size": 65557 + "Size": 65673 }, "assemblies/System.Net.Primitives.dll": { - "Size": 22482 + "Size": 22474 }, "assemblies/System.Net.Requests.dll": { "Size": 3632 }, "assemblies/System.ObjectModel.dll": { - "Size": 8159 + "Size": 8157 }, "assemblies/System.Private.CoreLib.dll": { - "Size": 834340 + "Size": 834482 }, "assemblies/System.Private.DataContractSerialization.dll": { - "Size": 192404 + "Size": 192929 }, "assemblies/System.Private.Uri.dll": { - "Size": 42947 + "Size": 43458 }, "assemblies/System.Private.Xml.dll": { - "Size": 215908 + "Size": 215826 }, "assemblies/System.Private.Xml.Linq.dll": { - "Size": 16681 + "Size": 16684 }, "assemblies/System.Runtime.dll": { - "Size": 2775 + "Size": 2776 }, "assemblies/System.Runtime.InteropServices.dll": { "Size": 3768 }, "assemblies/System.Runtime.Serialization.dll": { - "Size": 1867 + "Size": 1868 }, "assemblies/System.Runtime.Serialization.Formatters.dll": { - "Size": 2518 + "Size": 2520 }, "assemblies/System.Runtime.Serialization.Primitives.dll": { - "Size": 3802 + "Size": 3805 }, "assemblies/System.Security.Cryptography.dll": { - "Size": 8065 + "Size": 8133 }, "assemblies/System.Text.RegularExpressions.dll": { - "Size": 158997 + "Size": 159004 }, "assemblies/System.Xml.dll": { - "Size": 1760 + "Size": 1761 }, "assemblies/System.Xml.Linq.dll": { "Size": 1778 }, "assemblies/UnnamedProject.dll": { - "Size": 5290 + "Size": 5300 }, "assemblies/Xamarin.AndroidX.Activity.dll": { "Size": 5942 }, "assemblies/Xamarin.AndroidX.AppCompat.AppCompatResources.dll": { - "Size": 6261 + "Size": 6033 }, "assemblies/Xamarin.AndroidX.AppCompat.dll": { - "Size": 120195 + "Size": 119847 }, "assemblies/Xamarin.AndroidX.CardView.dll": { "Size": 6799 @@ -152,13 +152,13 @@ "Size": 17257 }, "assemblies/Xamarin.AndroidX.Core.dll": { - "Size": 100933 + "Size": 100666 }, "assemblies/Xamarin.AndroidX.DrawerLayout.dll": { - "Size": 14800 + "Size": 14631 }, "assemblies/Xamarin.AndroidX.Fragment.dll": { - "Size": 41993 + "Size": 41733 }, "assemblies/Xamarin.AndroidX.Legacy.Support.Core.UI.dll": { "Size": 6080 @@ -176,16 +176,16 @@ "Size": 12923 }, "assemblies/Xamarin.AndroidX.RecyclerView.dll": { - "Size": 90383 + "Size": 89997 }, "assemblies/Xamarin.AndroidX.SavedState.dll": { "Size": 4906 }, "assemblies/Xamarin.AndroidX.SwipeRefreshLayout.dll": { - "Size": 10781 + "Size": 10572 }, "assemblies/Xamarin.AndroidX.ViewPager.dll": { - "Size": 18877 + "Size": 18593 }, "assemblies/Xamarin.Forms.Core.dll": { "Size": 528450 @@ -200,31 +200,31 @@ "Size": 60774 }, "assemblies/Xamarin.Google.Android.Material.dll": { - "Size": 42522 + "Size": 42282 }, "classes.dex": { - "Size": 3117140 + "Size": 3514720 }, "lib/arm64-v8a/libmono-component-marshal-ilgen.so": { - "Size": 93552 + "Size": 97392 }, "lib/arm64-v8a/libmonodroid.so": { "Size": 380704 }, "lib/arm64-v8a/libmonosgen-2.0.so": { - "Size": 3169800 + "Size": 3177168 }, "lib/arm64-v8a/libSystem.IO.Compression.Native.so": { "Size": 723560 }, "lib/arm64-v8a/libSystem.Native.so": { - "Size": 94392 + "Size": 94424 }, "lib/arm64-v8a/libSystem.Security.Cryptography.Native.Android.so": { "Size": 154904 }, "lib/arm64-v8a/libxamarin-app.so": { - "Size": 333744 + "Size": 102136 }, "META-INF/android.support.design_material.version": { "Size": 12 @@ -1913,5 +1913,5 @@ "Size": 325240 } }, - "PackageSize": 7900078 + "PackageSize": 7953326 } \ No newline at end of file diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs index 22529434ca4..9fd466f4b35 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs @@ -141,7 +141,7 @@ void CopyFile (string source, string target) { log.LogDebugMessage ($"Copying rewritten assembly: {source} -> {target}"); - string targetBackup = "${target}.bak"; + string targetBackup = $"{target}.bak"; if (File.Exists (target)) { // Try to avoid sharing violations by first renaming the target File.Move (target, targetBackup); From e4108d43e4c04ff5b8c444c45c25fbacf4b61e5d Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 6 Jul 2023 16:10:52 +0200 Subject: [PATCH 13/71] More test fixes --- .../Utilities/MarshalMethodsAssemblyRewriter.cs | 8 +++++++- .../MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs | 5 +++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs index 9fd466f4b35..9869e66fed0 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs @@ -150,7 +150,13 @@ void CopyFile (string source, string target) File.Copy (source, target, true); if (File.Exists (targetBackup)) { - File.Delete (targetBackup); + try { + File.Delete (targetBackup); + } catch (Exception ex) { + // On Windows the deletion may fail, depending on lock state of the original `target` file before the move. + log.LogDebugMessage ($"While trying to delete '{targetBackup}', exception was thrown: {ex}"); + log.LogDebugMessage ($"Failed to delete backup file '{targetBackup}', ignoring."); + } } } diff --git a/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs b/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs index 72133c28f25..e6331a607d1 100644 --- a/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs @@ -29,6 +29,11 @@ public void Teardown () [Test] public void NativeAssemblyCacheWithSatelliteAssemblies ([Values (true, false)] bool enableMarshalMethods) { + // TODO: enable when marshal methods are fixed + if (enableMarshalMethods) { + Assert.Ignore ("Test is skipped when marshal methods are enabled, pending fixes to MM for .NET9"); + } + var path = Path.Combine ("temp", TestName); var lib = new XamarinAndroidLibraryProject { ProjectName = "Localization", From 55b1325d9dfe234a05ed545b50b5c2841d36096f Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 6 Jul 2023 21:22:20 +0200 Subject: [PATCH 14/71] Updates + fixes + per-rid typemaps --- .../JniRemappingAssemblyGenerator.cs | 1 - .../Utilities/LlvmIrGenerator/LlvmIrModule.cs | 1 - .../Utilities/TypeMapGenerator.cs | 67 ++++++++++++------- ...peMappingReleaseNativeAssemblyGenerator.cs | 49 +++++--------- src/monodroid/jni/application_dso_stub.cc | 2 +- src/monodroid/jni/monodroid-glue-internal.hh | 2 +- src/monodroid/jni/monodroid-glue.cc | 12 ++-- src/monodroid/jni/xamarin-app.hh | 2 +- 8 files changed, 67 insertions(+), 69 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/JniRemappingAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/JniRemappingAssemblyGenerator.cs index b7cbd4b49fd..93ab1cd7b31 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/JniRemappingAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/JniRemappingAssemblyGenerator.cs @@ -206,7 +206,6 @@ public JniRemappingAssemblyGenerator (List typeRepl } var typeReplacements = new List> (); - Console.WriteLine ($"Type replacement input count: {typeReplacementsInput.Count}"); foreach (JniRemappingTypeReplacement mtr in typeReplacementsInput) { var entry = new JniRemappingTypeReplacementEntry { name = MakeJniRemappingString (mtr.From), diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModule.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModule.cs index 3dd0c72290e..be1570dafe8 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModule.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModule.cs @@ -640,7 +640,6 @@ public LlvmIrFunction DeclareExternalFunction (LlvmIrFunctionSignature sig, Llvm /// public StructureInfo MapStructure () { - Console.WriteLine ($"Mapping structure: {typeof(T)}"); if (structures == null) { structures = new Dictionary (); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs index c71f154c72b..cc32ada49d3 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs @@ -113,7 +113,7 @@ public ReleaseGenerationState (string[] supportedAbis) if (TempModulesAbiAgnostic == null) { TempModulesAbiAgnostic = dict; } - tempModules.Add (AbiToArch (abi), dict); + tempModules.Add (GeneratePackageManagerJava.GetAndroidTargetArchForAbi (abi), dict); } TempModules = new ReadOnlyDictionary> (tempModules); @@ -465,7 +465,6 @@ bool GenerateRelease (bool skipJniAddNativeMethodRegistrationAttributeScan, List } } - var mappingData = new Dictionary (); foreach (var kvp in state.TempModules) { AndroidTargetArch arch = kvp.Key; Dictionary tempModules = kvp.Value; @@ -483,12 +482,10 @@ bool GenerateRelease (bool skipJniAddNativeMethodRegistrationAttributeScan, List module.Types = module.TypesScratch.Values.ToArray (); } - mappingData.Add (arch, new NativeTypeMappingData (logger, modules)); + var composer = new TypeMappingReleaseNativeAssemblyGenerator (new NativeTypeMappingData (logger, modules)); + GenerateNativeAssembly (arch, composer, composer.Construct (), outputDirectory); } - var composer = new TypeMappingReleaseNativeAssemblyGenerator (mappingData); - GenerateNativeAssembly (composer, composer.Construct (), outputDirectory); - return true; } @@ -497,34 +494,52 @@ bool ShouldSkipInJavaToManaged (TypeDefinition td) return td.IsInterface || td.HasGenericParameters; } + string GetOutputFilePath (string baseFileName, string abi) => $"{baseFileName}.{abi}.ll"; + + void GenerateNativeAssembly (AndroidTargetArch arch, LLVMIR.LlvmIrComposer composer, LLVMIR.LlvmIrModule typeMapModule, string baseFileName) + { + WriteNativeAssembly ( + arch, + composer, + typeMapModule, + GetOutputFilePath (baseFileName, ArchToAbi (arch)) + ); + } + void GenerateNativeAssembly (LLVMIR.LlvmIrComposer composer, LLVMIR.LlvmIrModule typeMapModule, string baseFileName) { - AndroidTargetArch arch; foreach (string abi in supportedAbis) { - arch = GeneratePackageManagerJava.GetAndroidTargetArchForAbi (abi); - - string outputFile = $"{baseFileName}.{abi}.ll"; - using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) { - try { - composer.Generate (typeMapModule, arch, sw, outputFile); - } catch { - throw; - } finally { - sw.Flush (); - Files.CopyIfStreamChanged (sw.BaseStream, outputFile); - } + WriteNativeAssembly ( + GeneratePackageManagerJava.GetAndroidTargetArchForAbi (abi), + composer, + typeMapModule, + GetOutputFilePath (baseFileName, abi) + ); + } + } + + void WriteNativeAssembly (AndroidTargetArch arch, LLVMIR.LlvmIrComposer composer, LLVMIR.LlvmIrModule typeMapModule, string outputFile) + { + using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) { + try { + composer.Generate (typeMapModule, arch, sw, outputFile); + } catch { + throw; + } finally { + sw.Flush (); + Files.CopyIfStreamChanged (sw.BaseStream, outputFile); } } } - static AndroidTargetArch AbiToArch (string abi) + static string ArchToAbi (AndroidTargetArch arch) { - return abi switch { - "armeabi-v7a" => AndroidTargetArch.Arm, - "arm64-v8a" => AndroidTargetArch.Arm64, - "x86_64" => AndroidTargetArch.X86_64, - "x86" => AndroidTargetArch.X86, - _ => throw new InvalidOperationException ($"Unknown ABI {abi}") + return arch switch { + AndroidTargetArch.Arm => "armeabi-v7a", + AndroidTargetArch.Arm64 => "arm64-v8a", + AndroidTargetArch.X86_64 => "x86_64", + AndroidTargetArch.X86 => "x86", + _ => throw new InvalidOperationException ($"Unknown architecture {arch}") }; } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs index 61e52f60f9e..a6e4fd465df 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs @@ -5,7 +5,6 @@ using System.Text; using Xamarin.Android.Tasks.LLVMIR; -using Xamarin.Android.Tools; namespace Xamarin.Android.Tasks { @@ -159,55 +158,38 @@ public int Compare (StructureInstance a, StructureInstance> MapModules; - public readonly List> JavaMap; - public readonly Dictionary JavaTypesByName; - public readonly List JavaNames; - public readonly NativeTypeMappingData MappingData; - public ulong ModuleCounter = 0; - + public List> MapModules; + public Dictionary JavaTypesByName; + public List JavaNames; + public List> JavaMap; public List AllModulesData; - - public ArchGenerationState (NativeTypeMappingData mappingData) - { - MapModules = new List> (); - JavaMap = new List> (); - JavaTypesByName = new Dictionary (StringComparer.Ordinal); - JavaNames = new List (); - MappingData = mappingData; - } } - readonly Dictionary mappingData; + readonly NativeTypeMappingData mappingData; StructureInfo typeMapJavaStructureInfo; StructureInfo typeMapModuleStructureInfo; StructureInfo typeMapModuleEntryStructureInfo; JavaNameHash32Comparer javaNameHash32Comparer; JavaNameHash64Comparer javaNameHash64Comparer; - Dictionary archState; - public TypeMappingReleaseNativeAssemblyGenerator (Dictionary mappingData) + ulong moduleCounter = 0; + + public TypeMappingReleaseNativeAssemblyGenerator (NativeTypeMappingData mappingData) { this.mappingData = mappingData ?? throw new ArgumentNullException (nameof (mappingData)); javaNameHash32Comparer = new JavaNameHash32Comparer (); javaNameHash64Comparer = new JavaNameHash64Comparer (); - - archState = new Dictionary (mappingData.Count); - foreach (var kvp in mappingData) { - if (kvp.Value == null) { - throw new ArgumentException ("must not contain null values", nameof (mappingData)); - } - - archState.Add (kvp.Key, new ArchGenerationState (kvp.Value)); - } } protected override void Construct (LlvmIrModule module) { MapStructures (module); + var cs = new ConstructionState (); + cs.JavaTypesByName = new Dictionary (StringComparer.Ordinal); + cs.JavaNames = new List (); InitJavaMap (cs); InitMapModules (cs); HashJavaNames (cs); @@ -322,9 +304,8 @@ void InitJavaMap (ConstructionState cs) { cs.JavaMap = new List> (); TypeMapJava map_entry; - - foreach (TypeMapGenerator.TypeMapReleaseEntry entry in state.MappingData.JavaTypes) { - state.JavaNames.Add (entry.JavaName); + foreach (TypeMapGenerator.TypeMapReleaseEntry entry in mappingData.JavaTypes) { + cs.JavaNames.Add (entry.JavaName); map_entry = new TypeMapJava { module_index = (uint)entry.ModuleIndex, // UInt32.MaxValue, @@ -364,7 +345,7 @@ void InitMapModules (ConstructionState cs) java_name_width = 0, }; - state.MapModules.Add (new StructureInstance (map_module)); + cs.MapModules.Add (new StructureInstance (typeMapModuleStructureInfo, map_module)); } } diff --git a/src/monodroid/jni/application_dso_stub.cc b/src/monodroid/jni/application_dso_stub.cc index 8a59a03e51a..97f554e4c25 100644 --- a/src/monodroid/jni/application_dso_stub.cc +++ b/src/monodroid/jni/application_dso_stub.cc @@ -202,7 +202,7 @@ const MarshalMethodName mm_method_names[] = { }, }; -void xamarin_app_init ([[maybe_unused]] get_function_pointer_fn fn) noexcept +void xamarin_app_init ([[maybe_unused]] JNIEnv *env, [[maybe_unused]] get_function_pointer_fn fn) noexcept { // Dummy } diff --git a/src/monodroid/jni/monodroid-glue-internal.hh b/src/monodroid/jni/monodroid-glue-internal.hh index 85128eb1915..84ac5f5188a 100644 --- a/src/monodroid/jni/monodroid-glue-internal.hh +++ b/src/monodroid/jni/monodroid-glue-internal.hh @@ -299,7 +299,7 @@ namespace xamarin::android::internal void set_debug_options (); void parse_gdb_options (); - void mono_runtime_init (dynamic_local_string& runtime_args); + void mono_runtime_init (JNIEnv *env, dynamic_local_string& runtime_args); #if defined (NET) void init_android_runtime (JNIEnv *env, jclass runtimeClass, jobject loader); #else //def NET diff --git a/src/monodroid/jni/monodroid-glue.cc b/src/monodroid/jni/monodroid-glue.cc index e7409351411..ab041f4b1c9 100644 --- a/src/monodroid/jni/monodroid-glue.cc +++ b/src/monodroid/jni/monodroid-glue.cc @@ -638,7 +638,7 @@ MonodroidRuntime::set_debug_options (void) } void -MonodroidRuntime::mono_runtime_init ([[maybe_unused]] dynamic_local_string& runtime_args) +MonodroidRuntime::mono_runtime_init ([[maybe_unused]] JNIEnv *env, [[maybe_unused]] dynamic_local_string& runtime_args) { #if defined (DEBUG) && !defined (WINDOWS) RuntimeOptions options{}; @@ -809,7 +809,9 @@ MonodroidRuntime::mono_runtime_init ([[maybe_unused]] dynamic_local_stringstart_event (TimingEventKind::MonoRuntimeInit); } - mono_runtime_init (runtime_args); + mono_runtime_init (env, runtime_args); if (XA_UNLIKELY (FastTiming::enabled ())) { internal_timing->end_event (mono_runtime_init_index); @@ -2355,7 +2357,9 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl } #if defined (RELEASE) && defined (ANDROID) && defined (NET) - xamarin_app_init (get_function_pointer_at_runtime); + if (application_config.marshal_methods_enabled) { + xamarin_app_init (env, get_function_pointer_at_runtime); + } #endif // def RELEASE && def ANDROID && def NET startup_in_progress = false; } diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index 3dbc037407b..d50fc39a683 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -351,7 +351,7 @@ MONO_API MONO_API_EXPORT const MarshalMethodName mm_method_names[]; using get_function_pointer_fn = void(*)(uint32_t mono_image_index, uint32_t class_index, uint32_t method_token, void*& target_ptr); -MONO_API MONO_API_EXPORT void xamarin_app_init (get_function_pointer_fn fn) noexcept; +MONO_API MONO_API_EXPORT void xamarin_app_init (JNIEnv *env, get_function_pointer_fn fn) noexcept; #endif // def RELEASE && def ANDROID && def NET #endif // __XAMARIN_ANDROID_TYPEMAP_H From f30e88e43bc81c6a15faa39ddeff9a07e1d956f1 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 7 Jul 2023 13:19:34 +0200 Subject: [PATCH 15/71] Hopefully fix Windows tests --- .../Tasks/GetAotArguments.cs | 30 +------------- .../Tasks/LinkApplicationSharedLibraries.cs | 4 +- .../Utilities/MonoAndroidHelper.cs | 41 +++++++++++++++++++ 3 files changed, 45 insertions(+), 30 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GetAotArguments.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GetAotArguments.cs index 3037f957c7d..b839175b2ed 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GetAotArguments.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GetAotArguments.cs @@ -293,14 +293,8 @@ string GetLdFlags (NdkTools ndk, AndroidTargetArch arch, int level, string toolP libs.Add (Path.Combine (androidLibPath, "libc.so")); libs.Add (Path.Combine (androidLibPath, "libm.so")); } else if (!UseAndroidNdk && EnableLLVM) { - // We need to link against libc and libm, but since NDK is not in use, the linker won't be able to find the actual Android libraries. - // Therefore, we will use their stubs to satisfy the linker. At runtime they will, of course, use the actual Android libraries. - string relPath = Path.Combine ("..", ".."); - if (!OS.IsWindows) { - // the `binutils` directory is one level down (${OS}/binutils) than the Windows one - relPath = Path.Combine (relPath, ".."); - } - string libstubsPath = Path.GetFullPath (Path.Combine (AndroidBinUtilsDirectory, relPath, "libstubs", ArchToRid (arch))); + string libstubsPath = MonoAndroidHelper.GetLibstubsArchDirectoryPath (AndroidBinUtilsDirectory, arch); + libs.Add (Path.Combine (libstubsPath, "libc.so")); libs.Add (Path.Combine (libstubsPath, "libm.so")); } @@ -332,26 +326,6 @@ string GetLdFlags (NdkTools ndk, AndroidTargetArch arch, int level, string toolP } return ldFlags.ToString (); - - string ArchToRid (AndroidTargetArch arch) - { - switch (arch) { - case AndroidTargetArch.Arm64: - return "android-arm64"; - - case AndroidTargetArch.Arm: - return "android-arm"; - - case AndroidTargetArch.X86: - return "android-x86"; - - case AndroidTargetArch.X86_64: - return "android-x64"; - - default: - throw new InvalidOperationException ($"Internal error: unsupported ABI '{arch}'"); - } - } } static string GetNdkToolchainLibraryDir (NdkTools ndk, string binDir, string archDir = null) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs b/src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs index cc39da6215c..5f4b09ececa 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs @@ -113,8 +113,8 @@ void RunLinker (Config config) IEnumerable GetLinkerConfigs () { - string runtimeNativeLibsDir = Path.GetFullPath (Path.Combine (AndroidBinUtilsDirectory, "..", "..", "..", "lib")); - string runtimeNativeLibStubsDir = Path.GetFullPath (Path.Combine (runtimeNativeLibsDir, "..", "libstubs")); + string runtimeNativeLibsDir = MonoAndroidHelper.GetNativeLibsRootDirectoryPath (AndroidBinUtilsDirectory); + string runtimeNativeLibStubsDir = MonoAndroidHelper.GetLibstubsRootDirectoryPath (AndroidBinUtilsDirectory); var abis = new Dictionary (StringComparer.Ordinal); ITaskItem[] dsos = ApplicationSharedLibraries; foreach (ITaskItem item in dsos) { diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs index 77fb0b8b356..6dc5926f0da 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs @@ -576,6 +576,17 @@ public static AndroidTargetArch AbiToTargetArch (string abi) } } + public static string ArchToRid (AndroidTargetArch arch) + { + return arch switch { + AndroidTargetArch.Arm64 => "android-arm64", + AndroidTargetArch.Arm => "android-arm", + AndroidTargetArch.X86 => "android-x86", + AndroidTargetArch.X86_64 => "android-x64", + _ => throw new InvalidOperationException ($"Internal error: unsupported ABI '{arch}'") + }; + } + public static string? CultureInvariantToString (object? obj) { if (obj == null) { @@ -601,5 +612,35 @@ public static int ConvertSupportedOSPlatformVersionToApiLevel (string version) } return apiLevel; } + + static string GetToolsRootDirectoryRelativePath (string androidBinUtilsDirectory) + { + // We need to link against libc and libm, but since NDK is not in use, the linker won't be able to find the actual Android libraries. + // Therefore, we will use their stubs to satisfy the linker. At runtime they will, of course, use the actual Android libraries. + string relPath = Path.Combine ("..", ".."); + if (!OS.IsWindows) { + // the `binutils` directory is one level down (${OS}/binutils) than the Windows one + relPath = Path.Combine (relPath, ".."); + } + + return relPath; + } + + public static string GetLibstubsArchDirectoryPath (string androidBinUtilsDirectory, AndroidTargetArch arch) + { + return Path.Combine (GetLibstubsRootDirectoryPath (androidBinUtilsDirectory), ArchToRid (arch)); + } + + public static string GetLibstubsRootDirectoryPath (string androidBinUtilsDirectory) + { + string relPath = GetToolsRootDirectoryRelativePath (androidBinUtilsDirectory); + return Path.GetFullPath (Path.Combine (androidBinUtilsDirectory, relPath, "libstubs")); + } + + public static string GetNativeLibsRootDirectoryPath (string androidBinUtilsDirectory) + { + string relPath = GetToolsRootDirectoryRelativePath (androidBinUtilsDirectory); + return Path.GetFullPath (Path.Combine (androidBinUtilsDirectory, relPath, "lib")); + } } } From 40a5d472f18d2c49ab9bea079ff7caf3f81501ea Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 7 Jul 2023 17:15:40 +0200 Subject: [PATCH 16/71] Add some timing debug prints --- .../Tasks/GenerateJavaStubs.cs | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 425ff8330cd..93b08198926 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -105,7 +105,11 @@ public override bool RunTask () // We're going to do 3 steps here instead of separate tasks so // we can share the list of JLO TypeDefinitions between them using (DirectoryAssemblyResolver res = MakeResolver (useMarshalMethods)) { + var runWatch = new Stopwatch (); + runWatch.Start (); Run (res, useMarshalMethods); + runWatch.Stop (); + Log.LogDebugMessage ($"Run took: {runWatch.Elapsed}"); } } catch (XamarinAndroidException e) { Log.LogCodedError (string.Format ("XA{0:0000}", e.Code), e.MessageWithoutCode); @@ -156,6 +160,8 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) bool haveMonoAndroid = false; var allTypemapAssemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); var userAssemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); + var stopwatch = new Stopwatch (); + stopwatch.Start (); foreach (var assembly in ResolvedAssemblies) { bool value; if (bool.TryParse (assembly.GetMetadata (AndroidSkipJavaStubGeneration), out value) && value) { @@ -189,7 +195,11 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) res.Load (assembly.ItemSpec); } + stopwatch.Stop (); + Log.LogDebugMessage ($"Stopwatch #1, elapsed: {stopwatch.Elapsed}"); + stopwatch.Reset (); + stopwatch.Start (); // However we only want to look for JLO types in user code for Java stub code generation foreach (var asm in ResolvedUserAssemblies) { if (bool.TryParse (asm.GetMetadata (AndroidSkipJavaStubGeneration), out bool value) && value) { @@ -205,6 +215,8 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) if (!userAssemblies.ContainsKey (name)) userAssemblies.Add (name, asm.ItemSpec); } + stopwatch.Stop (); + Log.LogDebugMessage ($"Stopwatch #2, elapsed: {stopwatch.Elapsed}"); // Step 1 - Find all the JLO types var cache = new TypeDefinitionCache (); @@ -212,7 +224,11 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) ErrorOnCustomJavaObject = ErrorOnCustomJavaObject, }; + stopwatch.Reset (); + stopwatch.Start (); List allJavaTypes = scanner.GetJavaTypes (allTypemapAssemblies.Values, res); + stopwatch.Stop (); + Log.LogDebugMessage ($"Stopwatch #3, elapsed: {stopwatch.Elapsed}"); var javaTypes = new List (); foreach (JavaType jt in allJavaTypes) { @@ -231,7 +247,11 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) } // Step 2 - Generate Java stub code + stopwatch.Reset (); + stopwatch.Start (); var success = CreateJavaSources (javaTypes, cache, classifier, useMarshalMethods); + stopwatch.Stop (); + Log.LogDebugMessage ($"Stopwatch #4, elapsed: {stopwatch.Elapsed}"); if (!success) return; @@ -249,7 +269,11 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) // Step 3 - Generate type maps // Type mappings need to use all the assemblies, always. + stopwatch.Reset (); + stopwatch.Start (); WriteTypeMappings (allJavaTypes, cache); + stopwatch.Stop (); + Log.LogDebugMessage ($"Stopwatch #5, elapsed: {stopwatch.Elapsed}"); // We need to save a map of .NET type -> ACW type for resource file fixups var managed = new Dictionary (javaTypes.Count, StringComparer.Ordinal); From 6810bc8cc8c7bcb90135480587a5fd99ca5da96f Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 7 Jul 2023 19:06:02 +0200 Subject: [PATCH 17/71] More printfs --- .../Utilities/XAJavaTypeScanner.cs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs index c238a394019..89cdd0e4b5c 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Collections.ObjectModel; +using System.Diagnostics; using System.IO; //using System.IO.MemoryMappedFiles; @@ -58,15 +59,25 @@ public XAJavaTypeScanner (TaskLoggingHelper log, TypeDefinitionCache cache) public List GetJavaTypes (ICollection inputAssemblies, DirectoryAssemblyResolver resolver) { var types = new Dictionary (StringComparer.Ordinal); + var stopwatch = new Stopwatch (); foreach (ITaskItem asmItem in inputAssemblies) { AndroidTargetArch arch = GetTargetArch (asmItem); + + stopwatch.Start (); AssemblyDefinition asmdef = LoadAssembly (asmItem.ItemSpec, resolver); + stopwatch.Stop (); + log.LogMessage ($"Load of assembly '{asmItem.ItemSpec}', elapsed: {stopwatch.Elapsed}"); + stopwatch.Reset (); + stopwatch.Start (); foreach (ModuleDefinition md in asmdef.Modules) { foreach (TypeDefinition td in md.Types) { AddJavaType (td, types, arch); } } + stopwatch.Stop (); + log.LogMessage ($"Add all types from assembly '{asmItem.ItemSpec}', elapsed: {stopwatch.Elapsed}"); + stopwatch.Reset (); } var ret = new List (); From 023005262bb427690bafec212aa6aec5197af5ff Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 7 Jul 2023 20:53:45 +0200 Subject: [PATCH 18/71] Let's see if using the resolver directly helps --- src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs index 89cdd0e4b5c..6cc5e4bc906 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs @@ -64,7 +64,7 @@ public List GetJavaTypes (ICollection inputAssemblies, Dire AndroidTargetArch arch = GetTargetArch (asmItem); stopwatch.Start (); - AssemblyDefinition asmdef = LoadAssembly (asmItem.ItemSpec, resolver); + AssemblyDefinition asmdef = resolver.GetAssembly (asmItem.ItemSpec); stopwatch.Stop (); log.LogMessage ($"Load of assembly '{asmItem.ItemSpec}', elapsed: {stopwatch.Elapsed}"); stopwatch.Reset (); From 04271476bdba5d61455953b2d10cbc5faadc0160 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 7 Jul 2023 22:27:36 +0200 Subject: [PATCH 19/71] Remove debug stuff --- .../Tasks/GenerateJavaStubs.cs | 28 ++--------------- .../Utilities/XAJavaTypeScanner.cs | 31 ------------------- 2 files changed, 2 insertions(+), 57 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 93b08198926..18d1d60d2d5 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -105,11 +105,7 @@ public override bool RunTask () // We're going to do 3 steps here instead of separate tasks so // we can share the list of JLO TypeDefinitions between them using (DirectoryAssemblyResolver res = MakeResolver (useMarshalMethods)) { - var runWatch = new Stopwatch (); - runWatch.Start (); Run (res, useMarshalMethods); - runWatch.Stop (); - Log.LogDebugMessage ($"Run took: {runWatch.Elapsed}"); } } catch (XamarinAndroidException e) { Log.LogCodedError (string.Format ("XA{0:0000}", e.Code), e.MessageWithoutCode); @@ -160,8 +156,7 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) bool haveMonoAndroid = false; var allTypemapAssemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); var userAssemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); - var stopwatch = new Stopwatch (); - stopwatch.Start (); + foreach (var assembly in ResolvedAssemblies) { bool value; if (bool.TryParse (assembly.GetMetadata (AndroidSkipJavaStubGeneration), out value) && value) { @@ -195,11 +190,7 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) res.Load (assembly.ItemSpec); } - stopwatch.Stop (); - Log.LogDebugMessage ($"Stopwatch #1, elapsed: {stopwatch.Elapsed}"); - stopwatch.Reset (); - stopwatch.Start (); // However we only want to look for JLO types in user code for Java stub code generation foreach (var asm in ResolvedUserAssemblies) { if (bool.TryParse (asm.GetMetadata (AndroidSkipJavaStubGeneration), out bool value) && value) { @@ -215,22 +206,15 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) if (!userAssemblies.ContainsKey (name)) userAssemblies.Add (name, asm.ItemSpec); } - stopwatch.Stop (); - Log.LogDebugMessage ($"Stopwatch #2, elapsed: {stopwatch.Elapsed}"); // Step 1 - Find all the JLO types var cache = new TypeDefinitionCache (); var scanner = new XAJavaTypeScanner (Log, cache) { ErrorOnCustomJavaObject = ErrorOnCustomJavaObject, }; - - stopwatch.Reset (); - stopwatch.Start (); List allJavaTypes = scanner.GetJavaTypes (allTypemapAssemblies.Values, res); - stopwatch.Stop (); - Log.LogDebugMessage ($"Stopwatch #3, elapsed: {stopwatch.Elapsed}"); - var javaTypes = new List (); + foreach (JavaType jt in allJavaTypes) { // Whem marshal methods are in use we do not want to skip non-user assemblies (such as Mono.Android) - we need to generate JCWs for them during // application build, unlike in Debug configuration or when marshal methods are disabled, in which case we use JCWs generated during Xamarin.Android @@ -247,11 +231,7 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) } // Step 2 - Generate Java stub code - stopwatch.Reset (); - stopwatch.Start (); var success = CreateJavaSources (javaTypes, cache, classifier, useMarshalMethods); - stopwatch.Stop (); - Log.LogDebugMessage ($"Stopwatch #4, elapsed: {stopwatch.Elapsed}"); if (!success) return; @@ -269,11 +249,7 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) // Step 3 - Generate type maps // Type mappings need to use all the assemblies, always. - stopwatch.Reset (); - stopwatch.Start (); WriteTypeMappings (allJavaTypes, cache); - stopwatch.Stop (); - Log.LogDebugMessage ($"Stopwatch #5, elapsed: {stopwatch.Elapsed}"); // We need to save a map of .NET type -> ACW type for resource file fixups var managed = new Dictionary (javaTypes.Count, StringComparer.Ordinal); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs index 6cc5e4bc906..539099d2908 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs @@ -1,9 +1,6 @@ using System; using System.Collections.Generic; using System.Collections.ObjectModel; -using System.Diagnostics; -using System.IO; -//using System.IO.MemoryMappedFiles; using Java.Interop.Tools.Cecil; using Microsoft.Build.Framework; @@ -59,25 +56,15 @@ public XAJavaTypeScanner (TaskLoggingHelper log, TypeDefinitionCache cache) public List GetJavaTypes (ICollection inputAssemblies, DirectoryAssemblyResolver resolver) { var types = new Dictionary (StringComparer.Ordinal); - var stopwatch = new Stopwatch (); foreach (ITaskItem asmItem in inputAssemblies) { AndroidTargetArch arch = GetTargetArch (asmItem); - - stopwatch.Start (); AssemblyDefinition asmdef = resolver.GetAssembly (asmItem.ItemSpec); - stopwatch.Stop (); - log.LogMessage ($"Load of assembly '{asmItem.ItemSpec}', elapsed: {stopwatch.Elapsed}"); - stopwatch.Reset (); - stopwatch.Start (); foreach (ModuleDefinition md in asmdef.Modules) { foreach (TypeDefinition td in md.Types) { AddJavaType (td, types, arch); } } - stopwatch.Stop (); - log.LogMessage ($"Add all types from assembly '{asmItem.ItemSpec}', elapsed: {stopwatch.Elapsed}"); - stopwatch.Reset (); } var ret = new List (); @@ -146,22 +133,4 @@ AndroidTargetArch GetTargetArch (ITaskItem asmItem) _ => throw new NotSupportedException ($"Unsupported ABI '{abi}' for assembly {asmItem.ItemSpec}") }; } - - AssemblyDefinition LoadAssembly (string path, DirectoryAssemblyResolver resolver) - { - string pdbPath = Path.ChangeExtension (path, ".pdb"); - var readerParameters = new ReaderParameters { - AssemblyResolver = resolver, - InMemory = true, - ReadingMode = ReadingMode.Immediate, - ReadSymbols = File.Exists (pdbPath), - ReadWrite = false, - }; - - try { - return AssemblyDefinition.ReadAssembly (path, readerParameters); - } catch (Exception ex) { - throw new InvalidOperationException ($"Failed to load assembly: {path}", ex); - } - } } From 0b63e281fe288aff63d30d5f77a10c2a90a1b183 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 10 Jul 2023 21:49:35 +0200 Subject: [PATCH 20/71] Use a custom assembly resolver --- .../Tasks/GenerateJavaStubs.cs | 19 +- .../Utilities/ManifestDocument.cs | 2 +- .../MarshalMethodsAssemblyRewriter.cs | 4 +- .../Utilities/MarshalMethodsClassifier.cs | 4 +- .../Utilities/MonoAndroidHelper.cs | 36 ++ .../Utilities/XAAssemblyResolver.cs | 333 ++++++++++++++++++ .../Utilities/XAJavaTypeScanner.cs | 22 +- 7 files changed, 387 insertions(+), 33 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/XAAssemblyResolver.cs diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 18d1d60d2d5..80f1f25b279 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -104,7 +104,7 @@ public override bool RunTask () bool useMarshalMethods = !Debug && EnableMarshalMethods; // We're going to do 3 steps here instead of separate tasks so // we can share the list of JLO TypeDefinitions between them - using (DirectoryAssemblyResolver res = MakeResolver (useMarshalMethods)) { + using (XAAssemblyResolver res = MakeResolver (useMarshalMethods)) { Run (res, useMarshalMethods); } } catch (XamarinAndroidException e) { @@ -123,7 +123,7 @@ public override bool RunTask () return !Log.HasLoggedErrors; } - DirectoryAssemblyResolver MakeResolver (bool useMarshalMethods) + XAAssemblyResolver MakeResolver (bool useMarshalMethods) { var readerParams = new ReaderParameters(); if (useMarshalMethods) { @@ -131,17 +131,17 @@ DirectoryAssemblyResolver MakeResolver (bool useMarshalMethods) readerParams.InMemory = true; } - var res = new DirectoryAssemblyResolver (this.CreateTaskLogger (), loadDebugSymbols: true, loadReaderParameters: readerParams); + var res = new XAAssemblyResolver (Log, loadDebugSymbols: true, loadReaderParameters: readerParams); foreach (var dir in FrameworkDirectories) { if (Directory.Exists (dir.ItemSpec)) { - res.SearchDirectories.Add (dir.ItemSpec); + res.FrameworkSearchDirectories.Add (dir.ItemSpec); } } return res; } - void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) + void Run (XAAssemblyResolver res, bool useMarshalMethods) { PackageNamingPolicy pnp; JavaNativeTypeManager.PackageNamingPolicy = Enum.TryParse (PackageNamingPolicy, out pnp) ? pnp : PackageNamingPolicyEnum.LowercaseCrc64; @@ -188,7 +188,7 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) } } - res.Load (assembly.ItemSpec); + res.Load (MonoAndroidHelper.GetTargetArch (assembly), assembly.ItemSpec); } // However we only want to look for JLO types in user code for Java stub code generation @@ -197,6 +197,7 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) Log.LogDebugMessage ($"Skipping Java Stub Generation for {asm.ItemSpec}"); continue; } + res.Load (MonoAndroidHelper.GetTargetArch (asm), asm.ItemSpec); MaybeAddAbiSpecifcAssembly (asm, Path.GetFileName (asm.ItemSpec)); if (!allTypemapAssemblies.ContainsKey (asm.ItemSpec)) { allTypemapAssemblies.Add (asm.ItemSpec, asm); @@ -421,7 +422,7 @@ void MaybeAddAbiSpecifcAssembly (ITaskItem assembly, string fileName) } } - AssemblyDefinition LoadAssembly (string path, DirectoryAssemblyResolver? resolver = null) + AssemblyDefinition LoadAssembly (string path, XAAssemblyResolver? resolver = null) { string pdbPath = Path.ChangeExtension (path, ".pdb"); var readerParameters = new ReaderParameters { @@ -594,7 +595,7 @@ void WriteTypeMappings (List types, TypeDefinitionCache cache) /// information is required by to be available for each /// /// - Dictionary AddMethodsFromAbiSpecificAssemblies (MarshalMethodsClassifier classifier, DirectoryAssemblyResolver resolver, Dictionary> abiSpecificAssemblies) + Dictionary AddMethodsFromAbiSpecificAssemblies (MarshalMethodsClassifier classifier, XAAssemblyResolver resolver, Dictionary> abiSpecificAssemblies) { IDictionary> marshalMethods = classifier.MarshalMethods; ICollection assemblies = classifier.Assemblies; @@ -653,7 +654,7 @@ List FindMarshalMethodsForAssembly (IDictionary assemblyMarshalMethods, DirectoryAssemblyResolver resolver, List newAssemblies, Dictionary assemblyPaths) + void FindMatchingMethodsInAssembly (ITaskItem assemblyItem, MarshalMethodsClassifier classifier, List assemblyMarshalMethods, XAAssemblyResolver resolver, List newAssemblies, Dictionary assemblyPaths) { AssemblyDefinition asm = LoadAssembly (assemblyItem.ItemSpec, resolver); newAssemblies.Add (asm); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs index 8fe24c3b4ff..8a1b8996794 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs @@ -83,7 +83,7 @@ internal class ManifestDocument public string ApplicationLabel { get; set; } public string [] Placeholders { get; set; } public List Assemblies { get; set; } - public DirectoryAssemblyResolver Resolver { get; set; } + public IAssemblyResolver Resolver { get; set; } public string SdkDir { get; set; } public string TargetSdkVersion { get; set; } public string MinSdkVersion { get; set; } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs index 9869e66fed0..96063bf2126 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs @@ -35,7 +35,7 @@ public MarshalMethodsAssemblyRewriter (IDictionary> marshalMethods; HashSet assemblies; TaskLoggingHelper log; @@ -237,7 +237,7 @@ public bool Matches (MethodDefinition method) public ulong RejectedMethodCount => rejectedMethodCount; public ulong WrappedMethodCount => wrappedMethodCount; - public MarshalMethodsClassifier (TypeDefinitionCache tdCache, DirectoryAssemblyResolver res, TaskLoggingHelper log) + public MarshalMethodsClassifier (TypeDefinitionCache tdCache, XAAssemblyResolver res, TaskLoggingHelper log) { this.log = log ?? throw new ArgumentNullException (nameof (log)); this.tdCache = tdCache ?? throw new ArgumentNullException (nameof (tdCache)); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs index 8579f37a8c0..67f43a92353 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs @@ -536,6 +536,26 @@ public static string GetRelativePathForAndroidAsset (string assetsDirectory, ITa return path; } + public static AndroidTargetArch AbiToTargetArch (string abi) + { + switch (abi) { + case "arm64-v8a": + return AndroidTargetArch.Arm64; + + case "armeabi-v7a": + return AndroidTargetArch.Arm; + + case "x86": + return AndroidTargetArch.X86; + + case "x86_64": + return AndroidTargetArch.X86_64; + + default: + throw new InvalidOperationException ($"Internal error: unsupported ABI '{abi}'"); + } + } + public static string? CultureInvariantToString (object? obj) { if (obj == null) { @@ -561,5 +581,21 @@ public static int ConvertSupportedOSPlatformVersionToApiLevel (string version) } return apiLevel; } + + public static AndroidTargetArch GetTargetArch (ITaskItem asmItem) + { + string? abi = asmItem.GetMetadata ("Abi"); + if (String.IsNullOrEmpty (abi)) { + return AndroidTargetArch.None; + } + + return abi switch { + "armeabi-v7a" => AndroidTargetArch.Arm, + "arm64-v8a" => AndroidTargetArch.Arm64, + "x86" => AndroidTargetArch.X86, + "x86_64" => AndroidTargetArch.X86_64, + _ => throw new NotSupportedException ($"Unsupported ABI '{abi}' for assembly {asmItem.ItemSpec}") + }; + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/XAAssemblyResolver.cs b/src/Xamarin.Android.Build.Tasks/Utilities/XAAssemblyResolver.cs new file mode 100644 index 00000000000..fb6bdf5c11f --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/XAAssemblyResolver.cs @@ -0,0 +1,333 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.MemoryMappedFiles; + +using Microsoft.Build.Utilities; +using Mono.Cecil; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +class XAAssemblyResolver : IAssemblyResolver +{ + sealed class CacheEntry : IDisposable + { + bool disposed; + + Dictionary assemblies; + TaskLoggingHelper log; + AndroidTargetArch defaultArch; + + /// + /// This field is to be used by the `Resolve` overloads which don't have a way of indicating the desired ABI target for the assembly, but only when the + /// `AndroidTargetArch.None` entry for the assembly in question is **absent**. The field is always set to some value: either the very first assembly added + /// or the one with the `AndroidTargetArch.None` ABI. The latter always wins. + /// + public AssemblyDefinition Default { get; private set; } + public Dictionary Assemblies => assemblies; + + public CacheEntry (TaskLoggingHelper log, string filePath, AssemblyDefinition asm, AndroidTargetArch arch) + { + if (asm == null) { + throw new ArgumentNullException (nameof (asm)); + } + + this.log = log; + Default = asm; + defaultArch = arch; + assemblies = new Dictionary { + { arch, asm }, + }; + } + + public void Add (AndroidTargetArch arch, AssemblyDefinition asm) + { + if (asm == null) { + throw new ArgumentNullException (nameof (asm)); + } + + if (assemblies.ContainsKey (arch)) { + log.LogWarning ($"Entry for assembly '{asm}', architecture '{arch}' already exists. Replacing the old entry."); + } + + assemblies[arch] = asm; + if (arch == AndroidTargetArch.None && defaultArch != AndroidTargetArch.None) { + Default = asm; + defaultArch = arch; + } + } + + void Dispose (bool disposing) + { + if (disposed || !disposing) { + return; + } + + Default = null; + foreach (var kvp in assemblies) { + kvp.Value?.Dispose (); + } + assemblies.Clear (); + disposed = true; + } + + public void Dispose () + { + Dispose (disposing: true); + GC.SuppressFinalize (this); + } + } + + /// + /// Contains a collection of directories where framework assemblies can be found. This collection **must not** + /// contain any directories which contain ABI-specific assemblies. For those, use + /// + public ICollection FrameworkSearchDirectories { get; } = new List (); + + /// + /// Contains a collection of directories where Xamarin.Android (via linker, for instance) has placed the ABI + /// specific assemblies. Each ABI has its own set of directories to search. + /// + public IDictionary> AbiSearchDirectories { get; } = new Dictionary> (); + + readonly List viewStreams = new List (); + bool disposed; + TaskLoggingHelper log; + bool loadDebugSymbols; + ReaderParameters readerParameters; + readonly Dictionary cache; + + public XAAssemblyResolver (TaskLoggingHelper log, bool loadDebugSymbols, ReaderParameters? loadReaderParameters = null) + { + this.log = log; + this.loadDebugSymbols = loadDebugSymbols; + this.readerParameters = loadReaderParameters ?? new ReaderParameters(); + + cache = new Dictionary (StringComparer.OrdinalIgnoreCase); + } + + public AssemblyDefinition Resolve (string fullName, ReaderParameters? parameters = null) + { + return Resolve (AssemblyNameReference.Parse (fullName), parameters); + } + + public AssemblyDefinition Resolve (AssemblyNameReference name) + { + return Resolve (name, null); + } + + public AssemblyDefinition Resolve (AssemblyNameReference name, ReaderParameters? parameters) + { + return Resolve (AndroidTargetArch.None, name, parameters); + } + + public AssemblyDefinition Resolve (AndroidTargetArch arch, AssemblyNameReference name, ReaderParameters? parameters = null) + { + string shortName = name.Name; + if (cache.TryGetValue (shortName, out CacheEntry? entry)) { + return SelectAssembly (arch, name.FullName, entry, loading: false); + } + + if (arch == AndroidTargetArch.None) { + return FindAndLoadFromDirectories (arch, FrameworkSearchDirectories, name, parameters); + } + + if (!AbiSearchDirectories.TryGetValue (arch, out ICollection? directories) || directories == null) { + throw CreateLoadException (name); + } + + return FindAndLoadFromDirectories (arch, directories, name, parameters); + } + + AssemblyDefinition FindAndLoadFromDirectories (AndroidTargetArch arch, ICollection directories, AssemblyNameReference name, ReaderParameters? parameters) + { + string? assemblyFile; + AssemblyDefinition? candidate = null; + foreach (string dir in directories) { + if ((assemblyFile = SearchDirectory (name.Name, dir)) != null) { + AssemblyDefinition? loaded = Load (arch, assemblyFile); + if (Array.Equals (loaded?.Name.MetadataToken, name.MetadataToken)) { + return loaded; + } + candidate = candidate ?? loaded; + } + } + + if (candidate != null) + return candidate; + + throw CreateLoadException (name); + } + + static FileNotFoundException CreateLoadException (AssemblyNameReference name) + { + return new FileNotFoundException ($"Could not load assembly '{name}'."); + } + + static string? SearchDirectory (string name, string directory) + { + if (Path.IsPathRooted (name) && File.Exists (name)) { + return name; + } + + var file = Path.Combine (directory, $"{name}.dll"); + if (File.Exists (file)) { + return file; + } + + return null; + } + + public virtual AssemblyDefinition? Load (AndroidTargetArch arch, string filePath, bool forceLoad = false) + { + string name = Path.GetFileNameWithoutExtension (filePath); + AssemblyDefinition? assembly; + if (!forceLoad && cache.TryGetValue (name, out CacheEntry? entry)) { + assembly = SelectAssembly (arch, name, entry, loading: true); + if (assembly != null) { + return assembly; + } + } + + try { + assembly = ReadAssembly (filePath); + } catch (Exception e) when (e is FileNotFoundException || e is DirectoryNotFoundException) { + // These are ok, we can return null + return null; + } + + if (!cache.TryGetValue (name, out entry)) { + entry = new CacheEntry (log, filePath, assembly, arch); + cache.Add (name, entry); + } else { + entry.Add (arch, assembly); + } + + return assembly; + } + + AssemblyDefinition ReadAssembly (string filePath) + { + bool haveDebugSymbols = loadDebugSymbols && File.Exists (Path.ChangeExtension (filePath, ".pdb")); + var loadReaderParams = new ReaderParameters () { + ApplyWindowsRuntimeProjections = readerParameters.ApplyWindowsRuntimeProjections, + AssemblyResolver = this, + MetadataImporterProvider = readerParameters.MetadataImporterProvider, + InMemory = readerParameters.InMemory, + MetadataResolver = readerParameters.MetadataResolver, + ReadingMode = readerParameters.ReadingMode, + ReadSymbols = haveDebugSymbols, + ReadWrite = readerParameters.ReadWrite, + ReflectionImporterProvider = readerParameters.ReflectionImporterProvider, + SymbolReaderProvider = readerParameters.SymbolReaderProvider, + SymbolStream = readerParameters.SymbolStream, + }; + try { + return LoadFromMemoryMappedFile (filePath, loadReaderParams); + } catch (Exception ex) { + log.LogWarning ($"Failed to read '{filePath}' with debugging symbols. Retrying to load it without it. Error details are logged below."); + log.LogWarning ($"{ex.ToString ()}"); + loadReaderParams.ReadSymbols = false; + return LoadFromMemoryMappedFile (filePath, loadReaderParams); + } + } + + AssemblyDefinition LoadFromMemoryMappedFile (string file, ReaderParameters options) + { + // We can't use MemoryMappedFile when ReadWrite is true + if (options.ReadWrite) { + return AssemblyDefinition.ReadAssembly (file, options); + } + + bool origReadSymbols = options.ReadSymbols; + MemoryMappedViewStream? viewStream = null; + try { + // We must disable reading of symbols, even if they were present, because Cecil is unable to find the symbols file when + // assembly file name is unknown, and this is precisely the case when reading module from a stream. + // Until this issue is resolved, skipping symbol read saves time because reading exception isn't thrown and we don't + // retry the load. + options.ReadSymbols = false; + + // Create stream because CreateFromFile(string, ...) uses FileShare.None which is too strict + using var fileStream = new FileStream (file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, false); + using var mappedFile = MemoryMappedFile.CreateFromFile (fileStream, null, fileStream.Length, MemoryMappedFileAccess.Read, HandleInheritability.None, true); + viewStream = mappedFile.CreateViewStream (0, 0, MemoryMappedFileAccess.Read); + + AssemblyDefinition result = ModuleDefinition.ReadModule (viewStream, options).Assembly; + viewStreams.Add (viewStream); + + // We transferred the ownership of the viewStream to the collection. + viewStream = null; + + return result; + } finally { + options.ReadSymbols = origReadSymbols; + viewStream?.Dispose (); + } + } + + AssemblyDefinition? SelectAssembly (AndroidTargetArch arch, string assemblyName, CacheEntry? entry, bool loading) + { + if (entry == null) { + // Should "never" happen... + throw new ArgumentNullException (nameof (entry)); + } + + if (arch == AndroidTargetArch.None) { + // Disabled for now, generates too much noise. + // if (entry.Assemblies.Count > 1) { + // log.LogWarning ($"Architecture-agnostic entry requested for architecture-specific assembly '{assemblyName}'"); + // } + return entry.Default; + } + + if (!entry.Assemblies.TryGetValue (arch, out AssemblyDefinition? asm)) { + if (loading) { + return null; + } + + if (!entry.Assemblies.TryGetValue (AndroidTargetArch.None, out asm)) { + throw new InvalidOperationException ($"Internal error: assembly '{assemblyName}' for architecture '{arch}' not found in cache entry and architecture-agnostic entry is missing as well"); + } + + if (asm == null) { + throw new InvalidOperationException ($"Internal error: architecture-agnostic cache entry for assembly '{assemblyName}' is null"); + } + + log.LogWarning ($"Returning architecture-agnostic cache entry for assembly '{assemblyName}'. Requested architecture was: {arch}"); + return asm; + } + + if (asm == null) { + throw new InvalidOperationException ($"Internal error: null reference for assembly '{assemblyName}' in assembly cache entry"); + } + + return asm; + } + + public void Dispose () + { + Dispose (disposing: true); + GC.SuppressFinalize (this); + } + + protected virtual void Dispose (bool disposing) + { + if (disposed || !disposing) { + return; + } + + foreach (var kvp in cache) { + kvp.Value?.Dispose (); + } + cache.Clear (); + + foreach (MemoryMappedViewStream viewStream in viewStreams) { + viewStream.Dispose (); + } + viewStreams.Clear (); + + disposed = true; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs index 539099d2908..5f965ab46ff 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs @@ -53,12 +53,12 @@ public XAJavaTypeScanner (TaskLoggingHelper log, TypeDefinitionCache cache) this.cache = cache; } - public List GetJavaTypes (ICollection inputAssemblies, DirectoryAssemblyResolver resolver) + public List GetJavaTypes (ICollection inputAssemblies, XAAssemblyResolver resolver) { var types = new Dictionary (StringComparer.Ordinal); foreach (ITaskItem asmItem in inputAssemblies) { - AndroidTargetArch arch = GetTargetArch (asmItem); - AssemblyDefinition asmdef = resolver.GetAssembly (asmItem.ItemSpec); + AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (asmItem); + AssemblyDefinition asmdef = resolver.Load (arch, asmItem.ItemSpec); foreach (ModuleDefinition md in asmdef.Modules) { foreach (TypeDefinition td in md.Types) { @@ -117,20 +117,4 @@ void AddJavaType (TypeDefinition type, Dictionary types, Andro AddJavaType (nested, types, arch); } } - - AndroidTargetArch GetTargetArch (ITaskItem asmItem) - { - string? abi = asmItem.GetMetadata ("Abi"); - if (String.IsNullOrEmpty (abi)) { - return AndroidTargetArch.None; - } - - return abi switch { - "armeabi-v7a" => AndroidTargetArch.Arm, - "arm64-v8a" => AndroidTargetArch.Arm64, - "x86" => AndroidTargetArch.X86, - "x86_64" => AndroidTargetArch.X86_64, - _ => throw new NotSupportedException ($"Unsupported ABI '{abi}' for assembly {asmItem.ItemSpec}") - }; - } } From 9bba7fccf85849d8f0146771222ee8062c721506 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 11 Jul 2023 13:30:01 +0200 Subject: [PATCH 21/71] Attribute gets added properly... ...but for now no idea how to get the RID --- .../AddRidMetadataAttributeStep.cs | 86 +++++++++++++++++++ .../Microsoft.Android.Sdk.ILLink.targets | 5 ++ 2 files changed, 91 insertions(+) create mode 100644 src/Xamarin.Android.Build.Tasks/Linker/MonoDroid.Tuner/AddRidMetadataAttributeStep.cs diff --git a/src/Xamarin.Android.Build.Tasks/Linker/MonoDroid.Tuner/AddRidMetadataAttributeStep.cs b/src/Xamarin.Android.Build.Tasks/Linker/MonoDroid.Tuner/AddRidMetadataAttributeStep.cs new file mode 100644 index 00000000000..fd8aedeed99 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Linker/MonoDroid.Tuner/AddRidMetadataAttributeStep.cs @@ -0,0 +1,86 @@ +using System; + +using Mono.Cecil; +using Mono.Linker; +using Mono.Linker.Steps; + +namespace MonoDroid.Tuner; + +public class AddRidMetadataAttributeStep : BaseStep +{ + protected override void ProcessAssembly (AssemblyDefinition assembly) + { + if (!Annotations.HasAction (assembly)) { + return; + } + + var action = Annotations.GetAction (assembly); + if (action == AssemblyAction.Skip || action == AssemblyAction.Delete) { + return; + } + + AssemblyDefinition corlib = GetCorlib (); + MethodDefinition assemblyMetadataAttributeCtor = FindAssemblyMetadataAttributeCtor (corlib); + TypeDefinition systemString = GetSystemString (corlib); + + var attr = new CustomAttribute (assembly.MainModule.ImportReference (assemblyMetadataAttributeCtor)); + attr.ConstructorArguments.Add (new CustomAttributeArgument (systemString, "XamarinAndroidAbi")); // key + + // TODO: figure out how to get the RID... + attr.ConstructorArguments.Add (new CustomAttributeArgument (systemString, assembly.MainModule.FileName)); // value + + assembly.CustomAttributes.Add (attr); + + if (action == AssemblyAction.Copy) { + Annotations.SetAction (assembly, AssemblyAction.Save); + } + } + + TypeDefinition GetSystemString (AssemblyDefinition asm) => FindType (asm, "System.String", required: true); + + AssemblyDefinition GetCorlib () + { + const string ImportAssembly = "System.Private.CoreLib"; + AssemblyDefinition? asm = Context.Resolve (AssemblyNameReference.Parse (ImportAssembly)); + if (asm == null) { + throw new InvalidOperationException ($"Unable to import assembly '{ImportAssembly}'"); + } + + return asm; + } + + MethodDefinition FindAssemblyMetadataAttributeCtor (AssemblyDefinition asm) + { + const string AttributeType = "System.Reflection.AssemblyMetadataAttribute"; + + TypeDefinition assemblyMetadataAttribute = FindType (asm, AttributeType, required: true); + foreach (MethodDefinition md in assemblyMetadataAttribute!.Methods) { + if (!md.IsConstructor) { + continue; + } + + return md; + } + + throw new InvalidOperationException ($"Unable to find the {AttributeType} type constructor"); + } + + TypeDefinition? FindType (AssemblyDefinition asm, string typeName, bool required) + { + foreach (ModuleDefinition md in asm.Modules) { + foreach (TypeDefinition et in md.Types) { + if (String.Compare (typeName, et.FullName, StringComparison.Ordinal) != 0) { + continue; + } + + return et; + } + } + + if (required) { + throw new InvalidOperationException ($"Internal error: required type '{typeName}' in assembly {asm} not found"); + } + + return null; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.ILLink.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.ILLink.targets index 7400de192ff..781845f0b9c 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.ILLink.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.ILLink.targets @@ -94,6 +94,11 @@ This file contains the .NET 5-specific targets to customize ILLink BeforeStep="MarkStep" Type="MonoDroid.Tuner.FixLegacyResourceDesignerStep" /> + <_TrimmerCustomSteps + Include="$(_AndroidLinkerCustomStepAssembly)" + AfterStep="CleanStep" + Type="MonoDroid.Tuner.AddRidMetadataAttributeStep" + /> <_PreserveLists Include="$(MSBuildThisFileDirectory)..\PreserveLists\*.xml" /> Date: Tue, 11 Jul 2023 22:36:40 +0200 Subject: [PATCH 22/71] Metadata set to the appropriate RID --- .../AddRidMetadataAttributeStep.cs | 16 +++++++++++++++- .../targets/Microsoft.Android.Sdk.ILLink.targets | 1 + 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/Xamarin.Android.Build.Tasks/Linker/MonoDroid.Tuner/AddRidMetadataAttributeStep.cs b/src/Xamarin.Android.Build.Tasks/Linker/MonoDroid.Tuner/AddRidMetadataAttributeStep.cs index fd8aedeed99..6ee4fe3d2e7 100644 --- a/src/Xamarin.Android.Build.Tasks/Linker/MonoDroid.Tuner/AddRidMetadataAttributeStep.cs +++ b/src/Xamarin.Android.Build.Tasks/Linker/MonoDroid.Tuner/AddRidMetadataAttributeStep.cs @@ -4,6 +4,10 @@ using Mono.Linker; using Mono.Linker.Steps; +#if ILLINK +using Microsoft.Android.Sdk.ILLink; +#endif + namespace MonoDroid.Tuner; public class AddRidMetadataAttributeStep : BaseStep @@ -19,6 +23,16 @@ protected override void ProcessAssembly (AssemblyDefinition assembly) return; } + string? rid = null; +#if ILLINK + if (!Context.TryGetCustomData ("XARuntimeIdentifier", out rid)) { + throw new InvalidOperationException ("Missing XARuntimeIdentifier custom data"); + } +#endif + if (String.IsNullOrEmpty (rid)) { + throw new InvalidOperationException ("RID must have a non-empty value"); + } + AssemblyDefinition corlib = GetCorlib (); MethodDefinition assemblyMetadataAttributeCtor = FindAssemblyMetadataAttributeCtor (corlib); TypeDefinition systemString = GetSystemString (corlib); @@ -27,7 +41,7 @@ protected override void ProcessAssembly (AssemblyDefinition assembly) attr.ConstructorArguments.Add (new CustomAttributeArgument (systemString, "XamarinAndroidAbi")); // key // TODO: figure out how to get the RID... - attr.ConstructorArguments.Add (new CustomAttributeArgument (systemString, assembly.MainModule.FileName)); // value + attr.ConstructorArguments.Add (new CustomAttributeArgument (systemString, rid)); // value assembly.CustomAttributes.Add (attr); diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.ILLink.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.ILLink.targets index 781845f0b9c..e63cad77f37 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.ILLink.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.ILLink.targets @@ -40,6 +40,7 @@ This file contains the .NET 5-specific targets to customize ILLink https://github.com/dotnet/sdk/blob/a5393731b5b7b225692fff121f747fbbc9e8b140/src/Tasks/Microsoft.NET.Build.Tasks/targets/Microsoft.NET.ILLink.targets#L147 --> <_TrimmerCustomData Include="XATargetFrameworkDirectories" Value="$(_XATargetFrameworkDirectories)" /> + <_TrimmerCustomData Include="XARuntimeIdentifier" Value="$(RuntimeIdentifier)" /> <_TrimmerCustomData Condition=" '$(_ProguardProjectConfiguration)' != '' " Include="ProguardConfiguration" From d5ca586b7e630505c07fcc83f7bf819b637d2497 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 12 Jul 2023 11:17:59 +0200 Subject: [PATCH 23/71] Try to fix link failures --- .../PreserveLists/System.Private.CoreLib.xml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 src/Microsoft.Android.Sdk.ILLink/PreserveLists/System.Private.CoreLib.xml diff --git a/src/Microsoft.Android.Sdk.ILLink/PreserveLists/System.Private.CoreLib.xml b/src/Microsoft.Android.Sdk.ILLink/PreserveLists/System.Private.CoreLib.xml new file mode 100644 index 00000000000..c5aea679db6 --- /dev/null +++ b/src/Microsoft.Android.Sdk.ILLink/PreserveLists/System.Private.CoreLib.xml @@ -0,0 +1,6 @@ + + + + + + From a90047a7f09e6c58c267dab1ced3cecfaad57681 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 12 Jul 2023 14:13:05 +0000 Subject: [PATCH 24/71] Update src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.ILLink.targets Co-authored-by: Jonathan Peppers --- .../targets/Microsoft.Android.Sdk.ILLink.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.ILLink.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.ILLink.targets index e63cad77f37..25018bc0b40 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.ILLink.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.ILLink.targets @@ -40,7 +40,7 @@ This file contains the .NET 5-specific targets to customize ILLink https://github.com/dotnet/sdk/blob/a5393731b5b7b225692fff121f747fbbc9e8b140/src/Tasks/Microsoft.NET.Build.Tasks/targets/Microsoft.NET.ILLink.targets#L147 --> <_TrimmerCustomData Include="XATargetFrameworkDirectories" Value="$(_XATargetFrameworkDirectories)" /> - <_TrimmerCustomData Include="XARuntimeIdentifier" Value="$(RuntimeIdentifier)" /> + <_TrimmerCustomData Include="XARuntimeIdentifier" Value="$(RuntimeIdentifier)" Condition=" '$(_AndroidAddRuntimeIdentifierToAssemblies)' == 'true' " /> <_TrimmerCustomData Condition=" '$(_ProguardProjectConfiguration)' != '' " Include="ProguardConfiguration" From bbcf701436608b9e5e9ef017c108d0e7617e8917 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 12 Jul 2023 16:37:36 +0200 Subject: [PATCH 25/71] Make the step itself conditional, too --- .../targets/Microsoft.Android.Sdk.ILLink.targets | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.ILLink.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.ILLink.targets index 25018bc0b40..0461633d57b 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.ILLink.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.ILLink.targets @@ -96,6 +96,7 @@ This file contains the .NET 5-specific targets to customize ILLink Type="MonoDroid.Tuner.FixLegacyResourceDesignerStep" /> <_TrimmerCustomSteps + Condition=" '$(_AndroidAddRuntimeIdentifierToAssemblies)' == 'true' " Include="$(_AndroidLinkerCustomStepAssembly)" AfterStep="CleanStep" Type="MonoDroid.Tuner.AddRidMetadataAttributeStep" From 46865e3ccfbb6ed8f604c73f9f96e2e07a68d69a Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 12 Jul 2023 22:43:38 +0200 Subject: [PATCH 26/71] Address comments --- .../Utilities/MonoAndroidHelper.cs | 31 +++--------- .../Utilities/XAAssemblyResolver.cs | 49 ++++++++----------- 2 files changed, 29 insertions(+), 51 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs index 67f43a92353..7e27360dd96 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs @@ -538,22 +538,13 @@ public static string GetRelativePathForAndroidAsset (string assetsDirectory, ITa public static AndroidTargetArch AbiToTargetArch (string abi) { - switch (abi) { - case "arm64-v8a": - return AndroidTargetArch.Arm64; - - case "armeabi-v7a": - return AndroidTargetArch.Arm; - - case "x86": - return AndroidTargetArch.X86; - - case "x86_64": - return AndroidTargetArch.X86_64; - - default: - throw new InvalidOperationException ($"Internal error: unsupported ABI '{abi}'"); - } + return abi switch { + "armeabi-v7a" => AndroidTargetArch.Arm, + "arm64-v8a" => AndroidTargetArch.Arm64, + "x86_64" => AndroidTargetArch.X86_64, + "x86" => AndroidTargetArch.X86, + _ => throw new NotSupportedException ($"Internal error: unsupported ABI '{abi}'") + }; } public static string? CultureInvariantToString (object? obj) @@ -589,13 +580,7 @@ public static AndroidTargetArch GetTargetArch (ITaskItem asmItem) return AndroidTargetArch.None; } - return abi switch { - "armeabi-v7a" => AndroidTargetArch.Arm, - "arm64-v8a" => AndroidTargetArch.Arm64, - "x86" => AndroidTargetArch.X86, - "x86_64" => AndroidTargetArch.X86_64, - _ => throw new NotSupportedException ($"Unsupported ABI '{abi}' for assembly {asmItem.ItemSpec}") - }; + return AbiToTargetArch (abi); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/XAAssemblyResolver.cs b/src/Xamarin.Android.Build.Tasks/Utilities/XAAssemblyResolver.cs index fb6bdf5c11f..746f45802e7 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/XAAssemblyResolver.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/XAAssemblyResolver.cs @@ -107,22 +107,22 @@ public XAAssemblyResolver (TaskLoggingHelper log, bool loadDebugSymbols, ReaderP cache = new Dictionary (StringComparer.OrdinalIgnoreCase); } - public AssemblyDefinition Resolve (string fullName, ReaderParameters? parameters = null) + public AssemblyDefinition? Resolve (string fullName, ReaderParameters? parameters = null) { return Resolve (AssemblyNameReference.Parse (fullName), parameters); } - public AssemblyDefinition Resolve (AssemblyNameReference name) + public AssemblyDefinition? Resolve (AssemblyNameReference name) { return Resolve (name, null); } - public AssemblyDefinition Resolve (AssemblyNameReference name, ReaderParameters? parameters) + public AssemblyDefinition? Resolve (AssemblyNameReference name, ReaderParameters? parameters) { return Resolve (AndroidTargetArch.None, name, parameters); } - public AssemblyDefinition Resolve (AndroidTargetArch arch, AssemblyNameReference name, ReaderParameters? parameters = null) + public AssemblyDefinition? Resolve (AndroidTargetArch arch, AssemblyNameReference name, ReaderParameters? parameters = null) { string shortName = name.Name; if (cache.TryGetValue (shortName, out CacheEntry? entry)) { @@ -140,24 +140,16 @@ public AssemblyDefinition Resolve (AndroidTargetArch arch, AssemblyNameReference return FindAndLoadFromDirectories (arch, directories, name, parameters); } - AssemblyDefinition FindAndLoadFromDirectories (AndroidTargetArch arch, ICollection directories, AssemblyNameReference name, ReaderParameters? parameters) + AssemblyDefinition? FindAndLoadFromDirectories (AndroidTargetArch arch, ICollection directories, AssemblyNameReference name, ReaderParameters? parameters) { string? assemblyFile; - AssemblyDefinition? candidate = null; foreach (string dir in directories) { if ((assemblyFile = SearchDirectory (name.Name, dir)) != null) { - AssemblyDefinition? loaded = Load (arch, assemblyFile); - if (Array.Equals (loaded?.Name.MetadataToken, name.MetadataToken)) { - return loaded; - } - candidate = candidate ?? loaded; + return Load (arch, assemblyFile, parameters); } } - if (candidate != null) - return candidate; - - throw CreateLoadException (name); + return null; } static FileNotFoundException CreateLoadException (AssemblyNameReference name) @@ -179,11 +171,11 @@ static FileNotFoundException CreateLoadException (AssemblyNameReference name) return null; } - public virtual AssemblyDefinition? Load (AndroidTargetArch arch, string filePath, bool forceLoad = false) + public virtual AssemblyDefinition? Load (AndroidTargetArch arch, string filePath, ReaderParameters? readerParameters = null) { string name = Path.GetFileNameWithoutExtension (filePath); AssemblyDefinition? assembly; - if (!forceLoad && cache.TryGetValue (name, out CacheEntry? entry)) { + if (cache.TryGetValue (name, out CacheEntry? entry)) { assembly = SelectAssembly (arch, name, entry, loading: true); if (assembly != null) { return assembly; @@ -191,7 +183,7 @@ static FileNotFoundException CreateLoadException (AssemblyNameReference name) } try { - assembly = ReadAssembly (filePath); + assembly = ReadAssembly (filePath, readerParameters); } catch (Exception e) when (e is FileNotFoundException || e is DirectoryNotFoundException) { // These are ok, we can return null return null; @@ -207,21 +199,22 @@ static FileNotFoundException CreateLoadException (AssemblyNameReference name) return assembly; } - AssemblyDefinition ReadAssembly (string filePath) + AssemblyDefinition ReadAssembly (string filePath, ReaderParameters? readerParametersOverride = null) { + ReaderParameters templateParameters = readerParametersOverride ?? this.readerParameters; bool haveDebugSymbols = loadDebugSymbols && File.Exists (Path.ChangeExtension (filePath, ".pdb")); var loadReaderParams = new ReaderParameters () { - ApplyWindowsRuntimeProjections = readerParameters.ApplyWindowsRuntimeProjections, + ApplyWindowsRuntimeProjections = templateParameters.ApplyWindowsRuntimeProjections, AssemblyResolver = this, - MetadataImporterProvider = readerParameters.MetadataImporterProvider, - InMemory = readerParameters.InMemory, - MetadataResolver = readerParameters.MetadataResolver, - ReadingMode = readerParameters.ReadingMode, + MetadataImporterProvider = templateParameters.MetadataImporterProvider, + InMemory = templateParameters.InMemory, + MetadataResolver = templateParameters.MetadataResolver, + ReadingMode = templateParameters.ReadingMode, ReadSymbols = haveDebugSymbols, - ReadWrite = readerParameters.ReadWrite, - ReflectionImporterProvider = readerParameters.ReflectionImporterProvider, - SymbolReaderProvider = readerParameters.SymbolReaderProvider, - SymbolStream = readerParameters.SymbolStream, + ReadWrite = templateParameters.ReadWrite, + ReflectionImporterProvider = templateParameters.ReflectionImporterProvider, + SymbolReaderProvider = templateParameters.SymbolReaderProvider, + SymbolStream = templateParameters.SymbolStream, }; try { return LoadFromMemoryMappedFile (filePath, loadReaderParams); From c909e4165f736e25a1ad82f5975a0c7488d10304 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 12 Jul 2023 23:04:22 +0200 Subject: [PATCH 27/71] WIP, the very beginning --- .../Tasks/GenerateJavaStubs.cs | 364 +++++++++++++++--- .../Utilities/IInputAssemblySet.cs | 14 + .../Utilities/RidAgnosticeInputAssemblySet.cs | 36 ++ .../Utilities/RidSpecificInputAssemblySet.cs | 56 +++ 4 files changed, 416 insertions(+), 54 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/IInputAssemblySet.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/RidAgnosticeInputAssemblySet.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/RidSpecificInputAssemblySet.cs diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 80f1f25b279..f37b2c83172 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -27,6 +27,21 @@ namespace Xamarin.Android.Tasks public class GenerateJavaStubs : AndroidTask { + sealed class RunState + { + public XAAssemblyResolver Resolver { get; set; } + public ICollection JavaTypeAssemblies { get; set; } + public ICollection UserAssemblies { get; set; } + public InputAssemblySet AssemblySet { get; set; } + public bool UseMarshalMethods { get; set; } + + /// + /// If `true`, generate code/data that doesn't depend on a specific RID (e.g. ACW maps or JCWs) + /// To be used once per multi-RID runs. + /// + public bool GenerateRidAgnosticParts { get; set; } + } + public const string MarshalMethodsRegisterTaskKey = ".:!MarshalMethods!:."; public override string TaskPrefix => "GJS"; @@ -101,12 +116,7 @@ public class GenerateJavaStubs : AndroidTask public override bool RunTask () { try { - bool useMarshalMethods = !Debug && EnableMarshalMethods; - // We're going to do 3 steps here instead of separate tasks so - // we can share the list of JLO TypeDefinitions between them - using (XAAssemblyResolver res = MakeResolver (useMarshalMethods)) { - Run (res, useMarshalMethods); - } + Run (); } catch (XamarinAndroidException e) { Log.LogCodedError (string.Format ("XA{0:0000}", e.Code), e.MessageWithoutCode); if (MonoAndroidHelper.LogInternalExceptions) @@ -141,23 +151,74 @@ XAAssemblyResolver MakeResolver (bool useMarshalMethods) return res; } - void Run (XAAssemblyResolver res, bool useMarshalMethods) + void Run () { - PackageNamingPolicy pnp; - JavaNativeTypeManager.PackageNamingPolicy = Enum.TryParse (PackageNamingPolicy, out pnp) ? pnp : PackageNamingPolicyEnum.LowercaseCrc64; + if (Debug) { + if (LinkingEnabled) { + RunDebugWithLinking (); + } else { + RunDebugNoLinking (); + } + return; + } - Dictionary>? abiSpecificAssembliesByPath = null; - if (useMarshalMethods) { - abiSpecificAssembliesByPath = new Dictionary> (StringComparer.Ordinal); + bool useMarshalMethods = !Debug && EnableMarshalMethods; + if (LinkingEnabled) { + RunReleaseWithLinking (useMarshalMethods); + } else { + RunReleaseNoLinking (useMarshalMethods); } + } - // Put every assembly we'll need in the resolver + // We have one set of assemblies, no RID-specific ones. + // Typemaps don't use MVIDs or metadata tokens + void RunDebugNoLinking () + { + var assemblies = CollectInterestingAssemblies (); + throw new NotImplementedException (); + } + + // We have as many sets of assemblies as there are RIDs, all assemblies are RID-specific + // Typemaps don't use MVIDs or metadata tokens + void RunDebugWithLinking () + { + var assemblies = CollectInterestingAssemblies (); + throw new NotImplementedException (); + } + + // We have one set of assemblies, no RID-specific ones. + // Typemaps use MVIDs and metadata tokens + void RunReleaseNoLinking (bool useMarshalMethods) + { + var assemblies = CollectInterestingAssemblies (); + var state = new RunState { + UseMarshalMethods = useMarshalMethods, + AssemblySet = assemblies, + JavaTypeAssemblies = assemblies.JavaTypeAssemblies, + UserAssemblies = assemblies.UserAssemblies, + GenerateRidAgnosticParts = true, + }; + DoRun (state); + } + + // We have as many sets of assemblies as there are RIDs, all assemblies are RID-specific + // Typemaps use MVIDs and metadata tokens, need to process all per-RID assemblies separately + void RunReleaseWithLinking (bool useMarshalMethods) + { + var assemblies = CollectInterestingAssemblies (); + throw new NotImplementedException (); + } + + RidSpecificInputAssemblySet CollectInterestingAssembliesRidSpecific () => CollectInterestingAssemblies (); + RidAgnosticInputAssemblySet CollectInterestingAssembliesRidAgnostic () => CollectInterestingAssemblies (); + + T CollectInterestingAssemblies () where T: InputAssemblySet, new() + { + var assemblies = new T (); bool hasExportReference = false; bool haveMonoAndroid = false; - var allTypemapAssemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); - var userAssemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); - foreach (var assembly in ResolvedAssemblies) { + foreach (ITaskItem assembly in ResolvedAssemblies) { bool value; if (bool.TryParse (assembly.GetMetadata (AndroidSkipJavaStubGeneration), out value) && value) { Log.LogDebugMessage ($"Skipping Java Stub Generation for {assembly.ItemSpec}"); @@ -175,83 +236,163 @@ void Run (XAAssemblyResolver res, bool useMarshalMethods) } else if (MonoAndroidHelper.FrameworkAssembliesToTreatAsUserAssemblies.Contains (fileName)) { if (!bool.TryParse (assembly.GetMetadata (AndroidSkipJavaStubGeneration), out value) || !value) { string name = Path.GetFileNameWithoutExtension (fileName); - if (!userAssemblies.ContainsKey (name)) - userAssemblies.Add (name, assembly.ItemSpec); + assemblies.AddUserAssembly (assembly); addAssembly = true; } } if (addAssembly) { - MaybeAddAbiSpecifcAssembly (assembly, fileName); - if (!allTypemapAssemblies.ContainsKey (assembly.ItemSpec)) { - allTypemapAssemblies.Add (assembly.ItemSpec, assembly); - } + assemblies.AddJavaTypeAssembly (assembly); } - - res.Load (MonoAndroidHelper.GetTargetArch (assembly), assembly.ItemSpec); } // However we only want to look for JLO types in user code for Java stub code generation - foreach (var asm in ResolvedUserAssemblies) { + foreach (ITaskItem asm in ResolvedUserAssemblies) { if (bool.TryParse (asm.GetMetadata (AndroidSkipJavaStubGeneration), out bool value) && value) { Log.LogDebugMessage ($"Skipping Java Stub Generation for {asm.ItemSpec}"); continue; } - res.Load (MonoAndroidHelper.GetTargetArch (asm), asm.ItemSpec); - MaybeAddAbiSpecifcAssembly (asm, Path.GetFileName (asm.ItemSpec)); - if (!allTypemapAssemblies.ContainsKey (asm.ItemSpec)) { - allTypemapAssemblies.Add (asm.ItemSpec, asm); - } - - string name = Path.GetFileNameWithoutExtension (asm.ItemSpec); - if (!userAssemblies.ContainsKey (name)) - userAssemblies.Add (name, asm.ItemSpec); + assemblies.AddJavaTypeAssembly (asm); + assemblies.AddUserAssembly (asm); } + return assemblies; + } + + void DoRun (RunState state) + { + PackageNamingPolicy pnp; + JavaNativeTypeManager.PackageNamingPolicy = Enum.TryParse (PackageNamingPolicy, out pnp) ? pnp : PackageNamingPolicyEnum.LowercaseCrc64; // Step 1 - Find all the JLO types var cache = new TypeDefinitionCache (); var scanner = new XAJavaTypeScanner (Log, cache) { ErrorOnCustomJavaObject = ErrorOnCustomJavaObject, }; - List allJavaTypes = scanner.GetJavaTypes (allTypemapAssemblies.Values, res); + List allJavaTypes = scanner.GetJavaTypes (state.JavaTypeAssemblies, state.Resolver); var javaTypes = new List (); foreach (JavaType jt in allJavaTypes) { // Whem marshal methods are in use we do not want to skip non-user assemblies (such as Mono.Android) - we need to generate JCWs for them during // application build, unlike in Debug configuration or when marshal methods are disabled, in which case we use JCWs generated during Xamarin.Android // build and stored in a jar file. - if ((!useMarshalMethods && !userAssemblies.ContainsKey (jt.Type.Module.Assembly.Name.Name)) || JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (jt.Type, cache)) { + if ((!state.UseMarshalMethods && !state.AssemblySet.IsUserAssembly (jt.Type.Module.Assembly.Name.Name)) || JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (jt.Type, cache)) { continue; } javaTypes.Add (jt); } - MarshalMethodsClassifier classifier = null; - if (useMarshalMethods) { - classifier = new MarshalMethodsClassifier (cache, res, Log); + MarshalMethodsClassifier? classifier = null; + if (state.UseMarshalMethods) { + classifier = new MarshalMethodsClassifier (cache, state.Resolver, Log); } - // Step 2 - Generate Java stub code - var success = CreateJavaSources (javaTypes, cache, classifier, useMarshalMethods); - if (!success) - return; - - if (useMarshalMethods) { - // We need to parse the environment files supplied by the user to see if they want to use broken exception transitions. This information is needed - // in order to properly generate wrapper methods in the marshal methods assembly rewriter. - // We don't care about those generated by us, since they won't contain the `XA_BROKEN_EXCEPTION_TRANSITIONS` variable we look for. - var environmentParser = new EnvironmentFilesParser (); - - Dictionary assemblyPaths = AddMethodsFromAbiSpecificAssemblies (classifier, res, abiSpecificAssembliesByPath); - - var rewriter = new MarshalMethodsAssemblyRewriter (classifier.MarshalMethods, classifier.Assemblies, assemblyPaths, Log); - rewriter.Rewrite (res, environmentParser.AreBrokenExceptionTransitionsEnabled (Environments)); + if (state.GenerateRidAgnosticParts) { + // Step 2 - Generate Java stub code + bool success = CreateJavaSources (javaTypes, cache, classifier, state.UseMarshalMethods); + if (!success) { + return; // TODO: throw? Return `false`? + } } + // TODO: marshal methods rewriter here + + // TODO: this likely needs to be handled in a different manner. // Step 3 - Generate type maps // Type mappings need to use all the assemblies, always. WriteTypeMappings (allJavaTypes, cache); + if (state.GenerateRidAgnosticParts) { + WriteAcwMaps (javaTypes, cache); + + // Step 4 - Merge [Activity] and friends into AndroidManifest.xml + UpdateAndroidManifest (state, cache, allJavaTypes); + CreateAdditionalJavaSources (javaTypes, cache, classifier); + } + + if (state.UseMarshalMethods) { + classifier.AddSpecialCaseMethods (); + + Log.LogDebugMessage ($"Number of generated marshal methods: {classifier.MarshalMethods.Count}"); + + if (classifier.RejectedMethodCount > 0) { + Log.LogWarning ($"Number of methods in the project that will be registered dynamically: {classifier.RejectedMethodCount}"); + } + + if (classifier.WrappedMethodCount > 0) { + // TODO: change to LogWarning once the generator can output code which requires no non-blittable wrappers + Log.LogDebugMessage ($"Number of methods in the project that need marshal method wrappers: {classifier.WrappedMethodCount}"); + } + } + } + + void CreateAdditionalJavaSources (List javaTypes, TypeDefinitionCache cache, MarshalMethodsClassifier? classifier) + { + StringWriter regCallsWriter = new StringWriter (); + regCallsWriter.WriteLine ("\t\t// Application and Instrumentation ACWs must be registered first."); + foreach (JavaType jt in javaTypes) { + TypeDefinition type = jt.Type; + if (JavaNativeTypeManager.IsApplication (type, cache) || JavaNativeTypeManager.IsInstrumentation (type, cache)) { + if (classifier != null && !classifier.FoundDynamicallyRegisteredMethods (type)) { + continue; + } + + string javaKey = JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.'); + regCallsWriter.WriteLine ("\t\tmono.android.Runtime.register (\"{0}\", {1}.class, {1}.__md_methods);", + type.GetAssemblyQualifiedName (cache), javaKey); + } + } + regCallsWriter.Close (); + + var real_app_dir = Path.Combine (OutputDirectory, "src", "mono", "android", "app"); + string applicationTemplateFile = "ApplicationRegistration.java"; + SaveResource (applicationTemplateFile, applicationTemplateFile, real_app_dir, + template => template.Replace ("// REGISTER_APPLICATION_AND_INSTRUMENTATION_CLASSES_HERE", regCallsWriter.ToString ())); + } + + void UpdateAndroidManifest (RunState state, TypeDefinitionCache cache, List allJavaTypes) + { + var manifest = new ManifestDocument (ManifestTemplate) { + PackageName = PackageName, + VersionName = VersionName, + ApplicationLabel = ApplicationLabel ?? PackageName, + Placeholders = ManifestPlaceholders, + Resolver = state.Resolver, + SdkDir = AndroidSdkDir, + TargetSdkVersion = AndroidSdkPlatform, + MinSdkVersion = MonoAndroidHelper.ConvertSupportedOSPlatformVersionToApiLevel (SupportedOSPlatformVersion).ToString (), + Debug = Debug, + MultiDex = MultiDex, + NeedsInternet = NeedsInternet, + InstantRunEnabled = InstantRunEnabled + }; + // Only set manifest.VersionCode if there is no existing value in AndroidManifest.xml. + if (manifest.HasVersionCode) { + Log.LogDebugMessage ($"Using existing versionCode in: {ManifestTemplate}"); + } else if (!string.IsNullOrEmpty (VersionCode)) { + manifest.VersionCode = VersionCode; + } + + foreach (ITaskItem assembly in state.UserAssemblies) { + manifest.Assemblies.Add (Path.GetFileNameWithoutExtension (assembly.ItemSpec)); + } + + if (!String.IsNullOrWhiteSpace (CheckedBuild)) { + // We don't validate CheckedBuild value here, this will be done in BuildApk. We just know that if it's + // on then we need android:debuggable=true and android:extractNativeLibs=true + manifest.ForceDebuggable = true; + manifest.ForceExtractNativeLibs = true; + } + + var additionalProviders = manifest.Merge (Log, cache, allJavaTypes, ApplicationJavaClass, EmbedAssemblies, BundledWearApplicationName, MergedManifestDocuments); + + // Only write the new manifest if it actually changed + if (manifest.SaveIfChanged (Log, MergedAndroidManifestOutput)) { + Log.LogDebugMessage ($"Saving: {MergedAndroidManifestOutput}"); + } + } + + void WriteAcwMaps (List javaTypes, TypeDefinitionCache cache) + { // We need to save a map of .NET type -> ACW type for resource file fixups var managed = new Dictionary (javaTypes.Count, StringComparer.Ordinal); var java = new Dictionary (javaTypes.Count, StringComparer.Ordinal); @@ -259,6 +400,7 @@ void Run (XAAssemblyResolver res, bool useMarshalMethods) var managedConflicts = new Dictionary> (0, StringComparer.Ordinal); var javaConflicts = new Dictionary> (0, StringComparer.Ordinal); + bool success = true; using (var acw_map = MemoryStreamPool.Shared.CreateStreamWriter ()) { foreach (JavaType jt in javaTypes) { TypeDefinition type = jt.Type; @@ -319,6 +461,120 @@ void Run (XAAssemblyResolver res, bool useMarshalMethods) foreach (var typeName in kvp.Value) Log.LogCodedError ("XA4215", Properties.Resources.XA4215_Details, kvp.Key, typeName); } + } + + void Run (XAAssemblyResolver res, bool useMarshalMethods) + { + PackageNamingPolicy pnp; + JavaNativeTypeManager.PackageNamingPolicy = Enum.TryParse (PackageNamingPolicy, out pnp) ? pnp : PackageNamingPolicyEnum.LowercaseCrc64; + + Dictionary>? abiSpecificAssembliesByPath = null; + if (useMarshalMethods) { + abiSpecificAssembliesByPath = new Dictionary> (StringComparer.Ordinal); + } + + // Put every assembly we'll need in the resolver + bool hasExportReference = false; + bool haveMonoAndroid = false; + var allTypemapAssemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); + var userAssemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); + + foreach (var assembly in ResolvedAssemblies) { + bool value; + if (bool.TryParse (assembly.GetMetadata (AndroidSkipJavaStubGeneration), out value) && value) { + Log.LogDebugMessage ($"Skipping Java Stub Generation for {assembly.ItemSpec}"); + continue; + } + + bool addAssembly = false; + string fileName = Path.GetFileName (assembly.ItemSpec); + if (!hasExportReference && String.Compare ("Mono.Android.Export.dll", fileName, StringComparison.OrdinalIgnoreCase) == 0) { + hasExportReference = true; + addAssembly = true; + } else if (!haveMonoAndroid && String.Compare ("Mono.Android.dll", fileName, StringComparison.OrdinalIgnoreCase) == 0) { + haveMonoAndroid = true; + addAssembly = true; + } else if (MonoAndroidHelper.FrameworkAssembliesToTreatAsUserAssemblies.Contains (fileName)) { + if (!bool.TryParse (assembly.GetMetadata (AndroidSkipJavaStubGeneration), out value) || !value) { + string name = Path.GetFileNameWithoutExtension (fileName); + if (!userAssemblies.ContainsKey (name)) + userAssemblies.Add (name, assembly.ItemSpec); + addAssembly = true; + } + } + + if (addAssembly) { + MaybeAddAbiSpecifcAssembly (assembly, fileName); + if (!allTypemapAssemblies.ContainsKey (assembly.ItemSpec)) { + allTypemapAssemblies.Add (assembly.ItemSpec, assembly); + } + } + + res.Load (MonoAndroidHelper.GetTargetArch (assembly), assembly.ItemSpec); + } + + // However we only want to look for JLO types in user code for Java stub code generation + foreach (var asm in ResolvedUserAssemblies) { + if (bool.TryParse (asm.GetMetadata (AndroidSkipJavaStubGeneration), out bool value) && value) { + Log.LogDebugMessage ($"Skipping Java Stub Generation for {asm.ItemSpec}"); + continue; + } + res.Load (MonoAndroidHelper.GetTargetArch (asm), asm.ItemSpec); + MaybeAddAbiSpecifcAssembly (asm, Path.GetFileName (asm.ItemSpec)); + if (!allTypemapAssemblies.ContainsKey (asm.ItemSpec)) { + allTypemapAssemblies.Add (asm.ItemSpec, asm); + } + + string name = Path.GetFileNameWithoutExtension (asm.ItemSpec); + if (!userAssemblies.ContainsKey (name)) + userAssemblies.Add (name, asm.ItemSpec); + } + + // Step 1 - Find all the JLO types + var cache = new TypeDefinitionCache (); + var scanner = new XAJavaTypeScanner (Log, cache) { + ErrorOnCustomJavaObject = ErrorOnCustomJavaObject, + }; + List allJavaTypes = scanner.GetJavaTypes (allTypemapAssemblies.Values, res); + var javaTypes = new List (); + + foreach (JavaType jt in allJavaTypes) { + // Whem marshal methods are in use we do not want to skip non-user assemblies (such as Mono.Android) - we need to generate JCWs for them during + // application build, unlike in Debug configuration or when marshal methods are disabled, in which case we use JCWs generated during Xamarin.Android + // build and stored in a jar file. + if ((!useMarshalMethods && !userAssemblies.ContainsKey (jt.Type.Module.Assembly.Name.Name)) || JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (jt.Type, cache)) { + continue; + } + javaTypes.Add (jt); + } + + MarshalMethodsClassifier classifier = null; + if (useMarshalMethods) { + classifier = new MarshalMethodsClassifier (cache, res, Log); + } + + // Step 2 - Generate Java stub code + var success = CreateJavaSources (javaTypes, cache, classifier, useMarshalMethods); + if (!success) + return; + + if (useMarshalMethods) { + // We need to parse the environment files supplied by the user to see if they want to use broken exception transitions. This information is needed + // in order to properly generate wrapper methods in the marshal methods assembly rewriter. + // We don't care about those generated by us, since they won't contain the `XA_BROKEN_EXCEPTION_TRANSITIONS` variable we look for. + var environmentParser = new EnvironmentFilesParser (); + + Dictionary assemblyPaths = AddMethodsFromAbiSpecificAssemblies (classifier, res, abiSpecificAssembliesByPath); + + var rewriter = new MarshalMethodsAssemblyRewriter (classifier.MarshalMethods, classifier.Assemblies, assemblyPaths, Log); + rewriter.Rewrite (res, environmentParser.AreBrokenExceptionTransitionsEnabled (Environments)); + } + + // Step 3 - Generate type maps + // Type mappings need to use all the assemblies, always. + WriteTypeMappings (allJavaTypes, cache); + + WriteAcwMaps (javaTypes, cache); // Step 3 - Merge [Activity] and friends into AndroidManifest.xml var manifest = new ManifestDocument (ManifestTemplate) { diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/IInputAssemblySet.cs b/src/Xamarin.Android.Build.Tasks/Utilities/IInputAssemblySet.cs new file mode 100644 index 00000000000..03c568806d3 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/IInputAssemblySet.cs @@ -0,0 +1,14 @@ +using System.IO; + +using Microsoft.Build.Framework; + +namespace Xamarin.Android.Tasks; + +abstract class InputAssemblySet +{ + public abstract void AddJavaTypeAssembly (ITaskItem assemblyItem); + public abstract void AddUserAssembly (ITaskItem assemblyItem); + public abstract bool IsUserAssembly (string name); + + protected string GetUserAssemblyKey (ITaskItem assemblyItem) => Path.GetFileNameWithoutExtension (assemblyItem.ItemSpec); +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/RidAgnosticeInputAssemblySet.cs b/src/Xamarin.Android.Build.Tasks/Utilities/RidAgnosticeInputAssemblySet.cs new file mode 100644 index 00000000000..065fd82b0c9 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/RidAgnosticeInputAssemblySet.cs @@ -0,0 +1,36 @@ +using System; +using System.Collections.Generic; + +using Microsoft.Build.Framework; + +namespace Xamarin.Android.Tasks; + +class RidAgnosticInputAssemblySet : InputAssemblySet +{ + Dictionary javaTypeAssemblies = new (StringComparer.OrdinalIgnoreCase); + Dictionary userAssemblies = new (StringComparer.OrdinalIgnoreCase); + + public ICollection JavaTypeAssemblies => javaTypeAssemblies.Values; + public ICollection UserAssemblies => userAssemblies.Values; + + public override void AddJavaTypeAssembly (ITaskItem assemblyItem) + { + Add (assemblyItem.ItemSpec, assemblyItem, javaTypeAssemblies); + } + + public override void AddUserAssembly (ITaskItem assemblyItem) + { + Add (GetUserAssemblyKey (assemblyItem), assemblyItem, userAssemblies); + } + + public override bool IsUserAssembly (string name) => userAssemblies.ContainsKey (name); + + void Add (string key, ITaskItem assemblyItem, Dictionary dict) + { + if (dict.ContainsKey (key)) { + return; + } + + dict.Add (key, assemblyItem); + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/RidSpecificInputAssemblySet.cs b/src/Xamarin.Android.Build.Tasks/Utilities/RidSpecificInputAssemblySet.cs new file mode 100644 index 00000000000..f5d4f726d6f --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/RidSpecificInputAssemblySet.cs @@ -0,0 +1,56 @@ +using System; +using System.Collections.Generic; + +using Microsoft.Build.Framework; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +class RidSpecificInputAssemblySet : InputAssemblySet +{ + Dictionary> javaTypeAssemblies = new (); + Dictionary> userAssemblies = new (); + + public Dictionary> JavaTypeAssemblies => javaTypeAssemblies; + public Dictionary> UserAssemblies => userAssemblies; + + public override void AddJavaTypeAssembly (ITaskItem assemblyItem) + { + Add (assemblyItem.ItemSpec, assemblyItem, javaTypeAssemblies); + } + + public override void AddUserAssembly (ITaskItem assemblyItem) + { + Add (GetUserAssemblyKey (assemblyItem), assemblyItem, userAssemblies); + } + + public override bool IsUserAssembly (string name) + { + foreach (var kvp in userAssemblies) { + if (kvp.Value.ContainsKey (name)) { + return true; + } + } + + return false; + } + + void Add (string key, ITaskItem assemblyItem, Dictionary> assemblies) + { + AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (assemblyItem); + if (arch == AndroidTargetArch.None) { + throw new InvalidOperationException ($"`Abi` metadata item is required for assembly '{assemblyItem.ItemSpec}'"); + } + + if (!assemblies.TryGetValue (arch, out Dictionary? dict)) { + dict = new (StringComparer.OrdinalIgnoreCase); + assemblies.Add (arch, dict); + } + + if (dict.ContainsKey (key)) { + return; + } + + dict.Add (key, assemblyItem); + } +} From b37684287d1b5ef59a79ec428a97761aae4095aa Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 13 Jul 2023 16:37:05 +0200 Subject: [PATCH 28/71] Refactoring continued --- .../Tasks/GenerateJavaStubs.cs | 489 ++---------------- .../Utilities/AcwMapWriter.cs | 93 ++++ .../MarshalMethodsAssemblyRewriter.cs | 5 + .../Utilities/NativeTypeMappingData.cs | 2 +- .../Utilities/TypeMapGenerator.cs | 13 +- 5 files changed, 163 insertions(+), 439 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/AcwMapWriter.cs diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index f37b2c83172..d34dcac9dd3 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -34,6 +34,7 @@ sealed class RunState public ICollection UserAssemblies { get; set; } public InputAssemblySet AssemblySet { get; set; } public bool UseMarshalMethods { get; set; } + public AndroidTargetArch TargetArch { get; set; } = AndroidTargetArch.None; /// /// If `true`, generate code/data that doesn't depend on a specific RID (e.g. ACW maps or JCWs) @@ -174,7 +175,11 @@ void Run () // Typemaps don't use MVIDs or metadata tokens void RunDebugNoLinking () { - var assemblies = CollectInterestingAssemblies (); + const AndroidTargetArch ArchHere = AndroidTargetArch.None; + + LogRunMode ("Debug, no linking"); + XAAssemblyResolver resolver = MakeResolver (useMarshalMethods: false); + var assemblies = CollectInterestingAssemblies (ArchHere, resolver); throw new NotImplementedException (); } @@ -182,7 +187,11 @@ void RunDebugNoLinking () // Typemaps don't use MVIDs or metadata tokens void RunDebugWithLinking () { - var assemblies = CollectInterestingAssemblies (); + const AndroidTargetArch ArchHere = AndroidTargetArch.None; + + LogRunMode ("Debug, with linking"); + XAAssemblyResolver resolver = MakeResolver (useMarshalMethods: false); + var assemblies = CollectInterestingAssemblies (ArchHere, resolver); throw new NotImplementedException (); } @@ -190,29 +199,45 @@ void RunDebugWithLinking () // Typemaps use MVIDs and metadata tokens void RunReleaseNoLinking (bool useMarshalMethods) { - var assemblies = CollectInterestingAssemblies (); + const AndroidTargetArch ArchHere = AndroidTargetArch.None; + + LogRunMode ("Release, no linking"); + XAAssemblyResolver resolver = MakeResolver (useMarshalMethods); + var assemblies = CollectInterestingAssemblies (ArchHere, resolver); var state = new RunState { UseMarshalMethods = useMarshalMethods, AssemblySet = assemblies, JavaTypeAssemblies = assemblies.JavaTypeAssemblies, UserAssemblies = assemblies.UserAssemblies, GenerateRidAgnosticParts = true, + Resolver = resolver, + TargetArch = ArchHere, }; - DoRun (state); + DoRun (state, out ApplicationConfigTaskState appConfState); + RegisterApplicationConfigState (appConfState); } // We have as many sets of assemblies as there are RIDs, all assemblies are RID-specific // Typemaps use MVIDs and metadata tokens, need to process all per-RID assemblies separately void RunReleaseWithLinking (bool useMarshalMethods) { - var assemblies = CollectInterestingAssemblies (); + LogRunMode ("Release, with linking"); + XAAssemblyResolver resolver = MakeResolver (useMarshalMethods); + var assemblies = CollectInterestingAssemblies (AndroidTargetArch.None, resolver); throw new NotImplementedException (); } - RidSpecificInputAssemblySet CollectInterestingAssembliesRidSpecific () => CollectInterestingAssemblies (); - RidAgnosticInputAssemblySet CollectInterestingAssembliesRidAgnostic () => CollectInterestingAssemblies (); + void RegisterApplicationConfigState (ApplicationConfigTaskState appConfState) + { + BuildEngine4.RegisterTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (ApplicationConfigTaskState.RegisterTaskObjectKey), appConfState, RegisteredTaskObjectLifetime.Build); + } - T CollectInterestingAssemblies () where T: InputAssemblySet, new() + void LogRunMode (string mode) + { + Log.LogDebugMessage ($"GenerateJavaStubs mode: {mode}"); + } + + T CollectInterestingAssemblies (AndroidTargetArch targetArch, XAAssemblyResolver resolver) where T: InputAssemblySet, new() { var assemblies = new T (); bool hasExportReference = false; @@ -244,6 +269,8 @@ void RunReleaseWithLinking (bool useMarshalMethods) if (addAssembly) { assemblies.AddJavaTypeAssembly (assembly); } + + resolver.Load (targetArch, assembly.ItemSpec); } // However we only want to look for JLO types in user code for Java stub code generation @@ -252,13 +279,15 @@ void RunReleaseWithLinking (bool useMarshalMethods) Log.LogDebugMessage ($"Skipping Java Stub Generation for {asm.ItemSpec}"); continue; } + + resolver.Load (targetArch, asm.ItemSpec); assemblies.AddJavaTypeAssembly (asm); assemblies.AddUserAssembly (asm); } return assemblies; } - void DoRun (RunState state) + void DoRun (RunState state, out ApplicationConfigTaskState? appConfState) { PackageNamingPolicy pnp; JavaNativeTypeManager.PackageNamingPolicy = Enum.TryParse (PackageNamingPolicy, out pnp) ? pnp : PackageNamingPolicyEnum.LowercaseCrc64; @@ -286,20 +315,30 @@ void DoRun (RunState state) classifier = new MarshalMethodsClassifier (cache, state.Resolver, Log); } - if (state.GenerateRidAgnosticParts) { + // TODO: JCWs don't need to be generated for every RID, but we do need the classifier for the marshal methods + // rewriter and generator. Add a mode to only classify marshal methods without generating the files. + // For now, always generate the JCWs if marshal methods are enabled + if (state.UseMarshalMethods || state.GenerateRidAgnosticParts) { // Step 2 - Generate Java stub code bool success = CreateJavaSources (javaTypes, cache, classifier, state.UseMarshalMethods); if (!success) { + appConfState = null; return; // TODO: throw? Return `false`? } } - // TODO: marshal methods rewriter here + if (state.UseMarshalMethods) { + // We need to parse the environment files supplied by the user to see if they want to use broken exception transitions. This information is needed + // in order to properly generate wrapper methods in the marshal methods assembly rewriter. + // We don't care about those generated by us, since they won't contain the `XA_BROKEN_EXCEPTION_TRANSITIONS` variable we look for. + var environmentParser = new EnvironmentFilesParser (); + var rewriter = new MarshalMethodsAssemblyRewriter (classifier.MarshalMethods, classifier.Assemblies, Log); + rewriter.Rewrite (state.Resolver, environmentParser.AreBrokenExceptionTransitionsEnabled (Environments)); + } - // TODO: this likely needs to be handled in a different manner. // Step 3 - Generate type maps // Type mappings need to use all the assemblies, always. - WriteTypeMappings (allJavaTypes, cache); + WriteTypeMappings (state.TargetArch, allJavaTypes, cache, out appConfState); if (state.GenerateRidAgnosticParts) { WriteAcwMaps (javaTypes, cache); @@ -393,289 +432,8 @@ void UpdateAndroidManifest (RunState state, TypeDefinitionCache cache, List javaTypes, TypeDefinitionCache cache) { - // We need to save a map of .NET type -> ACW type for resource file fixups - var managed = new Dictionary (javaTypes.Count, StringComparer.Ordinal); - var java = new Dictionary (javaTypes.Count, StringComparer.Ordinal); - - var managedConflicts = new Dictionary> (0, StringComparer.Ordinal); - var javaConflicts = new Dictionary> (0, StringComparer.Ordinal); - - bool success = true; - using (var acw_map = MemoryStreamPool.Shared.CreateStreamWriter ()) { - foreach (JavaType jt in javaTypes) { - TypeDefinition type = jt.Type; - string managedKey = type.FullName.Replace ('/', '.'); - string javaKey = JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.'); - - acw_map.Write (type.GetPartialAssemblyQualifiedName (cache)); - acw_map.Write (';'); - acw_map.Write (javaKey); - acw_map.WriteLine (); - - TypeDefinition conflict; - bool hasConflict = false; - if (managed.TryGetValue (managedKey, out conflict)) { - if (!conflict.Module.Name.Equals (type.Module.Name)) { - if (!managedConflicts.TryGetValue (managedKey, out var list)) - managedConflicts.Add (managedKey, list = new List { conflict.GetPartialAssemblyName (cache) }); - list.Add (type.GetPartialAssemblyName (cache)); - } - hasConflict = true; - } - if (java.TryGetValue (javaKey, out conflict)) { - if (!conflict.Module.Name.Equals (type.Module.Name)) { - if (!javaConflicts.TryGetValue (javaKey, out var list)) - javaConflicts.Add (javaKey, list = new List { conflict.GetAssemblyQualifiedName (cache) }); - list.Add (type.GetAssemblyQualifiedName (cache)); - success = false; - } - hasConflict = true; - } - if (!hasConflict) { - managed.Add (managedKey, type); - java.Add (javaKey, type); - - acw_map.Write (managedKey); - acw_map.Write (';'); - acw_map.Write (javaKey); - acw_map.WriteLine (); - - acw_map.Write (JavaNativeTypeManager.ToCompatJniName (type, cache).Replace ('/', '.')); - acw_map.Write (';'); - acw_map.Write (javaKey); - acw_map.WriteLine (); - } - } - - acw_map.Flush (); - Files.CopyIfStreamChanged (acw_map.BaseStream, AcwMapFile); - } - - foreach (var kvp in managedConflicts) { - Log.LogCodedWarning ("XA4214", Properties.Resources.XA4214, kvp.Key, string.Join (", ", kvp.Value)); - Log.LogCodedWarning ("XA4214", Properties.Resources.XA4214_Result, kvp.Key, kvp.Value [0]); - } - - foreach (var kvp in javaConflicts) { - Log.LogCodedError ("XA4215", Properties.Resources.XA4215, kvp.Key); - foreach (var typeName in kvp.Value) - Log.LogCodedError ("XA4215", Properties.Resources.XA4215_Details, kvp.Key, typeName); - } - } - - void Run (XAAssemblyResolver res, bool useMarshalMethods) - { - PackageNamingPolicy pnp; - JavaNativeTypeManager.PackageNamingPolicy = Enum.TryParse (PackageNamingPolicy, out pnp) ? pnp : PackageNamingPolicyEnum.LowercaseCrc64; - - Dictionary>? abiSpecificAssembliesByPath = null; - if (useMarshalMethods) { - abiSpecificAssembliesByPath = new Dictionary> (StringComparer.Ordinal); - } - - // Put every assembly we'll need in the resolver - bool hasExportReference = false; - bool haveMonoAndroid = false; - var allTypemapAssemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); - var userAssemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); - - foreach (var assembly in ResolvedAssemblies) { - bool value; - if (bool.TryParse (assembly.GetMetadata (AndroidSkipJavaStubGeneration), out value) && value) { - Log.LogDebugMessage ($"Skipping Java Stub Generation for {assembly.ItemSpec}"); - continue; - } - - bool addAssembly = false; - string fileName = Path.GetFileName (assembly.ItemSpec); - if (!hasExportReference && String.Compare ("Mono.Android.Export.dll", fileName, StringComparison.OrdinalIgnoreCase) == 0) { - hasExportReference = true; - addAssembly = true; - } else if (!haveMonoAndroid && String.Compare ("Mono.Android.dll", fileName, StringComparison.OrdinalIgnoreCase) == 0) { - haveMonoAndroid = true; - addAssembly = true; - } else if (MonoAndroidHelper.FrameworkAssembliesToTreatAsUserAssemblies.Contains (fileName)) { - if (!bool.TryParse (assembly.GetMetadata (AndroidSkipJavaStubGeneration), out value) || !value) { - string name = Path.GetFileNameWithoutExtension (fileName); - if (!userAssemblies.ContainsKey (name)) - userAssemblies.Add (name, assembly.ItemSpec); - addAssembly = true; - } - } - - if (addAssembly) { - MaybeAddAbiSpecifcAssembly (assembly, fileName); - if (!allTypemapAssemblies.ContainsKey (assembly.ItemSpec)) { - allTypemapAssemblies.Add (assembly.ItemSpec, assembly); - } - } - - res.Load (MonoAndroidHelper.GetTargetArch (assembly), assembly.ItemSpec); - } - - // However we only want to look for JLO types in user code for Java stub code generation - foreach (var asm in ResolvedUserAssemblies) { - if (bool.TryParse (asm.GetMetadata (AndroidSkipJavaStubGeneration), out bool value) && value) { - Log.LogDebugMessage ($"Skipping Java Stub Generation for {asm.ItemSpec}"); - continue; - } - res.Load (MonoAndroidHelper.GetTargetArch (asm), asm.ItemSpec); - MaybeAddAbiSpecifcAssembly (asm, Path.GetFileName (asm.ItemSpec)); - if (!allTypemapAssemblies.ContainsKey (asm.ItemSpec)) { - allTypemapAssemblies.Add (asm.ItemSpec, asm); - } - - string name = Path.GetFileNameWithoutExtension (asm.ItemSpec); - if (!userAssemblies.ContainsKey (name)) - userAssemblies.Add (name, asm.ItemSpec); - } - - // Step 1 - Find all the JLO types - var cache = new TypeDefinitionCache (); - var scanner = new XAJavaTypeScanner (Log, cache) { - ErrorOnCustomJavaObject = ErrorOnCustomJavaObject, - }; - List allJavaTypes = scanner.GetJavaTypes (allTypemapAssemblies.Values, res); - var javaTypes = new List (); - - foreach (JavaType jt in allJavaTypes) { - // Whem marshal methods are in use we do not want to skip non-user assemblies (such as Mono.Android) - we need to generate JCWs for them during - // application build, unlike in Debug configuration or when marshal methods are disabled, in which case we use JCWs generated during Xamarin.Android - // build and stored in a jar file. - if ((!useMarshalMethods && !userAssemblies.ContainsKey (jt.Type.Module.Assembly.Name.Name)) || JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (jt.Type, cache)) { - continue; - } - javaTypes.Add (jt); - } - - MarshalMethodsClassifier classifier = null; - if (useMarshalMethods) { - classifier = new MarshalMethodsClassifier (cache, res, Log); - } - - // Step 2 - Generate Java stub code - var success = CreateJavaSources (javaTypes, cache, classifier, useMarshalMethods); - if (!success) - return; - - if (useMarshalMethods) { - // We need to parse the environment files supplied by the user to see if they want to use broken exception transitions. This information is needed - // in order to properly generate wrapper methods in the marshal methods assembly rewriter. - // We don't care about those generated by us, since they won't contain the `XA_BROKEN_EXCEPTION_TRANSITIONS` variable we look for. - var environmentParser = new EnvironmentFilesParser (); - - Dictionary assemblyPaths = AddMethodsFromAbiSpecificAssemblies (classifier, res, abiSpecificAssembliesByPath); - - var rewriter = new MarshalMethodsAssemblyRewriter (classifier.MarshalMethods, classifier.Assemblies, assemblyPaths, Log); - rewriter.Rewrite (res, environmentParser.AreBrokenExceptionTransitionsEnabled (Environments)); - } - - // Step 3 - Generate type maps - // Type mappings need to use all the assemblies, always. - WriteTypeMappings (allJavaTypes, cache); - - WriteAcwMaps (javaTypes, cache); - - // Step 3 - Merge [Activity] and friends into AndroidManifest.xml - var manifest = new ManifestDocument (ManifestTemplate) { - PackageName = PackageName, - VersionName = VersionName, - ApplicationLabel = ApplicationLabel ?? PackageName, - Placeholders = ManifestPlaceholders, - Resolver = res, - SdkDir = AndroidSdkDir, - TargetSdkVersion = AndroidSdkPlatform, - MinSdkVersion = MonoAndroidHelper.ConvertSupportedOSPlatformVersionToApiLevel (SupportedOSPlatformVersion).ToString (), - Debug = Debug, - MultiDex = MultiDex, - NeedsInternet = NeedsInternet, - InstantRunEnabled = InstantRunEnabled - }; - // Only set manifest.VersionCode if there is no existing value in AndroidManifest.xml. - if (manifest.HasVersionCode) { - Log.LogDebugMessage ($"Using existing versionCode in: {ManifestTemplate}"); - } else if (!string.IsNullOrEmpty (VersionCode)) { - manifest.VersionCode = VersionCode; - } - manifest.Assemblies.AddRange (userAssemblies.Values); - - if (!String.IsNullOrWhiteSpace (CheckedBuild)) { - // We don't validate CheckedBuild value here, this will be done in BuildApk. We just know that if it's - // on then we need android:debuggable=true and android:extractNativeLibs=true - manifest.ForceDebuggable = true; - manifest.ForceExtractNativeLibs = true; - } - - var additionalProviders = manifest.Merge (Log, cache, allJavaTypes, ApplicationJavaClass, EmbedAssemblies, BundledWearApplicationName, MergedManifestDocuments); - - // Only write the new manifest if it actually changed - if (manifest.SaveIfChanged (Log, MergedAndroidManifestOutput)) { - Log.LogDebugMessage ($"Saving: {MergedAndroidManifestOutput}"); - } - - // Create additional runtime provider java sources. - string providerTemplateFile = "MonoRuntimeProvider.Bundled.java"; - string providerTemplate = GetResource (providerTemplateFile); - - foreach (var provider in additionalProviders) { - var contents = providerTemplate.Replace ("MonoRuntimeProvider", provider); - var real_provider = Path.Combine (OutputDirectory, "src", "mono", provider + ".java"); - Files.CopyIfStringChanged (contents, real_provider); - } - - // Create additional application java sources. - StringWriter regCallsWriter = new StringWriter (); - regCallsWriter.WriteLine ("\t\t// Application and Instrumentation ACWs must be registered first."); - foreach (JavaType jt in javaTypes) { - TypeDefinition type = jt.Type; - if (JavaNativeTypeManager.IsApplication (type, cache) || JavaNativeTypeManager.IsInstrumentation (type, cache)) { - if (classifier != null && !classifier.FoundDynamicallyRegisteredMethods (type)) { - continue; - } - - string javaKey = JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.'); - regCallsWriter.WriteLine ("\t\tmono.android.Runtime.register (\"{0}\", {1}.class, {1}.__md_methods);", - type.GetAssemblyQualifiedName (cache), javaKey); - } - } - regCallsWriter.Close (); - - var real_app_dir = Path.Combine (OutputDirectory, "src", "mono", "android", "app"); - string applicationTemplateFile = "ApplicationRegistration.java"; - SaveResource (applicationTemplateFile, applicationTemplateFile, real_app_dir, - template => template.Replace ("// REGISTER_APPLICATION_AND_INSTRUMENTATION_CLASSES_HERE", regCallsWriter.ToString ())); - - if (useMarshalMethods) { - classifier.AddSpecialCaseMethods (); - - Log.LogDebugMessage ($"Number of generated marshal methods: {classifier.MarshalMethods.Count}"); - - if (classifier.RejectedMethodCount > 0) { - Log.LogWarning ($"Number of methods in the project that will be registered dynamically: {classifier.RejectedMethodCount}"); - } - - if (classifier.WrappedMethodCount > 0) { - // TODO: change to LogWarning once the generator can output code which requires no non-blittable wrappers - Log.LogDebugMessage ($"Number of methods in the project that need marshal method wrappers: {classifier.WrappedMethodCount}"); - } - } - - void MaybeAddAbiSpecifcAssembly (ITaskItem assembly, string fileName) - { - if (abiSpecificAssembliesByPath == null) { - return; - } - - string? abi = assembly.GetMetadata ("Abi"); - if (!String.IsNullOrEmpty (abi)) { - if (!abiSpecificAssembliesByPath.TryGetValue (fileName, out List? items)) { - items = new List (); - abiSpecificAssembliesByPath.Add (fileName, items); - } - - items.Add (assembly); - } - } + var writer = new AcwMapWriter (Log, AcwMapFile); + writer.Write (javaTypes, cache); } AssemblyDefinition LoadAssembly (string path, XAAssemblyResolver? resolver = null) @@ -824,146 +582,13 @@ void SaveResource (string resource, string filename, string destDir, Func types, TypeDefinitionCache cache) + void WriteTypeMappings (AndroidTargetArch targetArch, List types, TypeDefinitionCache cache, out ApplicationConfigTaskState appConfState) { - var tmg = new TypeMapGenerator ((string message) => Log.LogDebugMessage (message), SupportedAbis); - if (!tmg.Generate (Debug, SkipJniAddNativeMethodRegistrationAttributeScan, types, cache, TypemapOutputDirectory, GenerateNativeAssembly, out ApplicationConfigTaskState appConfState)) { + var tmg = new TypeMapGenerator (targetArch, Log, SupportedAbis); + if (!tmg.Generate (Debug, SkipJniAddNativeMethodRegistrationAttributeScan, types, cache, TypemapOutputDirectory, GenerateNativeAssembly, out appConfState)) { throw new XamarinAndroidException (4308, Properties.Resources.XA4308); } GeneratedBinaryTypeMaps = tmg.GeneratedBinaryTypeMaps.ToArray (); - BuildEngine4.RegisterTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (ApplicationConfigTaskState.RegisterTaskObjectKey), appConfState, RegisteredTaskObjectLifetime.Build); - } - - /// - /// - /// Classifier will see only unique assemblies, since that's what's processed by the JI type scanner - even though some assemblies may have - /// abi-specific features (e.g. inlined `IntPtr.Size` or processor-specific intrinsics), the **types** and **methods** will all be the same and, thus, - /// there's no point in scanning all of the additional copies of the same assembly. - /// - /// - /// This, however, doesn't work for the rewriter which needs to rewrite all of the copies so that they all have the same generated wrappers. In - /// order to do that, we need to go over the list of assemblies found by the classifier, see if they are abi-specific ones and then add all the - /// marshal methods from the abi-specific assembly copies, so that the rewriter can easily rewrite them all. - /// - /// - /// This method returns a dictionary matching `AssemblyDefinition` instances to the path on disk to the assembly file they were loaded from. It is necessary - /// because uses a stream to load the data, in order to avoid later sharing violation issues when writing the assemblies. Path - /// information is required by to be available for each - /// - /// - Dictionary AddMethodsFromAbiSpecificAssemblies (MarshalMethodsClassifier classifier, XAAssemblyResolver resolver, Dictionary> abiSpecificAssemblies) - { - IDictionary> marshalMethods = classifier.MarshalMethods; - ICollection assemblies = classifier.Assemblies; - var newAssemblies = new List (); - var assemblyPaths = new Dictionary (); - - foreach (AssemblyDefinition asmdef in assemblies) { - string fileName = Path.GetFileName (asmdef.MainModule.FileName); - if (!abiSpecificAssemblies.TryGetValue (fileName, out List? abiAssemblyItems)) { - continue; - } - - List assemblyMarshalMethods = FindMarshalMethodsForAssembly (marshalMethods, asmdef);; - Log.LogDebugMessage ($"Assembly {fileName} is ABI-specific"); - foreach (ITaskItem abiAssemblyItem in abiAssemblyItems) { - if (String.Compare (abiAssemblyItem.ItemSpec, asmdef.MainModule.FileName, StringComparison.Ordinal) == 0) { - continue; - } - - Log.LogDebugMessage ($"Looking for matching mashal methods in {abiAssemblyItem.ItemSpec}"); - FindMatchingMethodsInAssembly (abiAssemblyItem, classifier, assemblyMarshalMethods, resolver, newAssemblies, assemblyPaths); - } - } - - if (newAssemblies.Count > 0) { - foreach (AssemblyDefinition asmdef in newAssemblies) { - assemblies.Add (asmdef); - } - } - - return assemblyPaths; - } - - List FindMarshalMethodsForAssembly (IDictionary> marshalMethods, AssemblyDefinition asm) - { - var seenNativeCallbacks = new HashSet (); - var assemblyMarshalMethods = new List (); - - foreach (var kvp in marshalMethods) { - foreach (MarshalMethodEntry method in kvp.Value) { - if (method.NativeCallback.Module.Assembly != asm) { - continue; - } - - // More than one overriden method can use the same native callback method, we're interested only in unique native - // callbacks, since that's what gets rewritten. - if (seenNativeCallbacks.Contains (method.NativeCallback)) { - continue; - } - - seenNativeCallbacks.Add (method.NativeCallback); - assemblyMarshalMethods.Add (method); - } - } - - return assemblyMarshalMethods; - } - - void FindMatchingMethodsInAssembly (ITaskItem assemblyItem, MarshalMethodsClassifier classifier, List assemblyMarshalMethods, XAAssemblyResolver resolver, List newAssemblies, Dictionary assemblyPaths) - { - AssemblyDefinition asm = LoadAssembly (assemblyItem.ItemSpec, resolver); - newAssemblies.Add (asm); - assemblyPaths.Add (asm, assemblyItem.ItemSpec); - - foreach (MarshalMethodEntry methodEntry in assemblyMarshalMethods) { - TypeDefinition wantedType = methodEntry.NativeCallback.DeclaringType; - TypeDefinition? type = asm.MainModule.FindType (wantedType.FullName); - if (type == null) { - throw new InvalidOperationException ($"Internal error: type '{wantedType.FullName}' not found in assembly '{assemblyItem.ItemSpec}', a linker error?"); - } - - if (type.MetadataToken != wantedType.MetadataToken) { - throw new InvalidOperationException ($"Internal error: type '{type.FullName}' in assembly '{assemblyItem.ItemSpec}' has a different token ID than the original type"); - } - - FindMatchingMethodInType (methodEntry, type, classifier); - } - } - - void FindMatchingMethodInType (MarshalMethodEntry methodEntry, TypeDefinition type, MarshalMethodsClassifier classifier) - { - string callbackName = methodEntry.NativeCallback.FullName; - - foreach (MethodDefinition typeNativeCallbackMethod in type.Methods) { - if (String.Compare (typeNativeCallbackMethod.FullName, callbackName, StringComparison.Ordinal) != 0) { - continue; - } - - if (typeNativeCallbackMethod.Parameters.Count != methodEntry.NativeCallback.Parameters.Count) { - continue; - } - - if (typeNativeCallbackMethod.MetadataToken != methodEntry.NativeCallback.MetadataToken) { - throw new InvalidOperationException ($"Internal error: tokens don't match for '{typeNativeCallbackMethod.FullName}'"); - } - - bool allMatch = true; - for (int i = 0; i < typeNativeCallbackMethod.Parameters.Count; i++) { - if (String.Compare (typeNativeCallbackMethod.Parameters[i].ParameterType.FullName, methodEntry.NativeCallback.Parameters[i].ParameterType.FullName, StringComparison.Ordinal) != 0) { - allMatch = false; - break; - } - } - - if (!allMatch) { - continue; - } - - Log.LogDebugMessage ($"Found match for '{typeNativeCallbackMethod.FullName}' in {type.Module.FileName}"); - string methodKey = classifier.GetStoreMethodKey (methodEntry); - classifier.MarshalMethods[methodKey].Add (new MarshalMethodEntry (methodEntry, typeNativeCallbackMethod)); - } } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AcwMapWriter.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AcwMapWriter.cs new file mode 100644 index 00000000000..d6f28f0625a --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AcwMapWriter.cs @@ -0,0 +1,93 @@ +using System; +using System.Collections.Generic; + +using Java.Interop.Tools.Cecil; +using Java.Interop.Tools.TypeNameMappings; +using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Mono.Cecil; + +namespace Xamarin.Android.Tasks; + +class AcwMapWriter +{ + readonly TaskLoggingHelper Log; + readonly string AcwMapFile; + + public AcwMapWriter (TaskLoggingHelper log, string acwMapFile) + { + Log = log; + AcwMapFile = acwMapFile; + } + + public void Write (List javaTypes, TypeDefinitionCache cache) + { + // We need to save a map of .NET type -> ACW type for resource file fixups + var managed = new Dictionary (javaTypes.Count, StringComparer.Ordinal); + var java = new Dictionary (javaTypes.Count, StringComparer.Ordinal); + + var managedConflicts = new Dictionary> (0, StringComparer.Ordinal); + var javaConflicts = new Dictionary> (0, StringComparer.Ordinal); + + using (var acw_map = MemoryStreamPool.Shared.CreateStreamWriter ()) { + foreach (JavaType jt in javaTypes) { + TypeDefinition type = jt.Type; + string managedKey = type.FullName.Replace ('/', '.'); + string javaKey = JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.'); + + acw_map.Write (type.GetPartialAssemblyQualifiedName (cache)); + acw_map.Write (';'); + acw_map.Write (javaKey); + acw_map.WriteLine (); + + TypeDefinition conflict; + bool hasConflict = false; + if (managed.TryGetValue (managedKey, out conflict)) { + if (!conflict.Module.Name.Equals (type.Module.Name)) { + if (!managedConflicts.TryGetValue (managedKey, out var list)) + managedConflicts.Add (managedKey, list = new List { conflict.GetPartialAssemblyName (cache) }); + list.Add (type.GetPartialAssemblyName (cache)); + } + hasConflict = true; + } + if (java.TryGetValue (javaKey, out conflict)) { + if (!conflict.Module.Name.Equals (type.Module.Name)) { + if (!javaConflicts.TryGetValue (javaKey, out var list)) + javaConflicts.Add (javaKey, list = new List { conflict.GetAssemblyQualifiedName (cache) }); + list.Add (type.GetAssemblyQualifiedName (cache)); + } + hasConflict = true; + } + if (!hasConflict) { + managed.Add (managedKey, type); + java.Add (javaKey, type); + + acw_map.Write (managedKey); + acw_map.Write (';'); + acw_map.Write (javaKey); + acw_map.WriteLine (); + + acw_map.Write (JavaNativeTypeManager.ToCompatJniName (type, cache).Replace ('/', '.')); + acw_map.Write (';'); + acw_map.Write (javaKey); + acw_map.WriteLine (); + } + } + + acw_map.Flush (); + Files.CopyIfStreamChanged (acw_map.BaseStream, AcwMapFile); + } + + foreach (var kvp in managedConflicts) { + Log.LogCodedWarning ("XA4214", Properties.Resources.XA4214, kvp.Key, string.Join (", ", kvp.Value)); + Log.LogCodedWarning ("XA4214", Properties.Resources.XA4214_Result, kvp.Key, kvp.Value [0]); + } + + foreach (var kvp in javaConflicts) { + Log.LogCodedError ("XA4215", Properties.Resources.XA4215, kvp.Key); + foreach (var typeName in kvp.Value) + Log.LogCodedError ("XA4215", Properties.Resources.XA4215_Details, kvp.Key, typeName); + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs index 96063bf2126..d8d890f8a3c 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs @@ -26,6 +26,11 @@ sealed class AssemblyImports IDictionary assemblyPaths; TaskLoggingHelper log; + public MarshalMethodsAssemblyRewriter (IDictionary> methods, ICollection uniqueAssemblies, TaskLoggingHelper log) + { + throw new NotImplementedException (); + } + public MarshalMethodsAssemblyRewriter (IDictionary> methods, ICollection uniqueAssemblies, IDictionary assemblyPaths, TaskLoggingHelper log) { this.assemblyPaths = assemblyPaths; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeTypeMappingData.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeTypeMappingData.cs index f1a5205f746..be2d4ad71dc 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeTypeMappingData.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeTypeMappingData.cs @@ -12,7 +12,7 @@ class NativeTypeMappingData public uint MapModuleCount { get; } public uint JavaTypeCount { get; } - public NativeTypeMappingData (Action logger, TypeMapGenerator.ModuleReleaseData[] modules) + public NativeTypeMappingData (TypeMapGenerator.ModuleReleaseData[] modules) { Modules = modules ?? throw new ArgumentNullException (nameof (modules)); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs index 1b020083282..317b70926fd 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs @@ -8,6 +8,7 @@ using Java.Interop.Tools.Cecil; using Mono.Cecil; using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Utilities; using Xamarin.Android.Tools; namespace Xamarin.Android.Tasks @@ -133,7 +134,8 @@ public void AddKnownAssembly (TypeDefinition td) public string GetAssemblyName (TypeDefinition td) => td.Module.Assembly.FullName; } - Action logger; + AndroidTargetArch targetArch; + readonly TaskLoggingHelper Log; Encoding outputEncoding; byte[] moduleMagicString; byte[] typemapIndexMagicString; @@ -141,11 +143,10 @@ public void AddKnownAssembly (TypeDefinition td) public IList GeneratedBinaryTypeMaps { get; } = new List (); - public TypeMapGenerator (Action logger, string[] supportedAbis) + public TypeMapGenerator (AndroidTargetArch targetArch, TaskLoggingHelper log, string[] supportedAbis) { - this.logger = logger ?? throw new ArgumentNullException (nameof (logger)); - if (supportedAbis == null) - throw new ArgumentNullException (nameof (supportedAbis)); + this.targetArch = targetArch; + Log = log; this.supportedAbis = supportedAbis; outputEncoding = Files.UTF8withoutBOM; @@ -485,7 +486,7 @@ bool GenerateRelease (bool skipJniAddNativeMethodRegistrationAttributeScan, List module.Types = module.TypesScratch.Values.ToArray (); } - mappingData.Add (arch, new NativeTypeMappingData (logger, modules)); + mappingData.Add (arch, new NativeTypeMappingData (modules)); } var generator = new TypeMappingReleaseNativeAssemblyGenerator (mappingData); From 61bf38735f1699a8cf5cb68b3c377f487b590ff7 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 13 Jul 2023 23:51:25 +0200 Subject: [PATCH 29/71] More WIP --- .../Tasks/GenerateJavaStubs.cs | 2 +- .../Utilities/XAAssemblyResolver.cs | 1 + .../Xamarin.Android.Build.Tasks.targets | 46 +++++++++---------- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index d34dcac9dd3..9e0526c276d 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -412,7 +412,7 @@ void UpdateAndroidManifest (RunState state, TypeDefinitionCache cache, List - - <_InputAssembliesThatExist Include="@(InputAssemblies)" Condition="Exists('%(Identity)')" /> - <_NetstandardPath Include="@(ReferencePath->'%(RootDir)%(Directory)')" Condition="'%(FileName)%(Extension)' == 'netstandard.dll'" /> - - - <_NetstandardDir>@(_NetstandardPath) - <_ILRepackArgs>/out:"$(MSBuildThisFileDirectory)$(IntermediateOutputPath)$(AssemblyName).dll" /internalize - <_ILRepackArgs>$(_ILRepackArgs) /keyfile:"$(XamarinAndroidSourcePath)product.snk" - <_ILRepackArgs>$(_ILRepackArgs) "$(MSBuildThisFileDirectory)$(IntermediateOutputPath)$(AssemblyName).dll" - <_ILRepackArgs>$(_ILRepackArgs) @(_InputAssembliesThatExist->'"%(Identity)"', ' ') - <_ILRepackArgs>$(_ILRepackArgs) /lib:"$(_NetstandardDir.TrimEnd('\'))" - - - - - - + + + + + + + + + + + + + + + + + + + + + + + From 6fa66c734fe095140bd4a135b59b3fc936b460d8 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 14 Jul 2023 17:29:18 +0200 Subject: [PATCH 30/71] Updates from main --- .../Utilities/NativeTypeMappingData.cs | 2 +- .../Utilities/TypeMapGenerator.cs | 4 +- ...peMappingReleaseNativeAssemblyGenerator.cs | 396 +++++++++--------- 3 files changed, 205 insertions(+), 197 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeTypeMappingData.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeTypeMappingData.cs index f1a5205f746..be2d4ad71dc 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeTypeMappingData.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeTypeMappingData.cs @@ -12,7 +12,7 @@ class NativeTypeMappingData public uint MapModuleCount { get; } public uint JavaTypeCount { get; } - public NativeTypeMappingData (Action logger, TypeMapGenerator.ModuleReleaseData[] modules) + public NativeTypeMappingData (TypeMapGenerator.ModuleReleaseData[] modules) { Modules = modules ?? throw new ArgumentNullException (nameof (modules)); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs index 9518f37d008..3730176e751 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs @@ -468,7 +468,7 @@ bool GenerateRelease (bool skipJniAddNativeMethodRegistrationAttributeScan, List foreach (var kvp in state.TempModules) { AndroidTargetArch arch = kvp.Key; Dictionary tempModules = kvp.Value; - var modules = tempModules.Values.ToArray (); + ModuleReleaseData[] modules = tempModules.Values.ToArray (); Array.Sort (modules, new ModuleUUIDArrayComparer ()); foreach (ModuleReleaseData module in modules) { @@ -482,7 +482,7 @@ bool GenerateRelease (bool skipJniAddNativeMethodRegistrationAttributeScan, List module.Types = module.TypesScratch.Values.ToArray (); } - var composer = new TypeMappingReleaseNativeAssemblyGenerator (new NativeTypeMappingData (logger, modules)); + var composer = new TypeMappingReleaseNativeAssemblyGenerator (new NativeTypeMappingData (modules)); GenerateNativeAssembly (arch, composer, composer.Construct (), outputDirectory); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs index f92af2e902c..a6e4fd465df 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs @@ -1,15 +1,14 @@ using System; +using System.Collections; using System.Collections.Generic; using System.IO.Hashing; -using System.Linq; using System.Text; using Xamarin.Android.Tasks.LLVMIR; -using Xamarin.Android.Tools; namespace Xamarin.Android.Tasks { - class TypeMappingReleaseNativeAssemblyGenerator : TypeMappingAssemblyGenerator + partial class TypeMappingReleaseNativeAssemblyGenerator : LlvmIrComposer { sealed class TypeMapModuleContextDataProvider : NativeAssemblerStructContextDataProvider { @@ -18,11 +17,11 @@ public override string GetComment (object data, string fieldName) var map_module = EnsureType (data); if (String.Compare ("module_uuid", fieldName, StringComparison.Ordinal) == 0) { - return $"module_uuid: {map_module.MVID}"; + return $" module_uuid: {map_module.MVID}"; } if (String.Compare ("assembly_name", fieldName, StringComparison.Ordinal) == 0) { - return $"assembly_name: {map_module.assembly_name}"; + return $" assembly_name: {map_module.assembly_name}"; } return String.Empty; @@ -59,14 +58,6 @@ public override ulong GetBufferSize (object data, string fieldName) } } - sealed class JavaNameHashComparer : IComparer> - { - public int Compare (StructureInstance a, StructureInstance b) - { - return a.Obj.JavaNameHash.CompareTo (b.Obj.JavaNameHash); - } - } - // This is here only to generate strongly-typed IR internal sealed class MonoImage {} @@ -75,6 +66,9 @@ internal sealed class MonoImage // src/monodroid/jni/xamarin-app.hh TypeMapModuleEntry structure sealed class TypeMapModuleEntry { + [NativeAssembler (Ignore = true)] + public TypeMapJava JavaTypeMapEntry; + public uint type_token_id; public uint java_map_index; } @@ -126,7 +120,10 @@ sealed class TypeMapJava public string JavaName; [NativeAssembler (Ignore = true)] - public ulong JavaNameHash; + public uint JavaNameHash32; + + [NativeAssembler (Ignore = true)] + public ulong JavaNameHash64; public uint module_index; public uint type_token_id; @@ -145,86 +142,195 @@ public ModuleMapData (string symbolLabel, List> { - public readonly List> MapModules; - public readonly List> JavaMap; - public readonly Dictionary JavaTypesByName; - public readonly List JavaNames; - public readonly NativeTypeMappingData MappingData; - public ulong ModuleCounter = 0; - - public ArchGenerationState (NativeTypeMappingData mappingData) + public int Compare (StructureInstance a, StructureInstance b) { - MapModules = new List> (); - JavaMap = new List> (); - JavaTypesByName = new Dictionary (StringComparer.Ordinal); - JavaNames = new List (); - MappingData = mappingData; + return a.Instance.JavaNameHash32.CompareTo (b.Instance.JavaNameHash32); } } - StructureInfo typeMapJavaStructureInfo; - StructureInfo typeMapModuleStructureInfo; - StructureInfo typeMapModuleEntryStructureInfo; - Dictionary archState; + sealed class JavaNameHash64Comparer : IComparer> + { + public int Compare (StructureInstance a, StructureInstance b) + { + return a.Instance.JavaNameHash64.CompareTo (b.Instance.JavaNameHash64); + } + } - JavaNameHashComparer javaNameHashComparer; + sealed class ConstructionState + { + public List> MapModules; + public Dictionary JavaTypesByName; + public List JavaNames; + public List> JavaMap; + public List AllModulesData; + } + + readonly NativeTypeMappingData mappingData; + StructureInfo typeMapJavaStructureInfo; + StructureInfo typeMapModuleStructureInfo; + StructureInfo typeMapModuleEntryStructureInfo; + JavaNameHash32Comparer javaNameHash32Comparer; + JavaNameHash64Comparer javaNameHash64Comparer; + + ulong moduleCounter = 0; + + public TypeMappingReleaseNativeAssemblyGenerator (NativeTypeMappingData mappingData) + { + this.mappingData = mappingData ?? throw new ArgumentNullException (nameof (mappingData)); + javaNameHash32Comparer = new JavaNameHash32Comparer (); + javaNameHash64Comparer = new JavaNameHash64Comparer (); + } - public TypeMappingReleaseNativeAssemblyGenerator (Dictionary mappingData) + protected override void Construct (LlvmIrModule module) { - if (mappingData == null) { - throw new ArgumentNullException (nameof (mappingData)); + MapStructures (module); + + var cs = new ConstructionState (); + cs.JavaTypesByName = new Dictionary (StringComparer.Ordinal); + cs.JavaNames = new List (); + InitJavaMap (cs); + InitMapModules (cs); + HashJavaNames (cs); + PrepareModules (cs); + + module.AddGlobalVariable ("map_module_count", mappingData.MapModuleCount); + module.AddGlobalVariable ("java_type_count", cs.JavaMap.Count); + + var map_modules = new LlvmIrGlobalVariable (cs.MapModules, "map_modules", LlvmIrVariableOptions.GlobalWritable) { + Comment = " Managed modules map", + }; + module.Add (map_modules); + + // Java hashes are output bafore Java type map **and** managed modules, because they will also sort the Java map for us. + // This is not strictly necessary, as we could do the sorting in the java map BeforeWriteCallback, but this way we save + // time sorting only once. + var map_java_hashes = new LlvmIrGlobalVariable (typeof(List), "map_java_hashes") { + Comment = " Java types name hashes", + BeforeWriteCallback = GenerateAndSortJavaHashes, + BeforeWriteCallbackCallerState = cs, + GetArrayItemCommentCallback = GetJavaHashesItemComment, + GetArrayItemCommentCallbackCallerState = cs, + }; + map_java_hashes.WriteOptions &= ~LlvmIrVariableWriteOptions.ArrayWriteIndexComments; + module.Add (map_java_hashes); + + foreach (ModuleMapData mmd in cs.AllModulesData) { + var mmdVar = new LlvmIrGlobalVariable (mmd.Entries, mmd.SymbolLabel, LlvmIrVariableOptions.LocalConstant) { + BeforeWriteCallback = UpdateJavaIndexes, + BeforeWriteCallbackCallerState = cs, + }; + module.Add (mmdVar); } - javaNameHashComparer = new JavaNameHashComparer (); - archState = new Dictionary (mappingData.Count); + module.AddGlobalVariable ("map_java", cs.JavaMap, LlvmIrVariableOptions.GlobalConstant, " Java to managed map"); + module.AddGlobalVariable ("java_type_names", cs.JavaNames, LlvmIrVariableOptions.GlobalConstant, " Java type names"); + } + + void UpdateJavaIndexes (LlvmIrVariable variable, LlvmIrModuleTarget target, object? callerState) + { + ConstructionState cs = EnsureConstructionState (callerState); + LlvmIrGlobalVariable gv = EnsureGlobalVariable (variable); + IComparer> hashComparer = target.Is64Bit ? javaNameHash64Comparer : javaNameHash32Comparer; + + var entries = (List>)variable.Value; + foreach (StructureInstance entry in entries) { + entry.Instance.java_map_index = GetJavaEntryIndex (entry.Instance.JavaTypeMapEntry); + } - foreach (var kvp in mappingData) { - if (kvp.Value == null) { - throw new ArgumentException ("must not contain null values", nameof (mappingData)); + uint GetJavaEntryIndex (TypeMapJava javaEntry) + { + var key = new StructureInstance (typeMapJavaStructureInfo, javaEntry); + int idx = cs.JavaMap.BinarySearch (key, hashComparer); + if (idx < 0) { + throw new InvalidOperationException ($"Could not map entry '{javaEntry.JavaName}' to array index"); } - archState.Add (kvp.Key, new ArchGenerationState (kvp.Value)); + return (uint)idx; } } - public override void Init () + string? GetJavaHashesItemComment (LlvmIrVariable v, LlvmIrModuleTarget target, ulong index, object? value, object? callerState) { - foreach (var kvp in archState) { - InitMapModules (kvp.Value); - InitJavaMap (kvp.Value); + var cs = callerState as ConstructionState; + if (cs == null) { + throw new InvalidOperationException ("Internal error: construction state expected but not found"); } + + return $" {index}: 0x{value:x} => {cs.JavaMap[(int)index].Instance.JavaName}"; } - void InitJavaMap (ArchGenerationState state) + void GenerateAndSortJavaHashes (LlvmIrVariable variable, LlvmIrModuleTarget target, object? callerState) { + ConstructionState cs = EnsureConstructionState (callerState); + LlvmIrGlobalVariable gv = EnsureGlobalVariable (variable); + Type listType; + IList hashes; + if (target.Is64Bit) { + listType = typeof(List); + cs.JavaMap.Sort ((StructureInstance a, StructureInstance b) => a.Instance.JavaNameHash64.CompareTo (b.Instance.JavaNameHash64)); + + var list = new List (); + foreach (StructureInstance si in cs.JavaMap) { + list.Add (si.Instance.JavaNameHash64); + } + hashes = list; + } else { + listType = typeof(List); + cs.JavaMap.Sort ((StructureInstance a, StructureInstance b) => a.Instance.JavaNameHash32.CompareTo (b.Instance.JavaNameHash32)); + + var list = new List (); + foreach (StructureInstance si in cs.JavaMap) { + list.Add (si.Instance.JavaNameHash32); + } + hashes = list; + } + + gv.OverrideValueAndType (listType, hashes); + } + + ConstructionState EnsureConstructionState (object? callerState) + { + var cs = callerState as ConstructionState; + if (cs == null) { + throw new InvalidOperationException ("Internal error: construction state expected but not found"); + } + + return cs; + } + + void InitJavaMap (ConstructionState cs) + { + cs.JavaMap = new List> (); TypeMapJava map_entry; - foreach (TypeMapGenerator.TypeMapReleaseEntry entry in state.MappingData.JavaTypes) { - state.JavaNames.Add (entry.JavaName); + foreach (TypeMapGenerator.TypeMapReleaseEntry entry in mappingData.JavaTypes) { + cs.JavaNames.Add (entry.JavaName); map_entry = new TypeMapJava { module_index = (uint)entry.ModuleIndex, // UInt32.MaxValue, type_token_id = entry.SkipInJavaToManaged ? 0 : entry.Token, - java_name_index = (uint)(state.JavaNames.Count - 1), + java_name_index = (uint)(cs.JavaNames.Count - 1), JavaName = entry.JavaName, }; - state.JavaMap.Add (new StructureInstance (map_entry)); - state.JavaTypesByName.Add (map_entry.JavaName, map_entry); + cs.JavaMap.Add (new StructureInstance (typeMapJavaStructureInfo, map_entry)); + cs.JavaTypesByName.Add (map_entry.JavaName, map_entry); } } - void InitMapModules (ArchGenerationState state) + void InitMapModules (ConstructionState cs) { - foreach (TypeMapGenerator.ModuleReleaseData data in state.MappingData.Modules) { - string mapName = $"module{state.ModuleCounter++}_managed_to_java"; + cs.MapModules = new List> (); + foreach (TypeMapGenerator.ModuleReleaseData data in mappingData.Modules) { + string mapName = $"module{moduleCounter++}_managed_to_java"; string duplicateMapName; - if (data.DuplicateTypes.Count == 0) + if (data.DuplicateTypes.Count == 0) { duplicateMapName = String.Empty; - else + } else { duplicateMapName = $"{mapName}_duplicates"; + } var map_module = new TypeMapModule { MVID = data.Mvid, @@ -239,85 +345,69 @@ void InitMapModules (ArchGenerationState state) java_name_width = 0, }; - state.MapModules.Add (new StructureInstance (map_module)); + cs.MapModules.Add (new StructureInstance (typeMapModuleStructureInfo, map_module)); } } - protected override void MapStructures (LlvmIrGenerator generator) + void MapStructures (LlvmIrModule module) { - generator.MapStructure (); - typeMapJavaStructureInfo = generator.MapStructure (); - typeMapModuleStructureInfo = generator.MapStructure (); - typeMapModuleEntryStructureInfo = generator.MapStructure (); + typeMapJavaStructureInfo = module.MapStructure (); + typeMapModuleStructureInfo = module.MapStructure (); + typeMapModuleEntryStructureInfo = module.MapStructure (); } - // Prepare module map entries by sorting them on the managed token, and then mapping each entry to its corresponding Java type map index. - // Requires that `javaMap` is sorted on the type name hash. - void PrepareMapModuleData (ArchGenerationState state, string moduleDataSymbolLabel, IEnumerable moduleEntries, List allModulesData) + void PrepareMapModuleData (string moduleDataSymbolLabel, IEnumerable moduleEntries, ConstructionState cs) { var mapModuleEntries = new List> (); foreach (TypeMapGenerator.TypeMapReleaseEntry entry in moduleEntries) { + if (!cs.JavaTypesByName.TryGetValue (entry.JavaName, out TypeMapJava javaType)) { + throw new InvalidOperationException ($"Internal error: Java type '{entry.JavaName}' not found in cache"); + } + var map_entry = new TypeMapModuleEntry { + JavaTypeMapEntry = javaType, type_token_id = entry.Token, - java_map_index = GetJavaEntryIndex (entry.JavaName), + java_map_index = UInt32.MaxValue, // will be set later, when the target is known }; - mapModuleEntries.Add (new StructureInstance (map_entry)); + mapModuleEntries.Add (new StructureInstance (typeMapModuleEntryStructureInfo, map_entry)); } - mapModuleEntries.Sort ((StructureInstance a, StructureInstance b) => a.Obj.type_token_id.CompareTo (b.Obj.type_token_id)); - allModulesData.Add (new ModuleMapData (moduleDataSymbolLabel, mapModuleEntries)); - - uint GetJavaEntryIndex (string javaTypeName) - { - if (!state.JavaTypesByName.TryGetValue (javaTypeName, out TypeMapJava javaType)) { - throw new InvalidOperationException ($"INTERNAL ERROR: Java type '{javaTypeName}' not found in cache"); - } + mapModuleEntries.Sort ((StructureInstance a, StructureInstance b) => a.Instance.type_token_id.CompareTo (b.Instance.type_token_id)); + cs.AllModulesData.Add (new ModuleMapData (moduleDataSymbolLabel, mapModuleEntries)); + } - var key = new StructureInstance (javaType); - int idx = state.JavaMap.BinarySearch (key, javaNameHashComparer); - if (idx < 0) { - throw new InvalidOperationException ($"Could not map entry '{javaTypeName}' to array index"); + void PrepareModules (ConstructionState cs) + { + cs.AllModulesData = new List (); + foreach (StructureInstance moduleInstance in cs.MapModules) { + TypeMapModule module = moduleInstance.Instance; + PrepareMapModuleData (module.MapSymbolName, module.Data.Types, cs); + if (module.Data.DuplicateTypes.Count > 0) { + PrepareMapModuleData (module.DuplicateMapSymbolName, module.Data.DuplicateTypes, cs); } - - return (uint)idx; } } - // Generate hashes for all Java type names, then sort javaMap on the name hash. This has to be done in the writing phase because hashes - // will depend on architecture (or, actually, on its bitness) and may differ between architectures (they will be the same for all architectures - // with the same bitness) - (List allMapModulesData, List javaMapHashes) PrepareMapsForWriting (ArchGenerationState state, LlvmIrGenerator generator) + void HashJavaNames (ConstructionState cs) { - bool is64Bit = generator.Is64Bit; + // We generate both 32-bit and 64-bit hashes at the construction time. Which set will be used depends on the target. + // Java map list will also be sorted when the target is known + var hashes32 = new HashSet (); + var hashes64 = new HashSet (); // Generate Java type name hashes... - for (int i = 0; i < state.JavaMap.Count; i++) { - TypeMapJava entry = state.JavaMap[i].Obj; - entry.JavaNameHash = HashName (entry.JavaName); - } - - // ...sort them... - state.JavaMap.Sort ((StructureInstance a, StructureInstance b) => a.Obj.JavaNameHash.CompareTo (b.Obj.JavaNameHash)); + for (int i = 0; i < cs.JavaMap.Count; i++) { + TypeMapJava entry = cs.JavaMap[i].Instance; - var allMapModulesData = new List (); + // The cast is safe, xxHash will return a 32-bit value which (for convenience) was upcast to 64-bit + entry.JavaNameHash32 = (uint)HashName (entry.JavaName, is64Bit: false); + hashes32.Add (entry.JavaNameHash32); - // ...and match managed types to Java... - foreach (StructureInstance moduleInstance in state.MapModules) { - TypeMapModule module = moduleInstance.Obj; - PrepareMapModuleData (state, module.MapSymbolName, module.Data.Types, allMapModulesData); - if (module.Data.DuplicateTypes.Count > 0) { - PrepareMapModuleData (state, module.DuplicateMapSymbolName, module.Data.DuplicateTypes, allMapModulesData); - } + entry.JavaNameHash64 = HashName (entry.JavaName, is64Bit: true); + hashes64.Add (entry.JavaNameHash64); } - var javaMapHashes = new HashSet (); - foreach (StructureInstance entry in state.JavaMap) { - javaMapHashes.Add (entry.Obj.JavaNameHash); - } - - return (allMapModulesData, javaMapHashes.ToList ()); - - ulong HashName (string name) + ulong HashName (string name, bool is64Bit) { if (name.Length == 0) { return UInt64.MaxValue; @@ -325,10 +415,10 @@ ulong HashName (string name) // Native code (EmbeddedAssemblies::typemap_java_to_managed in embedded-assemblies.cc) will operate on wchar_t cast to a byte array, we need to do // the same - return HashBytes (Encoding.Unicode.GetBytes (name)); + return HashBytes (Encoding.Unicode.GetBytes (name), is64Bit); } - ulong HashBytes (byte[] bytes) + ulong HashBytes (byte[] bytes, bool is64Bit) { if (is64Bit) { return XxHash64.HashToUInt64 (bytes); @@ -337,87 +427,5 @@ ulong HashBytes (byte[] bytes) return (ulong)XxHash32.HashToUInt32 (bytes); } } - - protected override void Write (LlvmIrGenerator generator) - { - ArchGenerationState state; - - try { - state = archState[generator.TargetArch]; - } catch (KeyNotFoundException ex) { - throw new InvalidOperationException ($"Internal error: architecture {generator.TargetArch} has not been prepared for writing.", ex); - } - - generator.WriteVariable ("map_module_count", state.MappingData.MapModuleCount); - generator.WriteVariable ("java_type_count", state.JavaMap.Count); // must include the padding item, if any - - (List allMapModulesData, List javaMapHashes) = PrepareMapsForWriting (state, generator); - WriteMapModules (state, generator, allMapModulesData); - WriteJavaMap (state, generator, javaMapHashes); - } - - void WriteJavaMap (ArchGenerationState state, LlvmIrGenerator generator, List javaMapHashes) - { - generator.WriteEOL (); - generator.WriteEOL ("Java to managed map"); - - generator.WriteStructureArray ( - typeMapJavaStructureInfo, - state.JavaMap, - LlvmIrVariableOptions.GlobalConstant, - "map_java" - ); - - if (generator.Is64Bit) { - WriteHashes (javaMapHashes); - } else { - // A bit ugly, but simple. We know that hashes are really 32-bit, so we can cast without - // worrying. - var hashes = new List (javaMapHashes.Count); - foreach (ulong hash in javaMapHashes) { - hashes.Add ((uint)hash); - } - WriteHashes (hashes); - } - - generator.WriteArray (state.JavaNames, "java_type_names"); - - void WriteHashes (List hashes) where T: struct - { - generator.WriteArray ( - hashes, - LlvmIrVariableOptions.GlobalConstant, - "map_java_hashes", - (int idx, T value) => $"{idx}: 0x{value:x} => {state.JavaMap[idx].Obj.JavaName}" - ); - } - } - - void WriteMapModules (ArchGenerationState state, LlvmIrGenerator generator, List mapModulesData) - { - if (state.MapModules.Count == 0) { - return; - } - - generator.WriteEOL (); - generator.WriteEOL ("Map modules data"); - - foreach (ModuleMapData mmd in mapModulesData) { - generator.WriteStructureArray ( - typeMapModuleEntryStructureInfo, - mmd.Entries, - LlvmIrVariableOptions.LocalConstant, - mmd.SymbolLabel - ); - } - - generator.WriteEOL ("Map modules"); - generator.WriteStructureArray ( - typeMapModuleStructureInfo, - state.MapModules, - LlvmIrVariableOptions.GlobalWritable, - "map_modules" - ); - } } } From 6a92c296609b8216f7ad415085e896063ae97056 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 17 Jul 2023 23:13:54 +0200 Subject: [PATCH 31/71] Slowly moving forward --- .../Tasks/GenerateJavaStubs.cs | 92 +++++++++++++------ 1 file changed, 66 insertions(+), 26 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index daec746d31e..882628084d9 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -179,7 +179,7 @@ void RunDebugNoLinking () LogRunMode ("Debug, no linking"); XAAssemblyResolver resolver = MakeResolver (useMarshalMethods: false); - var assemblies = CollectInterestingAssemblies (ArchHere, resolver); + var assemblies = CollectInterestingAssemblies ((AndroidTargetArch arch) => resolver); throw new NotImplementedException (); } @@ -191,7 +191,7 @@ void RunDebugWithLinking () LogRunMode ("Debug, with linking"); XAAssemblyResolver resolver = MakeResolver (useMarshalMethods: false); - var assemblies = CollectInterestingAssemblies (ArchHere, resolver); + var assemblies = CollectInterestingAssemblies ((AndroidTargetArch arch) => resolver); throw new NotImplementedException (); } @@ -203,15 +203,15 @@ void RunReleaseNoLinking (bool useMarshalMethods) LogRunMode ("Release, no linking"); XAAssemblyResolver resolver = MakeResolver (useMarshalMethods); - var assemblies = CollectInterestingAssemblies (ArchHere, resolver); + var assemblies = CollectInterestingAssemblies ((AndroidTargetArch arch) => resolver); var state = new RunState { - UseMarshalMethods = useMarshalMethods, - AssemblySet = assemblies, - JavaTypeAssemblies = assemblies.JavaTypeAssemblies, - UserAssemblies = assemblies.UserAssemblies, + UseMarshalMethods = useMarshalMethods, + AssemblySet = assemblies, + JavaTypeAssemblies = assemblies.JavaTypeAssemblies, + UserAssemblies = assemblies.UserAssemblies, GenerateRidAgnosticParts = true, - Resolver = resolver, - TargetArch = ArchHere, + Resolver = resolver, + TargetArch = ArchHere, }; DoRun (state, out ApplicationConfigTaskState appConfState); RegisterApplicationConfigState (appConfState); @@ -222,9 +222,37 @@ void RunReleaseNoLinking (bool useMarshalMethods) void RunReleaseWithLinking (bool useMarshalMethods) { LogRunMode ("Release, with linking"); - XAAssemblyResolver resolver = MakeResolver (useMarshalMethods); - var assemblies = CollectInterestingAssemblies (AndroidTargetArch.None, resolver); - throw new NotImplementedException (); + + var resolvers = new Dictionary (); + foreach (string abi in SupportedAbis) { + // Each ABI gets its own resolver in this mode... + XAAssemblyResolver resolver = MakeResolver (useMarshalMethods); + resolvers.Add (MonoAndroidHelper.AbiToTargetArch (abi), resolver); + } + + // We don't check whether we have a resolver for `arch` on purpose, if it throws then it means we have a bug which + // should be fixed since there shouldn't be any assemblies passed to this task that belong in ABIs other than those + // specified in `SupportedAbis` + var assemblies = CollectInterestingAssemblies ((AndroidTargetArch arch) => resolvers[arch]); + bool first = true; + + foreach (var kvp in resolvers) { + var state = new RunState { + UseMarshalMethods = useMarshalMethods, + AssemblySet = assemblies, + JavaTypeAssemblies = assemblies.JavaTypeAssemblies[kvp.Key].Values, + UserAssemblies = assemblies.UserAssemblies[kvp.Key].Values, + GenerateRidAgnosticParts = first, + Resolver = kvp.Value, + TargetArch = kvp.Key, + }; + + DoRun (state, out ApplicationConfigTaskState appConfState); + if (first) { + RegisterApplicationConfigState (appConfState); + first = false; + } + } } void RegisterApplicationConfigState (ApplicationConfigTaskState appConfState) @@ -237,11 +265,10 @@ void LogRunMode (string mode) Log.LogDebugMessage ($"GenerateJavaStubs mode: {mode}"); } - T CollectInterestingAssemblies (AndroidTargetArch targetArch, XAAssemblyResolver resolver) where T: InputAssemblySet, new() + T CollectInterestingAssemblies (Func getResolver) where T: InputAssemblySet, new() { var assemblies = new T (); - bool hasExportReference = false; - bool haveMonoAndroid = false; + AndroidTargetArch targetArch; foreach (ITaskItem assembly in ResolvedAssemblies) { bool value; if (bool.TryParse (assembly.GetMetadata (AndroidSkipJavaStubGeneration), out value) && value) { @@ -251,11 +278,9 @@ void LogRunMode (string mode) bool addAssembly = false; string fileName = Path.GetFileName (assembly.ItemSpec); - if (!hasExportReference && String.Compare ("Mono.Android.Export.dll", fileName, StringComparison.OrdinalIgnoreCase) == 0) { - hasExportReference = true; + if (String.Compare ("Mono.Android.Export.dll", fileName, StringComparison.OrdinalIgnoreCase) == 0) { addAssembly = true; - } else if (!haveMonoAndroid && String.Compare ("Mono.Android.dll", fileName, StringComparison.OrdinalIgnoreCase) == 0) { - haveMonoAndroid = true; + } else if (String.Compare ("Mono.Android.dll", fileName, StringComparison.OrdinalIgnoreCase) == 0) { addAssembly = true; } else if (MonoAndroidHelper.FrameworkAssembliesToTreatAsUserAssemblies.Contains (fileName)) { if (!bool.TryParse (assembly.GetMetadata (AndroidSkipJavaStubGeneration), out value) || !value) { @@ -269,25 +294,39 @@ void LogRunMode (string mode) assemblies.AddJavaTypeAssembly (assembly); } - resolver.Load (targetArch, assembly.ItemSpec); + targetArch = MonoAndroidHelper.GetTargetArch (assembly); + getResolver (targetArch).Load (targetArch, assembly.ItemSpec); } // However we only want to look for JLO types in user code for Java stub code generation - foreach (ITaskItem asm in ResolvedUserAssemblies) { - if (bool.TryParse (asm.GetMetadata (AndroidSkipJavaStubGeneration), out bool value) && value) { - Log.LogDebugMessage ($"Skipping Java Stub Generation for {asm.ItemSpec}"); + foreach (ITaskItem assembly in ResolvedUserAssemblies) { + if (bool.TryParse (assembly.GetMetadata (AndroidSkipJavaStubGeneration), out bool value) && value) { + Log.LogDebugMessage ($"Skipping Java Stub Generation for {assembly.ItemSpec}"); continue; } - resolver.Load (targetArch, asm.ItemSpec); - assemblies.AddJavaTypeAssembly (asm); - assemblies.AddUserAssembly (asm); + targetArch = MonoAndroidHelper.GetTargetArch (assembly); + getResolver (targetArch).Load (targetArch, assembly.ItemSpec); + + assemblies.AddJavaTypeAssembly (assembly); + assemblies.AddUserAssembly (assembly); } return assemblies; } void DoRun (RunState state, out ApplicationConfigTaskState? appConfState) { + Log.LogDebugMessage ($"DoRun for arch {state.TargetArch}"); + Log.LogDebugMessage ("Java type assemblies:"); + foreach (ITaskItem assembly in state.JavaTypeAssemblies) { + Log.LogDebugMessage ($" {assembly.ItemSpec}"); + } + + + Log.LogDebugMessage ("User assemblies:"); + foreach (ITaskItem assembly in state.UserAssemblies) { + Log.LogDebugMessage ($" {assembly.ItemSpec}"); + } PackageNamingPolicy pnp; JavaNativeTypeManager.PackageNamingPolicy = Enum.TryParse (PackageNamingPolicy, out pnp) ? pnp : PackageNamingPolicyEnum.LowercaseCrc64; @@ -583,6 +622,7 @@ void SaveResource (string resource, string filename, string destDir, Func types, TypeDefinitionCache cache, out ApplicationConfigTaskState appConfState) { + Log.LogDebugMessage ($"Generating typemaps for arch {targetArch}, {types.Count} types"); var tmg = new TypeMapGenerator (targetArch, Log, SupportedAbis); if (!tmg.Generate (Debug, SkipJniAddNativeMethodRegistrationAttributeScan, types, cache, TypemapOutputDirectory, GenerateNativeAssembly, out appConfState)) { throw new XamarinAndroidException (4308, Properties.Resources.XA4308); From 27814337f4362a177e2d90956c55db4b8910c0d9 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 18 Jul 2023 16:20:17 +0200 Subject: [PATCH 32/71] Release+Lining typemaps --- .../Tasks/GenerateJavaStubs.cs | 53 +++---- .../Utilities/AcwMapWriter.cs | 5 +- .../Utilities/ManifestDocument.cs | 15 +- .../Utilities/TypeMapGenerator.cs | 130 ++++++------------ .../Utilities/XAJavaTypeScanner.cs | 50 ++----- 5 files changed, 88 insertions(+), 165 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 882628084d9..4ddfa58f18e 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -2,14 +2,12 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.IO; using System.IO.MemoryMappedFiles; using System.Linq; using System.Reflection; using System.Text; using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; using Mono.Cecil; @@ -179,7 +177,7 @@ void RunDebugNoLinking () LogRunMode ("Debug, no linking"); XAAssemblyResolver resolver = MakeResolver (useMarshalMethods: false); - var assemblies = CollectInterestingAssemblies ((AndroidTargetArch arch) => resolver); + var assemblies = CollectInterestingAssemblies (allowAbiSpecific: false, (AndroidTargetArch arch) => resolver); throw new NotImplementedException (); } @@ -191,7 +189,7 @@ void RunDebugWithLinking () LogRunMode ("Debug, with linking"); XAAssemblyResolver resolver = MakeResolver (useMarshalMethods: false); - var assemblies = CollectInterestingAssemblies ((AndroidTargetArch arch) => resolver); + var assemblies = CollectInterestingAssemblies (allowAbiSpecific: true, (AndroidTargetArch arch) => resolver); throw new NotImplementedException (); } @@ -203,7 +201,7 @@ void RunReleaseNoLinking (bool useMarshalMethods) LogRunMode ("Release, no linking"); XAAssemblyResolver resolver = MakeResolver (useMarshalMethods); - var assemblies = CollectInterestingAssemblies ((AndroidTargetArch arch) => resolver); + var assemblies = CollectInterestingAssemblies (allowAbiSpecific: false, (AndroidTargetArch arch) => resolver); var state = new RunState { UseMarshalMethods = useMarshalMethods, AssemblySet = assemblies, @@ -233,7 +231,7 @@ void RunReleaseWithLinking (bool useMarshalMethods) // We don't check whether we have a resolver for `arch` on purpose, if it throws then it means we have a bug which // should be fixed since there shouldn't be any assemblies passed to this task that belong in ABIs other than those // specified in `SupportedAbis` - var assemblies = CollectInterestingAssemblies ((AndroidTargetArch arch) => resolvers[arch]); + var assemblies = CollectInterestingAssemblies (allowAbiSpecific: true, (AndroidTargetArch arch) => resolvers[arch]); bool first = true; foreach (var kvp in resolvers) { @@ -265,7 +263,7 @@ void LogRunMode (string mode) Log.LogDebugMessage ($"GenerateJavaStubs mode: {mode}"); } - T CollectInterestingAssemblies (Func getResolver) where T: InputAssemblySet, new() + T CollectInterestingAssemblies (bool allowAbiSpecific, Func getResolver) where T: InputAssemblySet, new() { var assemblies = new T (); AndroidTargetArch targetArch; @@ -294,7 +292,7 @@ void LogRunMode (string mode) assemblies.AddJavaTypeAssembly (assembly); } - targetArch = MonoAndroidHelper.GetTargetArch (assembly); + targetArch = GetTargetArch (assembly); getResolver (targetArch).Load (targetArch, assembly.ItemSpec); } @@ -305,13 +303,24 @@ void LogRunMode (string mode) continue; } - targetArch = MonoAndroidHelper.GetTargetArch (assembly); + targetArch = GetTargetArch (assembly); getResolver (targetArch).Load (targetArch, assembly.ItemSpec); assemblies.AddJavaTypeAssembly (assembly); assemblies.AddUserAssembly (assembly); } + return assemblies; + + AndroidTargetArch GetTargetArch (ITaskItem assembly) + { + AndroidTargetArch targetArch = MonoAndroidHelper.GetTargetArch (assembly); + if (!allowAbiSpecific && targetArch != AndroidTargetArch.None) { + throw new InvalidOperationException ($"Internal error: ABI-specific assemblies are not allowed in this build configuration"); + } + + return targetArch; + } } void DoRun (RunState state, out ApplicationConfigTaskState? appConfState) @@ -335,17 +344,17 @@ void DoRun (RunState state, out ApplicationConfigTaskState? appConfState) var scanner = new XAJavaTypeScanner (Log, cache) { ErrorOnCustomJavaObject = ErrorOnCustomJavaObject, }; - List allJavaTypes = scanner.GetJavaTypes (state.JavaTypeAssemblies, state.Resolver); - var javaTypes = new List (); + ICollection allJavaTypes = scanner.GetJavaTypes (state.JavaTypeAssemblies, state.Resolver); + var javaTypes = new List (); - foreach (JavaType jt in allJavaTypes) { + foreach (TypeDefinition javaType in allJavaTypes) { // Whem marshal methods are in use we do not want to skip non-user assemblies (such as Mono.Android) - we need to generate JCWs for them during // application build, unlike in Debug configuration or when marshal methods are disabled, in which case we use JCWs generated during Xamarin.Android // build and stored in a jar file. - if ((!state.UseMarshalMethods && !state.AssemblySet.IsUserAssembly (jt.Type.Module.Assembly.Name.Name)) || JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (jt.Type, cache)) { + if ((!state.UseMarshalMethods && !state.AssemblySet.IsUserAssembly (javaType.Module.Assembly.Name.Name)) || JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (javaType, cache)) { continue; } - javaTypes.Add (jt); + javaTypes.Add (javaType); } MarshalMethodsClassifier? classifier = null; @@ -402,12 +411,11 @@ void DoRun (RunState state, out ApplicationConfigTaskState? appConfState) } } - void CreateAdditionalJavaSources (List javaTypes, TypeDefinitionCache cache, MarshalMethodsClassifier? classifier) + void CreateAdditionalJavaSources (ICollection javaTypes, TypeDefinitionCache cache, MarshalMethodsClassifier? classifier) { StringWriter regCallsWriter = new StringWriter (); regCallsWriter.WriteLine ("\t\t// Application and Instrumentation ACWs must be registered first."); - foreach (JavaType jt in javaTypes) { - TypeDefinition type = jt.Type; + foreach (TypeDefinition type in javaTypes) { if (JavaNativeTypeManager.IsApplication (type, cache) || JavaNativeTypeManager.IsInstrumentation (type, cache)) { if (classifier != null && !classifier.FoundDynamicallyRegisteredMethods (type)) { continue; @@ -426,7 +434,7 @@ void CreateAdditionalJavaSources (List javaTypes, TypeDefinitionCache template => template.Replace ("// REGISTER_APPLICATION_AND_INSTRUMENTATION_CLASSES_HERE", regCallsWriter.ToString ())); } - void UpdateAndroidManifest (RunState state, TypeDefinitionCache cache, List allJavaTypes) + void UpdateAndroidManifest (RunState state, TypeDefinitionCache cache, ICollection allJavaTypes) { var manifest = new ManifestDocument (ManifestTemplate) { PackageName = PackageName, @@ -468,7 +476,7 @@ void UpdateAndroidManifest (RunState state, TypeDefinitionCache cache, List javaTypes, TypeDefinitionCache cache) + void WriteAcwMaps (List javaTypes, TypeDefinitionCache cache) { var writer = new AcwMapWriter (Log, AcwMapFile); writer.Write (javaTypes, cache); @@ -504,7 +512,7 @@ AssemblyDefinition LoadAssembly (string path, XAAssemblyResolver? resolver = nul } } - bool CreateJavaSources (IEnumerable newJavaTypes, TypeDefinitionCache cache, MarshalMethodsClassifier classifier, bool useMarshalMethods) + bool CreateJavaSources (IEnumerable newJavaTypes, TypeDefinitionCache cache, MarshalMethodsClassifier classifier, bool useMarshalMethods) { if (useMarshalMethods && classifier == null) { throw new ArgumentNullException (nameof (classifier)); @@ -516,8 +524,7 @@ bool CreateJavaSources (IEnumerable newJavaTypes, TypeDefinitionCache bool generateOnCreateOverrides = int.Parse (AndroidSdkPlatform) <= 10; bool ok = true; - foreach (JavaType jt in newJavaTypes) { - TypeDefinition t = jt.Type; // JCW generator doesn't care about ABI-specific types or token ids + foreach (TypeDefinition t in newJavaTypes) { if (t.IsInterface) { // Interfaces are in typemap but they shouldn't have JCW generated for them continue; @@ -620,7 +627,7 @@ void SaveResource (string resource, string filename, string destDir, Func types, TypeDefinitionCache cache, out ApplicationConfigTaskState appConfState) + void WriteTypeMappings (AndroidTargetArch targetArch, ICollection types, TypeDefinitionCache cache, out ApplicationConfigTaskState appConfState) { Log.LogDebugMessage ($"Generating typemaps for arch {targetArch}, {types.Count} types"); var tmg = new TypeMapGenerator (targetArch, Log, SupportedAbis); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AcwMapWriter.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AcwMapWriter.cs index d6f28f0625a..315387ecbbf 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AcwMapWriter.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AcwMapWriter.cs @@ -21,7 +21,7 @@ public AcwMapWriter (TaskLoggingHelper log, string acwMapFile) AcwMapFile = acwMapFile; } - public void Write (List javaTypes, TypeDefinitionCache cache) + public void Write (ICollection javaTypes, TypeDefinitionCache cache) { // We need to save a map of .NET type -> ACW type for resource file fixups var managed = new Dictionary (javaTypes.Count, StringComparer.Ordinal); @@ -31,8 +31,7 @@ public void Write (List javaTypes, TypeDefinitionCache cache) var javaConflicts = new Dictionary> (0, StringComparer.Ordinal); using (var acw_map = MemoryStreamPool.Shared.CreateStreamWriter ()) { - foreach (JavaType jt in javaTypes) { - TypeDefinition type = jt.Type; + foreach (TypeDefinition type in javaTypes) { string managedKey = type.FullName.Replace ('/', '.'); string javaKey = JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.'); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs index 8a1b8996794..bbf926358dd 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs @@ -255,7 +255,7 @@ void ReorderActivityAliases (TaskLoggingHelper log, XElement app) } } - public IList Merge (TaskLoggingHelper log, TypeDefinitionCache cache, List subclasses, string applicationClass, bool embed, string bundledWearApplicationName, IEnumerable mergedManifestDocuments) + public IList Merge (TaskLoggingHelper log, TypeDefinitionCache cache, ICollection subclasses, string applicationClass, bool embed, string bundledWearApplicationName, IEnumerable mergedManifestDocuments) { var manifest = doc.Root; @@ -330,8 +330,7 @@ public IList Merge (TaskLoggingHelper log, TypeDefinitionCache cache, Li throw new InvalidOperationException (string.Format ("The targetSdkVersion ({0}) is not a valid API level", targetSdkVersion)); int targetSdkVersionValue = tryTargetSdkVersion.Value; - foreach (JavaType jt in subclasses) { - TypeDefinition t = jt.Type; + foreach (TypeDefinition t in subclasses) { if (t.IsAbstract) continue; @@ -568,7 +567,7 @@ Func GetGenerator (T return null; } - XElement CreateApplicationElement (XElement manifest, string applicationClass, List subclasses, TypeDefinitionCache cache) + XElement CreateApplicationElement (XElement manifest, string applicationClass, ICollection subclasses, TypeDefinitionCache cache) { var application = manifest.Descendants ("application").FirstOrDefault (); @@ -592,8 +591,7 @@ XElement CreateApplicationElement (XElement manifest, string applicationClass, L List typeAttr = new List (); List typeUsesLibraryAttr = new List (); List typeUsesConfigurationAttr = new List (); - foreach (JavaType jt in subclasses) { - TypeDefinition t = jt.Type; + foreach (TypeDefinition t in subclasses) { ApplicationAttribute aa = ApplicationAttribute.FromCustomAttributeProvider (t); if (aa == null) continue; @@ -925,7 +923,7 @@ void AddSupportsGLTextures (XElement application, TypeDefinitionCache cache) } } - void AddInstrumentations (XElement manifest, IList subclasses, int targetSdkVersion, TypeDefinitionCache cache) + void AddInstrumentations (XElement manifest, ICollection subclasses, int targetSdkVersion, TypeDefinitionCache cache) { var assemblyAttrs = Assemblies.SelectMany (path => InstrumentationAttribute.FromCustomAttributeProvider (Resolver.GetAssembly (path))); @@ -938,8 +936,7 @@ void AddInstrumentations (XElement manifest, IList subclasses, int tar manifest.Add (ia.ToElement (PackageName, cache)); } - foreach (JavaType jt in subclasses) { - TypeDefinition type = jt.Type; + foreach (TypeDefinition type in subclasses) { if (type.IsSubclassOf ("Android.App.Instrumentation", cache)) { var xe = InstrumentationFromTypeDefinition (type, JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.'), cache); if (xe != null) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs index 71282ae7fbd..8819832ac3a 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs @@ -97,27 +97,13 @@ sealed class ReleaseGenerationState public readonly Dictionary KnownAssemblies; public readonly Dictionary MvidCache; - public readonly IDictionary> TempModules; - - // Just a convenient way to access one of the temp modules dictionaries, to be used when dealing with ABI-agnostic - // types in ProcessReleaseType. - public readonly Dictionary TempModulesAbiAgnostic; + public readonly Dictionary TempModules; public ReleaseGenerationState (string[] supportedAbis) { KnownAssemblies = new Dictionary (StringComparer.Ordinal); MvidCache = new Dictionary (); - - var tempModules = new Dictionary> (); - foreach (string abi in supportedAbis) { - var dict = new Dictionary (); - if (TempModulesAbiAgnostic == null) { - TempModulesAbiAgnostic = dict; - } - tempModules.Add (MonoAndroidHelper.AbiToTargetArch (abi), dict); - } - - TempModules = new ReadOnlyDictionary> (tempModules); + TempModules = new Dictionary (); } public void AddKnownAssembly (TypeDefinition td) @@ -169,7 +155,7 @@ void UpdateApplicationConfig (TypeDefinition javaType, ApplicationConfigTaskStat } } - public bool Generate (bool debugBuild, bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, bool generateNativeAssembly, out ApplicationConfigTaskState appConfState) + public bool Generate (bool debugBuild, bool skipJniAddNativeMethodRegistrationAttributeScan, ICollection javaTypes, TypeDefinitionCache cache, string outputDirectory, bool generateNativeAssembly, out ApplicationConfigTaskState appConfState) { if (String.IsNullOrEmpty (outputDirectory)) throw new ArgumentException ("must not be null or empty", nameof (outputDirectory)); @@ -190,7 +176,7 @@ public bool Generate (bool debugBuild, bool skipJniAddNativeMethodRegistrationAt return GenerateRelease (skipJniAddNativeMethodRegistrationAttributeScan, javaTypes, cache, typemapsOutputDirectory, appConfState); } - bool GenerateDebug (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, bool generateNativeAssembly, ApplicationConfigTaskState appConfState) + bool GenerateDebug (bool skipJniAddNativeMethodRegistrationAttributeScan, ICollection javaTypes, TypeDefinitionCache cache, string outputDirectory, bool generateNativeAssembly, ApplicationConfigTaskState appConfState) { if (generateNativeAssembly) { return GenerateDebugNativeAssembly (skipJniAddNativeMethodRegistrationAttributeScan, javaTypes, cache, outputDirectory, appConfState); @@ -198,15 +184,14 @@ bool GenerateDebug (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, ApplicationConfigTaskState appConfState) + bool GenerateDebugFiles (bool skipJniAddNativeMethodRegistrationAttributeScan, ICollection javaTypes, TypeDefinitionCache cache, string outputDirectory, ApplicationConfigTaskState appConfState) { var modules = new Dictionary (StringComparer.Ordinal); int maxModuleFileNameWidth = 0; int maxModuleNameWidth = 0; var javaDuplicates = new Dictionary> (StringComparer.Ordinal); - foreach (JavaType jt in javaTypes) { - TypeDefinition td = jt.Type; + foreach (TypeDefinition td in javaTypes) { UpdateApplicationConfig (td, appConfState); string moduleName = td.Module.Assembly.Name.Name; ModuleDebugData module; @@ -264,14 +249,13 @@ bool GenerateDebugFiles (bool skipJniAddNativeMethodRegistrationAttributeScan, L return true; } - bool GenerateDebugNativeAssembly (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, ApplicationConfigTaskState appConfState) + bool GenerateDebugNativeAssembly (bool skipJniAddNativeMethodRegistrationAttributeScan, ICollection javaTypes, TypeDefinitionCache cache, string outputDirectory, ApplicationConfigTaskState appConfState) { var javaToManaged = new List (); var managedToJava = new List (); var javaDuplicates = new Dictionary> (StringComparer.Ordinal); - foreach (JavaType jt in javaTypes) { - TypeDefinition td = jt.Type; + foreach (TypeDefinition td in javaTypes) { UpdateApplicationConfig (td, appConfState); TypeMapDebugEntry entry = GetDebugEntry (td, cache); @@ -376,7 +360,7 @@ string GetManagedTypeName (TypeDefinition td) return $"{managedTypeName}, {td.Module.Assembly.Name.Name}"; } - void ProcessReleaseType (ReleaseGenerationState state, TypeDefinition td, AndroidTargetArch typeArch, ApplicationConfigTaskState appConfState, TypeDefinitionCache cache) + void ProcessReleaseType (ReleaseGenerationState state, TypeDefinition td, ApplicationConfigTaskState appConfState, TypeDefinitionCache cache) { UpdateApplicationConfig (td, appConfState); @@ -391,21 +375,7 @@ void ProcessReleaseType (ReleaseGenerationState state, TypeDefinition td, Androi state.MvidCache.Add (td.Module.Mvid, moduleUUID); } - bool abiAgnosticType = typeArch == AndroidTargetArch.None; - Dictionary tempModules; - if (abiAgnosticType) { - tempModules = state.TempModulesAbiAgnostic; - } else { - // It will throw if `typeArch` isn't in the dictionary. This is intentional, since we must have no TypeDefinition entries for architectures not - // mentioned in `supportedAbis`. - try { - tempModules = state.TempModules[typeArch]; - } catch (KeyNotFoundException ex) { - throw new InvalidOperationException ($"Internal error: cannot process type specific to architecture '{typeArch}', since that architecture isn't mentioned in the set of supported ABIs", ex); - } - } - - if (!tempModules.TryGetValue (moduleUUID, out ModuleReleaseData moduleData)) { + if (!state.TempModules.TryGetValue (moduleUUID, out ModuleReleaseData moduleData)) { moduleData = new ModuleReleaseData { Mvid = td.Module.Mvid, MvidBytes = moduleUUID, @@ -415,15 +385,8 @@ void ProcessReleaseType (ReleaseGenerationState state, TypeDefinition td, Androi DuplicateTypes = new List (), }; - if (abiAgnosticType) { - // ABI-agnostic types must be added to all the ABIs - foreach (var kvp in state.TempModules) { - kvp.Value.Add (moduleUUID, moduleData); - } - } else { - // ABI-specific types are added only to their respective tempModules - tempModules.Add (moduleUUID, moduleData); - } + // ABI-specific types are added only to their respective tempModules + state.TempModules.Add (moduleUUID, moduleData); } string javaName = Java.Interop.Tools.TypeNameMappings.JavaNativeTypeManager.ToJniName (td, cache); @@ -451,42 +414,31 @@ void ProcessReleaseType (ReleaseGenerationState state, TypeDefinition td, Androi } } - bool GenerateRelease (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, ApplicationConfigTaskState appConfState) + bool GenerateRelease (bool skipJniAddNativeMethodRegistrationAttributeScan, ICollection javaTypes, TypeDefinitionCache cache, string outputDirectory, ApplicationConfigTaskState appConfState) { var state = new ReleaseGenerationState (supportedAbis); - foreach (JavaType jt in javaTypes) { - if (!jt.IsABiSpecific) { - ProcessReleaseType (state, jt.Type, AndroidTargetArch.None, appConfState, cache); - continue; - } - - foreach (var kvp in jt.PerAbiTypes) { - ProcessReleaseType (state, kvp.Value, kvp.Key, appConfState, cache); - } + foreach (TypeDefinition td in javaTypes) { + ProcessReleaseType (state, td, appConfState, cache); } - foreach (var kvp in state.TempModules) { - AndroidTargetArch arch = kvp.Key; - Dictionary tempModules = kvp.Value; - ModuleReleaseData[] modules = tempModules.Values.ToArray (); - Array.Sort (modules, new ModuleUUIDArrayComparer ()); + ModuleReleaseData[] modules = state.TempModules.Values.ToArray (); + Array.Sort (modules, new ModuleUUIDArrayComparer ()); - foreach (ModuleReleaseData module in modules) { - if (module.TypesScratch.Count == 0) { - module.Types = Array.Empty (); - continue; - } - - // No need to sort here, the LLVM IR generator will compute hashes and sort - // the array on write. - module.Types = module.TypesScratch.Values.ToArray (); + foreach (ModuleReleaseData module in modules) { + if (module.TypesScratch.Count == 0) { + module.Types = Array.Empty (); + continue; } - var composer = new TypeMappingReleaseNativeAssemblyGenerator (new NativeTypeMappingData (modules)); - GenerateNativeAssembly (arch, composer, composer.Construct (), outputDirectory); + // No need to sort here, the LLVM IR generator will compute hashes and sort + // the array on write. + module.Types = module.TypesScratch.Values.ToArray (); } + var composer = new TypeMappingReleaseNativeAssemblyGenerator (new NativeTypeMappingData (modules)); + GenerateNativeAssembly (composer, composer.Construct (), outputDirectory); + return true; } @@ -497,27 +449,27 @@ bool ShouldSkipInJavaToManaged (TypeDefinition td) string GetOutputFilePath (string baseFileName, string abi) => $"{baseFileName}.{abi}.ll"; - void GenerateNativeAssembly (AndroidTargetArch arch, LLVMIR.LlvmIrComposer composer, LLVMIR.LlvmIrModule typeMapModule, string baseFileName) + void GenerateNativeAssembly (LLVMIR.LlvmIrComposer composer, LLVMIR.LlvmIrModule typeMapModule, string baseFileName) { WriteNativeAssembly ( - arch, + targetArch, composer, typeMapModule, - GetOutputFilePath (baseFileName, ArchToAbi (arch)) + GetOutputFilePath (baseFileName, ArchToAbi (targetArch)) ); } - void GenerateNativeAssembly (LLVMIR.LlvmIrComposer composer, LLVMIR.LlvmIrModule typeMapModule, string baseFileName) - { - foreach (string abi in supportedAbis) { - WriteNativeAssembly ( - GeneratePackageManagerJava.GetAndroidTargetArchForAbi (abi), - composer, - typeMapModule, - GetOutputFilePath (baseFileName, abi) - ); - } - } + // void GenerateNativeAssembly (LLVMIR.LlvmIrComposer composer, LLVMIR.LlvmIrModule typeMapModule, string baseFileName) + // { + // foreach (string abi in supportedAbis) { + // WriteNativeAssembly ( + // GeneratePackageManagerJava.GetAndroidTargetArchForAbi (abi), + // composer, + // typeMapModule, + // GetOutputFilePath (baseFileName, abi) + // ); + // } + // } void WriteNativeAssembly (AndroidTargetArch arch, LLVMIR.LlvmIrComposer composer, LLVMIR.LlvmIrModule typeMapModule, string outputFile) { diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs index 5f965ab46ff..c6eb367f638 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs @@ -28,20 +28,6 @@ public JavaType (TypeDefinition type, IDictionary PerAbi; - - public bool IsAbiSpecific => !PerAbi.ContainsKey (AndroidTargetArch.None); - - public TypeData (TypeDefinition firstType) - { - FirstType = firstType; - PerAbi = new Dictionary (); - } - } - public bool ErrorOnCustomJavaObject { get; set; } TaskLoggingHelper log; @@ -53,51 +39,33 @@ public XAJavaTypeScanner (TaskLoggingHelper log, TypeDefinitionCache cache) this.cache = cache; } - public List GetJavaTypes (ICollection inputAssemblies, XAAssemblyResolver resolver) + public ICollection GetJavaTypes (ICollection inputAssemblies, XAAssemblyResolver resolver) { - var types = new Dictionary (StringComparer.Ordinal); + var types = new Dictionary (StringComparer.Ordinal); foreach (ITaskItem asmItem in inputAssemblies) { AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (asmItem); AssemblyDefinition asmdef = resolver.Load (arch, asmItem.ItemSpec); foreach (ModuleDefinition md in asmdef.Modules) { foreach (TypeDefinition td in md.Types) { - AddJavaType (td, types, arch); + AddJavaType (td, types); } } } - var ret = new List (); - foreach (var kvp in types) { - ret.Add (new JavaType (kvp.Value.FirstType, kvp.Value.IsAbiSpecific ? kvp.Value.PerAbi : null)); - } - - return ret; + return types.Values; } - void AddJavaType (TypeDefinition type, Dictionary types, AndroidTargetArch arch) + void AddJavaType (TypeDefinition type, Dictionary types) { if (type.IsSubclassOf ("Java.Lang.Object", cache) || type.IsSubclassOf ("Java.Lang.Throwable", cache) || (type.IsInterface && type.ImplementsInterface ("Java.Interop.IJavaPeerable", cache))) { // For subclasses of e.g. Android.App.Activity. string typeName = type.GetPartialAssemblyQualifiedName (cache); - if (!types.TryGetValue (typeName, out TypeData typeData)) { - typeData = new TypeData (type); - types.Add (typeName, typeData); - } - - if (typeData.PerAbi.ContainsKey (AndroidTargetArch.None)) { - if (arch == AndroidTargetArch.None) { - throw new InvalidOperationException ($"Duplicate type '{type.FullName}' in assembly {type.Module.FileName}"); - } - - throw new InvalidOperationException ($"Previously added type '{type.FullName}' was in ABI-agnostic assembly, new one comes from ABI {arch} assembly"); - } - - if (typeData.PerAbi.ContainsKey (arch)) { - throw new InvalidOperationException ($"Duplicate type '{type.FullName}' in assembly {type.Module.FileName}, for ABI {arch}"); + if (types.ContainsKey (typeName)) { + return; } - typeData.PerAbi.Add (arch, type); + types.Add (typeName, type); } else if (type.IsClass && !type.IsSubclassOf ("System.Exception", cache) && type.ImplementsInterface ("Android.Runtime.IJavaObject", cache)) { string message = $"XA4212: Type `{type.FullName}` implements `Android.Runtime.IJavaObject` but does not inherit `Java.Lang.Object` or `Java.Lang.Throwable`. This is not supported."; @@ -114,7 +82,7 @@ void AddJavaType (TypeDefinition type, Dictionary types, Andro } foreach (TypeDefinition nested in type.NestedTypes) { - AddJavaType (nested, types, arch); + AddJavaType (nested, types); } } } From 49e6ff4bfeffb71153019125bf202f9febcf2fcf Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 18 Jul 2023 23:05:02 +0200 Subject: [PATCH 33/71] Need a different approach... Tomorrow --- .../Tasks/GenerateJavaStubs.cs | 88 +++++++++---------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 4ddfa58f18e..b0f1d32527a 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -27,12 +27,12 @@ public class GenerateJavaStubs : AndroidTask { sealed class RunState { - public XAAssemblyResolver Resolver { get; set; } - public ICollection JavaTypeAssemblies { get; set; } - public ICollection UserAssemblies { get; set; } - public InputAssemblySet AssemblySet { get; set; } - public bool UseMarshalMethods { get; set; } - public AndroidTargetArch TargetArch { get; set; } = AndroidTargetArch.None; + public Dictionary Resolvers { get; set; } + public ICollection JavaTypeAssemblies { get; set; } + public ICollection UserAssemblies { get; set; } + public InputAssemblySet AssemblySet { get; set; } + public bool UseMarshalMethods { get; set; } + public AndroidTargetArch TargetArch { get; set; } = AndroidTargetArch.None; /// /// If `true`, generate code/data that doesn't depend on a specific RID (e.g. ACW maps or JCWs) @@ -173,11 +173,9 @@ void Run () // Typemaps don't use MVIDs or metadata tokens void RunDebugNoLinking () { - const AndroidTargetArch ArchHere = AndroidTargetArch.None; - LogRunMode ("Debug, no linking"); - XAAssemblyResolver resolver = MakeResolver (useMarshalMethods: false); - var assemblies = CollectInterestingAssemblies (allowAbiSpecific: false, (AndroidTargetArch arch) => resolver); + Dictionary resolvers = MakeResolvers (includeRidAgnostic: true, useMarshalMethods: false); + var assemblies = CollectInterestingAssemblies (resolvers); throw new NotImplementedException (); } @@ -185,11 +183,9 @@ void RunDebugNoLinking () // Typemaps don't use MVIDs or metadata tokens void RunDebugWithLinking () { - const AndroidTargetArch ArchHere = AndroidTargetArch.None; - LogRunMode ("Debug, with linking"); - XAAssemblyResolver resolver = MakeResolver (useMarshalMethods: false); - var assemblies = CollectInterestingAssemblies (allowAbiSpecific: true, (AndroidTargetArch arch) => resolver); + Dictionary resolvers = MakeResolvers (includeRidAgnostic: false, useMarshalMethods: false); + var assemblies = CollectInterestingAssemblies (resolvers); throw new NotImplementedException (); } @@ -200,15 +196,15 @@ void RunReleaseNoLinking (bool useMarshalMethods) const AndroidTargetArch ArchHere = AndroidTargetArch.None; LogRunMode ("Release, no linking"); - XAAssemblyResolver resolver = MakeResolver (useMarshalMethods); - var assemblies = CollectInterestingAssemblies (allowAbiSpecific: false, (AndroidTargetArch arch) => resolver); + Dictionary resolvers = MakeResolvers (includeRidAgnostic: true, useMarshalMethods); + var assemblies = CollectInterestingAssemblies (resolvers); var state = new RunState { UseMarshalMethods = useMarshalMethods, AssemblySet = assemblies, JavaTypeAssemblies = assemblies.JavaTypeAssemblies, UserAssemblies = assemblies.UserAssemblies, GenerateRidAgnosticParts = true, - Resolver = resolver, + Resolvers = resolvers, TargetArch = ArchHere, }; DoRun (state, out ApplicationConfigTaskState appConfState); @@ -221,17 +217,8 @@ void RunReleaseWithLinking (bool useMarshalMethods) { LogRunMode ("Release, with linking"); - var resolvers = new Dictionary (); - foreach (string abi in SupportedAbis) { - // Each ABI gets its own resolver in this mode... - XAAssemblyResolver resolver = MakeResolver (useMarshalMethods); - resolvers.Add (MonoAndroidHelper.AbiToTargetArch (abi), resolver); - } - - // We don't check whether we have a resolver for `arch` on purpose, if it throws then it means we have a bug which - // should be fixed since there shouldn't be any assemblies passed to this task that belong in ABIs other than those - // specified in `SupportedAbis` - var assemblies = CollectInterestingAssemblies (allowAbiSpecific: true, (AndroidTargetArch arch) => resolvers[arch]); + Dictionary resolvers = MakeResolvers (includeRidAgnostic: false, useMarshalMethods); + var assemblies = CollectInterestingAssemblies (resolvers); bool first = true; foreach (var kvp in resolvers) { @@ -241,7 +228,7 @@ void RunReleaseWithLinking (bool useMarshalMethods) JavaTypeAssemblies = assemblies.JavaTypeAssemblies[kvp.Key].Values, UserAssemblies = assemblies.UserAssemblies[kvp.Key].Values, GenerateRidAgnosticParts = first, - Resolver = kvp.Value, + Resolvers = resolvers, TargetArch = kvp.Key, }; @@ -253,6 +240,22 @@ void RunReleaseWithLinking (bool useMarshalMethods) } } + Dictionary MakeResolvers (bool includeRidAgnostic, bool useMarshalMethods) + { + var resolvers = new Dictionary (); + foreach (string abi in SupportedAbis) { + // Each ABI gets its own resolver in this mode... + XAAssemblyResolver resolver = MakeResolver (useMarshalMethods); + resolvers.Add (MonoAndroidHelper.AbiToTargetArch (abi), resolver); + } + + if (includeRidAgnostic) { + resolvers.Add (AndroidTargetArch.None, MakeResolver (useMarshalMethods)); + } + + return resolvers; + } + void RegisterApplicationConfigState (ApplicationConfigTaskState appConfState) { BuildEngine4.RegisterTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (ApplicationConfigTaskState.RegisterTaskObjectKey), appConfState, RegisteredTaskObjectLifetime.Build); @@ -263,7 +266,7 @@ void LogRunMode (string mode) Log.LogDebugMessage ($"GenerateJavaStubs mode: {mode}"); } - T CollectInterestingAssemblies (bool allowAbiSpecific, Func getResolver) where T: InputAssemblySet, new() + T CollectInterestingAssemblies (Dictionary resolvers) where T: InputAssemblySet, new() { var assemblies = new T (); AndroidTargetArch targetArch; @@ -292,8 +295,12 @@ void LogRunMode (string mode) assemblies.AddJavaTypeAssembly (assembly); } - targetArch = GetTargetArch (assembly); - getResolver (targetArch).Load (targetArch, assembly.ItemSpec); + targetArch = MonoAndroidHelper.GetTargetArch (assembly); + + // We don't check whether we have a resolver for `targetArch` on purpose, if it throws then it means we have a bug which + // should be fixed since there shouldn't be any assemblies passed to this task that belong in ABIs other than those + // specified in `SupportedAbis` (and, perhaps, a RID-agnostic one) + resolvers[targetArch].Load (targetArch, assembly.ItemSpec); } // However we only want to look for JLO types in user code for Java stub code generation @@ -303,26 +310,19 @@ void LogRunMode (string mode) continue; } - targetArch = GetTargetArch (assembly); - getResolver (targetArch).Load (targetArch, assembly.ItemSpec); + targetArch = MonoAndroidHelper.GetTargetArch (assembly); + resolvers[targetArch].Load (targetArch, assembly.ItemSpec); assemblies.AddJavaTypeAssembly (assembly); assemblies.AddUserAssembly (assembly); } return assemblies; - - AndroidTargetArch GetTargetArch (ITaskItem assembly) - { - AndroidTargetArch targetArch = MonoAndroidHelper.GetTargetArch (assembly); - if (!allowAbiSpecific && targetArch != AndroidTargetArch.None) { - throw new InvalidOperationException ($"Internal error: ABI-specific assemblies are not allowed in this build configuration"); - } - - return targetArch; - } } + // TODO: a different approach needed... In all the scenarios, there always are going to be RID-specific assemblies + // even if it's CoreLib right now. We MUST expect that even without linking, some assemblies can contain RID-specific + // code. void DoRun (RunState state, out ApplicationConfigTaskState? appConfState) { Log.LogDebugMessage ($"DoRun for arch {state.TargetArch}"); From 535ec49a0d21dffaa161c875776e8b54179cab69 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 20 Jul 2023 11:40:57 +0200 Subject: [PATCH 34/71] A slightly different approach --- .../Tasks/GenerateJavaStubs.cs | 107 ++++++++++-------- .../Utilities/IInputAssemblySet.cs | 3 + .../Utilities/RidAgnosticeInputAssemblySet.cs | 4 +- .../Utilities/RidAwareInputAssemblySet.cs | 34 ++++++ .../Utilities/RidSensitiveInputAssemblySet.cs | 62 ++++++++++ .../Utilities/RidSpecificInputAssemblySet.cs | 47 ++------ .../Utilities/TypeMapGenerator.cs | 19 +++- 7 files changed, 183 insertions(+), 93 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/RidAwareInputAssemblySet.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/RidSensitiveInputAssemblySet.cs diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index b0f1d32527a..029cba7d110 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -27,12 +27,12 @@ public class GenerateJavaStubs : AndroidTask { sealed class RunState { - public Dictionary Resolvers { get; set; } - public ICollection JavaTypeAssemblies { get; set; } - public ICollection UserAssemblies { get; set; } - public InputAssemblySet AssemblySet { get; set; } - public bool UseMarshalMethods { get; set; } - public AndroidTargetArch TargetArch { get; set; } = AndroidTargetArch.None; + public XAAssemblyResolver Resolver { get; set; } + public ICollection JavaTypeAssemblies { get; set; } + public ICollection UserAssemblies { get; set; } + public InputAssemblySet AssemblySet { get; set; } + public bool UseMarshalMethods { get; set; } + public AndroidTargetArch TargetArch { get; set; } = AndroidTargetArch.None; /// /// If `true`, generate code/data that doesn't depend on a specific RID (e.g. ACW maps or JCWs) @@ -169,56 +169,79 @@ void Run () } } - // We have one set of assemblies, no RID-specific ones. - // Typemaps don't use MVIDs or metadata tokens + // * We have one set of assemblies in general, some RID-specific ones (e.g. `System.Private.CoreLib`, potentially others). + // * Typemaps don't use MVIDs or metadata tokens, so we can process one each of the RID-specific ones together with the others + // * Marshal methods are never used void RunDebugNoLinking () { LogRunMode ("Debug, no linking"); - Dictionary resolvers = MakeResolvers (includeRidAgnostic: true, useMarshalMethods: false); - var assemblies = CollectInterestingAssemblies (resolvers); - throw new NotImplementedException (); + + XAAssemblyResolver resolver = MakeResolver (useMarshalMethods: false); + var assemblies = CollectInterestingAssemblies (new RidAgnosticInputAssemblySet (), (AndroidTargetArch arch) => resolver); + var state = new RunState { + UseMarshalMethods = false, + AssemblySet = assemblies, + JavaTypeAssemblies = assemblies.JavaTypeAssemblies, + UserAssemblies = assemblies.UserAssemblies, + GenerateRidAgnosticParts = true, + Resolver = resolver, + TargetArch = AndroidTargetArch.None, + }; + DoRun (state, out ApplicationConfigTaskState appConfState); + RegisterApplicationConfigState (appConfState); } - // We have as many sets of assemblies as there are RIDs, all assemblies are RID-specific - // Typemaps don't use MVIDs or metadata tokens + // * We have as many sets of assemblies as there are RIDs, all assemblies are RID-specific + // * Typemaps don't use MVIDs or metadata tokens, so we can process a single set of RID-specific assemblies + // * Marshal methods are never used void RunDebugWithLinking () { LogRunMode ("Debug, with linking"); - Dictionary resolvers = MakeResolvers (includeRidAgnostic: false, useMarshalMethods: false); - var assemblies = CollectInterestingAssemblies (resolvers); - throw new NotImplementedException (); - } - // We have one set of assemblies, no RID-specific ones. - // Typemaps use MVIDs and metadata tokens - void RunReleaseNoLinking (bool useMarshalMethods) - { - const AndroidTargetArch ArchHere = AndroidTargetArch.None; + XAAssemblyResolver resolver = MakeResolver (useMarshalMethods: false); + var assemblies = CollectInterestingAssemblies (new RidSpecificInputAssemblySet (), (AndroidTargetArch arch) => resolver); - LogRunMode ("Release, no linking"); - Dictionary resolvers = MakeResolvers (includeRidAgnostic: true, useMarshalMethods); - var assemblies = CollectInterestingAssemblies (resolvers); + AndroidTargetArch firstArch = assemblies.JavaTypeAssemblies.Keys.First (); var state = new RunState { - UseMarshalMethods = useMarshalMethods, + UseMarshalMethods = false, AssemblySet = assemblies, - JavaTypeAssemblies = assemblies.JavaTypeAssemblies, - UserAssemblies = assemblies.UserAssemblies, + JavaTypeAssemblies = assemblies.JavaTypeAssemblies[firstArch].Values, + UserAssemblies = assemblies.UserAssemblies[firstArch].Values, GenerateRidAgnosticParts = true, - Resolvers = resolvers, - TargetArch = ArchHere, + Resolver = resolver, + TargetArch = AndroidTargetArch.None, }; DoRun (state, out ApplicationConfigTaskState appConfState); RegisterApplicationConfigState (appConfState); } - // We have as many sets of assemblies as there are RIDs, all assemblies are RID-specific - // Typemaps use MVIDs and metadata tokens, need to process all per-RID assemblies separately + // * We have one set of assemblies in general, some RID-specific ones (e.g. `System.Private.CoreLib`, potentially others). + // * Typemaps use MVIDs and metadata tokens, so we need to process all assemblies as per-RID ones (different MVIDs in the + // actually RID-specific assemblies may affect sorting of the RID-agnostic ones) + // * Marshal methods may be used + void RunReleaseNoLinking (bool useMarshalMethods) + { + LogRunMode ("Release, no linking"); + + Dictionary resolvers = MakeResolvers (useMarshalMethods); + var assemblies = CollectInterestingAssemblies (new RidAwareInputAssemblySet (resolvers.Keys), (AndroidTargetArch arch) => resolvers[arch]); + RunReleaseCommon (useMarshalMethods, assemblies, resolvers); + } + + // * We have as many sets of assemblies as there are RIDs, all assemblies are RID-specific + // * Typemaps use MVIDs and metadata tokens, so we need per-RID set processing + // * Marshal methods may be used void RunReleaseWithLinking (bool useMarshalMethods) { LogRunMode ("Release, with linking"); - Dictionary resolvers = MakeResolvers (includeRidAgnostic: false, useMarshalMethods); - var assemblies = CollectInterestingAssemblies (resolvers); + Dictionary resolvers = MakeResolvers (useMarshalMethods); + var assemblies = CollectInterestingAssemblies (new RidSpecificInputAssemblySet (), (AndroidTargetArch arch) => resolvers[arch]); + RunReleaseCommon (useMarshalMethods, assemblies, resolvers); + } + + void RunReleaseCommon (bool useMarshalMethods, RidSensitiveInputAssemblySet assemblies, Dictionary resolvers) + { bool first = true; foreach (var kvp in resolvers) { @@ -228,7 +251,7 @@ void RunReleaseWithLinking (bool useMarshalMethods) JavaTypeAssemblies = assemblies.JavaTypeAssemblies[kvp.Key].Values, UserAssemblies = assemblies.UserAssemblies[kvp.Key].Values, GenerateRidAgnosticParts = first, - Resolvers = resolvers, + Resolver = kvp.Value, TargetArch = kvp.Key, }; @@ -240,7 +263,7 @@ void RunReleaseWithLinking (bool useMarshalMethods) } } - Dictionary MakeResolvers (bool includeRidAgnostic, bool useMarshalMethods) + Dictionary MakeResolvers (bool useMarshalMethods) { var resolvers = new Dictionary (); foreach (string abi in SupportedAbis) { @@ -249,10 +272,6 @@ Dictionary MakeResolvers (bool includeRid resolvers.Add (MonoAndroidHelper.AbiToTargetArch (abi), resolver); } - if (includeRidAgnostic) { - resolvers.Add (AndroidTargetArch.None, MakeResolver (useMarshalMethods)); - } - return resolvers; } @@ -266,9 +285,8 @@ void LogRunMode (string mode) Log.LogDebugMessage ($"GenerateJavaStubs mode: {mode}"); } - T CollectInterestingAssemblies (Dictionary resolvers) where T: InputAssemblySet, new() + T CollectInterestingAssemblies (T assemblies, Func getResolver) where T: InputAssemblySet { - var assemblies = new T (); AndroidTargetArch targetArch; foreach (ITaskItem assembly in ResolvedAssemblies) { bool value; @@ -300,7 +318,7 @@ void LogRunMode (string mode) // We don't check whether we have a resolver for `targetArch` on purpose, if it throws then it means we have a bug which // should be fixed since there shouldn't be any assemblies passed to this task that belong in ABIs other than those // specified in `SupportedAbis` (and, perhaps, a RID-agnostic one) - resolvers[targetArch].Load (targetArch, assembly.ItemSpec); + getResolver (targetArch).Load (targetArch, assembly.ItemSpec); } // However we only want to look for JLO types in user code for Java stub code generation @@ -311,7 +329,7 @@ void LogRunMode (string mode) } targetArch = MonoAndroidHelper.GetTargetArch (assembly); - resolvers[targetArch].Load (targetArch, assembly.ItemSpec); + getResolver (targetArch).Load (targetArch, assembly.ItemSpec); assemblies.AddJavaTypeAssembly (assembly); assemblies.AddUserAssembly (assembly); @@ -331,7 +349,6 @@ void DoRun (RunState state, out ApplicationConfigTaskState? appConfState) Log.LogDebugMessage ($" {assembly.ItemSpec}"); } - Log.LogDebugMessage ("User assemblies:"); foreach (ITaskItem assembly in state.UserAssemblies) { Log.LogDebugMessage ($" {assembly.ItemSpec}"); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/IInputAssemblySet.cs b/src/Xamarin.Android.Build.Tasks/Utilities/IInputAssemblySet.cs index 03c568806d3..9c4a7c16cab 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/IInputAssemblySet.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/IInputAssemblySet.cs @@ -1,3 +1,4 @@ +using System; using System.IO; using Microsoft.Build.Framework; @@ -10,5 +11,7 @@ abstract class InputAssemblySet public abstract void AddUserAssembly (ITaskItem assemblyItem); public abstract bool IsUserAssembly (string name); + protected static readonly StringComparer AssemblyNameStringComparer = StringComparer.OrdinalIgnoreCase; + protected string GetUserAssemblyKey (ITaskItem assemblyItem) => Path.GetFileNameWithoutExtension (assemblyItem.ItemSpec); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/RidAgnosticeInputAssemblySet.cs b/src/Xamarin.Android.Build.Tasks/Utilities/RidAgnosticeInputAssemblySet.cs index 065fd82b0c9..511fe24164e 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/RidAgnosticeInputAssemblySet.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/RidAgnosticeInputAssemblySet.cs @@ -7,8 +7,8 @@ namespace Xamarin.Android.Tasks; class RidAgnosticInputAssemblySet : InputAssemblySet { - Dictionary javaTypeAssemblies = new (StringComparer.OrdinalIgnoreCase); - Dictionary userAssemblies = new (StringComparer.OrdinalIgnoreCase); + Dictionary javaTypeAssemblies = new (AssemblyNameStringComparer); + Dictionary userAssemblies = new (AssemblyNameStringComparer); public ICollection JavaTypeAssemblies => javaTypeAssemblies.Values; public ICollection UserAssemblies => userAssemblies.Values; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/RidAwareInputAssemblySet.cs b/src/Xamarin.Android.Build.Tasks/Utilities/RidAwareInputAssemblySet.cs new file mode 100644 index 00000000000..f77cec38968 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/RidAwareInputAssemblySet.cs @@ -0,0 +1,34 @@ +using System.Collections.Generic; + +using Microsoft.Build.Framework; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +/// +/// Assembly input set where **some** assemblies are expected to be RID-specific but **all** of +/// them need to be placed in RID-specific sets. Any assembly that is not RID-specific will be +/// copied to all the RID-specific sets. This is to be used in `Release` builds when linking is +/// **disabled**, as these builds use assembly MVIDs and type+method metadata token IDs. +/// +class RidAwareInputAssemblySet : RidSensitiveInputAssemblySet +{ + List targetArches; + + public RidAwareInputAssemblySet (ICollection targetArches) + { + this.targetArches = new List (targetArches); + } + + protected override void Add (AndroidTargetArch arch, string key, ITaskItem assemblyItem, Dictionary> assemblies) + { + if (arch != AndroidTargetArch.None) { + base.Add (arch, key, assemblyItem, assemblies); + return; + } + + foreach (AndroidTargetArch targetArch in targetArches) { + base.Add (targetArch, key, assemblyItem, assemblies); + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/RidSensitiveInputAssemblySet.cs b/src/Xamarin.Android.Build.Tasks/Utilities/RidSensitiveInputAssemblySet.cs new file mode 100644 index 00000000000..6adce1df85e --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/RidSensitiveInputAssemblySet.cs @@ -0,0 +1,62 @@ +using System.Collections.Generic; + +using Microsoft.Build.Framework; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +abstract class RidSensitiveInputAssemblySet : InputAssemblySet +{ + // Both dictionaries must have the same keys present, even if the associated value is an empty dictionary + Dictionary> javaTypeAssemblies = new (); + Dictionary> userAssemblies = new (); + + public Dictionary> JavaTypeAssemblies => javaTypeAssemblies; + public Dictionary> UserAssemblies => userAssemblies; + + public override void AddJavaTypeAssembly (ITaskItem assemblyItem) + { + AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (assemblyItem); + Add (arch, assemblyItem.ItemSpec, assemblyItem, JavaTypeAssemblies); + EnsureArchKey (arch, userAssemblies); + } + + public override void AddUserAssembly (ITaskItem assemblyItem) + { + AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (assemblyItem); + Add (arch, GetUserAssemblyKey (assemblyItem), assemblyItem, UserAssemblies); + EnsureArchKey (arch, javaTypeAssemblies); + } + + public override bool IsUserAssembly (string name) + { + foreach (var kvp in userAssemblies) { + if (kvp.Value.ContainsKey (name)) { + return true; + } + } + + return false; + } + + protected virtual void Add (AndroidTargetArch targetArch, string key, ITaskItem assemblyItem, Dictionary> assemblies) + { + Dictionary dict = EnsureArchKey (targetArch, assemblies); + if (dict.ContainsKey (key)) { + return; + } + + dict.Add (key, assemblyItem); + } + + Dictionary EnsureArchKey (AndroidTargetArch targetArch, Dictionary> assemblies) + { + if (assemblies.TryGetValue (targetArch, out Dictionary dict)) { + return dict; + } + + dict = new (AssemblyNameStringComparer); + assemblies.Add (targetArch, dict); + return dict; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/RidSpecificInputAssemblySet.cs b/src/Xamarin.Android.Build.Tasks/Utilities/RidSpecificInputAssemblySet.cs index f5d4f726d6f..e5312fb2440 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/RidSpecificInputAssemblySet.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/RidSpecificInputAssemblySet.cs @@ -6,51 +6,18 @@ namespace Xamarin.Android.Tasks; -class RidSpecificInputAssemblySet : InputAssemblySet +/// +/// Assembly input set where **all** assemblies are expected to be RID-specific, any assembly which is +/// not will cause an exception. This is meant to be used whenever linking is enabled. +/// +class RidSpecificInputAssemblySet : RidSensitiveInputAssemblySet { - Dictionary> javaTypeAssemblies = new (); - Dictionary> userAssemblies = new (); - - public Dictionary> JavaTypeAssemblies => javaTypeAssemblies; - public Dictionary> UserAssemblies => userAssemblies; - - public override void AddJavaTypeAssembly (ITaskItem assemblyItem) - { - Add (assemblyItem.ItemSpec, assemblyItem, javaTypeAssemblies); - } - - public override void AddUserAssembly (ITaskItem assemblyItem) + protected override void Add (AndroidTargetArch arch, string key, ITaskItem assemblyItem, Dictionary> assemblies) { - Add (GetUserAssemblyKey (assemblyItem), assemblyItem, userAssemblies); - } - - public override bool IsUserAssembly (string name) - { - foreach (var kvp in userAssemblies) { - if (kvp.Value.ContainsKey (name)) { - return true; - } - } - - return false; - } - - void Add (string key, ITaskItem assemblyItem, Dictionary> assemblies) - { - AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (assemblyItem); if (arch == AndroidTargetArch.None) { throw new InvalidOperationException ($"`Abi` metadata item is required for assembly '{assemblyItem.ItemSpec}'"); } - if (!assemblies.TryGetValue (arch, out Dictionary? dict)) { - dict = new (StringComparer.OrdinalIgnoreCase); - assemblies.Add (arch, dict); - } - - if (dict.ContainsKey (key)) { - return; - } - - dict.Add (key, assemblyItem); + base.Add (arch, key, assemblyItem, assemblies); } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs index 8819832ac3a..8d2595b07d6 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs @@ -451,12 +451,19 @@ bool ShouldSkipInJavaToManaged (TypeDefinition td) void GenerateNativeAssembly (LLVMIR.LlvmIrComposer composer, LLVMIR.LlvmIrModule typeMapModule, string baseFileName) { - WriteNativeAssembly ( - targetArch, - composer, - typeMapModule, - GetOutputFilePath (baseFileName, ArchToAbi (targetArch)) - ); + if (targetArch != AndroidTargetArch.None) { + DoWriteNativeAssembly (targetArch, GetOutputFilePath (baseFileName, ArchToAbi (targetArch))); + return; + } + + foreach (string abi in supportedAbis) { + DoWriteNativeAssembly (MonoAndroidHelper.AbiToTargetArch (abi), GetOutputFilePath (baseFileName, abi)); + } + + void DoWriteNativeAssembly (AndroidTargetArch arch, string outputFile) + { + WriteNativeAssembly (arch, composer, typeMapModule, outputFile); + } } // void GenerateNativeAssembly (LLVMIR.LlvmIrComposer composer, LLVMIR.LlvmIrModule typeMapModule, string baseFileName) From 52d5e5de6d408ed5a8b8e544f3285d1713afab47 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 20 Jul 2023 16:58:04 +0200 Subject: [PATCH 35/71] A fixlet --- src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 029cba7d110..ec3eaafd967 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -224,6 +224,12 @@ void RunReleaseNoLinking (bool useMarshalMethods) LogRunMode ("Release, no linking"); Dictionary resolvers = MakeResolvers (useMarshalMethods); + + // All the RID-agnostic asseemblies will use resolvers of this architecture. This is because the RidAwareInputAssemblySet does not store + // such assemblies separately, but it copies them to **all** the target RIDs. This, in turn, is done because of typemaps and marshal methods + // which process data in a way that requires proper sorting of assemblies per MVID and it requires valid type and method token IDs. + AndroidTargetArch firstArch = resolvers.First ().Key; + resolvers.Add (AndroidTargetArch.None, resolvers[firstArch]); var assemblies = CollectInterestingAssemblies (new RidAwareInputAssemblySet (resolvers.Keys), (AndroidTargetArch arch) => resolvers[arch]); RunReleaseCommon (useMarshalMethods, assemblies, resolvers); } From ba00a5b581c56d374a7abbfe5b28279fa90279e3 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 20 Jul 2023 23:17:14 +0200 Subject: [PATCH 36/71] MSBuild work started --- .../Microsoft.Android.Sdk.After.targets | 1 + .../Microsoft.Android.Sdk.BuildOrder.targets | 30 +++ ...soft.Android.Sdk.NativeCompilation.targets | 228 ++++++++++++++++++ .../GenerateAssemblyDsoNativeSourceFiles.cs | 13 + .../Tasks/GenerateJavaStubs.cs | 3 - .../Tasks/PrepareAbiItems.cs | 3 + .../Utilities/CommonAssemblyStore.cs | 5 +- .../Xamarin.Android.Common.targets | 191 +-------------- 8 files changed, 277 insertions(+), 197 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets create mode 100644 src/Xamarin.Android.Build.Tasks/Tasks/GenerateAssemblyDsoNativeSourceFiles.cs diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.After.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.After.targets index 662aff0130b..aa9d99dd4c8 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.After.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.After.targets @@ -26,4 +26,5 @@ This file is imported *after* the Microsoft.NET.Sdk/Sdk.targets. + diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.BuildOrder.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.BuildOrder.targets index 4a4a3432258..f9f8f860db9 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.BuildOrder.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.BuildOrder.targets @@ -67,6 +67,7 @@ properties that determine build ordering. _GenerateEnvironmentFiles; _CompileJava; _CreateApplicationSharedLibraries; + _CreateAssemblySharedLibraries; _CompileDex; $(_AfterCompileDex); _CreateBaseApk; @@ -77,6 +78,35 @@ properties that determine build ordering. _IncludeNativeSystemLibraries; _CheckGoogleSdkRequirements; + + <_GenerateJavaStubsDependsOnTargets> + _SetLatestTargetFrameworkVersion; + _PrepareAssemblies; + _PrepareNativeAssemblySources; + $(_AfterPrepareAssemblies); + + + <_GeneratePackageManagerJavaDependsOn> + _GenerateJavaStubs; + _RunAotForAllRIDs; + _ManifestMerger; + _ConvertCustomView; + $(_AfterConvertCustomView); + _AddStaticResources; + $(_AfterAddStaticResources); + _PrepareAssemblies; + _PrepareEnvironmentAssemblySources; + _GenerateEnvironmentFiles; + _GenerateAndroidRemapNativeCode; + _GenerateEmptyAndroidRemapNativeCode; + _IncludeNativeSystemLibraries; + + + <_GenerateAndroidRemapNativeCodeDependsOn> + _ConvertAndroidMamMappingFileToXml; + _CollectAndroidRemapMembers; + _PrepareAndroidRemapNativeAssemblySources + diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets new file mode 100644 index 00000000000..0391c286777 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets @@ -0,0 +1,228 @@ + + + + + + + + + + + <_AndroidUseAssemblySharedLibraries Condition=" '$(EmbedAssembliesIntoApk)' != 'true' or '$(AndroidIncludeDebugSymbols)' == 'true' ">false + <_AndroidUseAssemblySharedLibraries Condition=" '$(_AndroidUseAssemblySharedLibraries)' == '' ">true + + <_AndroidApplicationSharedLibraryPath>$(IntermediateOutputPath)app_shared_libraries\ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <_NativeAssemblyTarget Include="@(_TypeMapAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> + %(_TypeMapAssemblySource.abi) + + + <_NativeAssemblyTarget Include="@(_EnvironmentAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> + %(_EnvironmentAssemblySource.abi) + + + <_NativeAssemblyTarget Include="@(_CompressedAssembliesAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> + %(_CompressedAssembliesAssemblySource.abi) + + + <_NativeAssemblyTarget Include="@(_MarshalMethodsAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> + %(_MarshalMethodsAssemblySource.abi) + + + <_NativeAssemblyTarget Include="@(_AndroidRemapAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> + %(_AndroidRemapAssemblySource.abi) + + + + + + + <_NativeAssemblySharedLibrariesTarget Include="@(_AssemblyDSOSource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> + %(_AssemblyDSOSource.abi) + + + + + + + <_ApplicationSharedLibrary Include="$(_AndroidApplicationSharedLibraryPath)%(_BuildTargetAbis.Identity)\libxamarin-app.so"> + %(_BuildTargetAbis.Identity) + + + + + + + <_AssemblySharedLibrary Include="$(_AndroidApplicationSharedLibraryPath)%(_BuildTargetAbis.Identity)\libxamarin-assemblies.so"> + %(_BuildTargetAbis.Identity) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAssemblyDsoNativeSourceFiles.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAssemblyDsoNativeSourceFiles.cs new file mode 100644 index 00000000000..670da74e7a5 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAssemblyDsoNativeSourceFiles.cs @@ -0,0 +1,13 @@ +using Microsoft.Android.Build.Tasks; + +namespace Xamarin.Android.Tasks; + +public class GenerateAssemblyDsoNativeSourceFiles : AndroidTask +{ + public override string TaskPrefix => "GADNSF"; + + public override bool RunTask () + { + throw new System.NotImplementedException(); + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index ec3eaafd967..0e4c5ace83b 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -344,9 +344,6 @@ T CollectInterestingAssemblies (T assemblies, Func "PAI"; @@ -56,6 +57,8 @@ public override bool RunTask () baseName = JniRemappingBase; } else if (String.Compare ("marshal_methods", Mode, StringComparison.OrdinalIgnoreCase) == 0) { baseName = MarshalMethodsBase; + } else if (String.Compare ("assembly_dsos", Mode, StringComparison.OrdinalIgnoreCase) == 0) { + baseName = AssemblyDSOBase; } else { Log.LogError ($"Unknown mode: {Mode}"); return false; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyStore.cs b/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyStore.cs index 9709a200e5a..5059ac60a67 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyStore.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyStore.cs @@ -29,10 +29,7 @@ public override void Add (AssemblyStoreAssemblyInfo blobAssembly) public override void Generate (string outputDirectory, List globalIndex, List blobPaths) { - if (assemblies.Count == 0) { - return; - } - + // Always generate this blob, even if there are no assembly entries as this blob contains the global index Generate (Path.Combine (outputDirectory, $"{ApkName}_{BlobPrefix}{BlobExtension}"), assemblies, globalIndex, blobPaths); } } diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 3bf64758b7c..37f2ca625b4 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -87,13 +87,9 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. - - - - - + - - - - - - - - - <_GenerateJavaStubsDependsOnTargets> - _SetLatestTargetFrameworkVersion; - _PrepareAssemblies; - _PrepareNativeAssemblySources; - $(_AfterPrepareAssemblies); - - - - - - - - - - - - - - - <_GeneratedAndroidEnvironment Include="__XA_PACKAGE_NAMING_POLICY__=$(AndroidPackageNamingPolicy)" /> @@ -1664,49 +1611,6 @@ because xbuild doesn't support framework reference assemblies. - - - - - - - - - - - - <_GenerateAndroidRemapNativeCodeDependsOn> - _ConvertAndroidMamMappingFileToXml; - _CollectAndroidRemapMembers; - _PrepareAndroidRemapNativeAssemblySources - - - - - - - @@ -1718,24 +1622,6 @@ because xbuild doesn't support framework reference assemblies. - - <_GeneratePackageManagerJavaDependsOn> - _GenerateJavaStubs; - _RunAotForAllRIDs; - _ManifestMerger; - _ConvertCustomView; - $(_AfterConvertCustomView); - _AddStaticResources; - $(_AfterAddStaticResources); - _PrepareAssemblies; - _PrepareEnvironmentAssemblySources; - _GenerateEnvironmentFiles; - _GenerateAndroidRemapNativeCode; - _GenerateEmptyAndroidRemapNativeCode; - _IncludeNativeSystemLibraries; - - - - - - <_NativeAssemblyTarget Include="@(_TypeMapAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> - %(_TypeMapAssemblySource.abi) - - <_NativeAssemblyTarget Include="@(_EnvironmentAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> - %(_EnvironmentAssemblySource.abi) - - <_NativeAssemblyTarget Include="@(_CompressedAssembliesAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> - %(_CompressedAssembliesAssemblySource.abi) - - <_CompressedNativeAssemblyTarget Include="@(_CompressedAssembliesAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> - %(_CompressedAssembliesAssemblySource.abi) - - <_NativeAssemblyTarget Include="@(_MarshalMethodsAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> - %(_MarshalMethodsAssemblySource.abi) - - <_NativeAssemblyTarget Include="@(_AndroidRemapAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> - %(_AndroidRemapAssemblySource.abi) - - - - - - - - - - - - - - - - - - - - - <_ApplicationSharedLibrary Include="$(_AndroidApplicationSharedLibraryPath)%(_BuildTargetAbis.Identity)\libxamarin-app.so"> - %(_BuildTargetAbis.Identity) - - - - - - - - - - - From fc7e28881f74630f09609bfc6c8f063304e9c008 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 21 Jul 2023 23:36:49 +0200 Subject: [PATCH 37/71] Compression for DSOs works --- ...soft.Android.Sdk.NativeCompilation.targets | 13 +- .../Tasks/BuildApk.cs | 56 ++------ .../GenerateAssemblyDsoNativeSourceFiles.cs | 78 ++++++++++- .../Tasks/PrepareAbiItems.cs | 4 +- .../Utilities/AssemblyCompression.cs | 132 ++++++++++++++++-- .../Utilities/CompressedAssemblyInfo.cs | 35 +++++ .../Utilities/MonoAndroidHelper.cs | 2 + .../Utilities/TypeMapGenerator.cs | 6 +- 8 files changed, 260 insertions(+), 66 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets index 0391c286777..dd5a7879a10 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets @@ -12,6 +12,7 @@ <_AndroidUseAssemblySharedLibraries Condition=" '$(_AndroidUseAssemblySharedLibraries)' == '' ">true <_AndroidApplicationSharedLibraryPath>$(IntermediateOutputPath)app_shared_libraries\ + <_CompressedAssembliesOutputDir>$(IntermediateOutputPath)android\lz4-2 @@ -129,8 +130,16 @@ - + Condition=" '$(_AndroidUseAssemblySharedLibraries)' == 'true' " + Inputs="@(_ShrunkUserAssemblies);@(_ShrunkFrameworkAssemblies);@(_AndroidResolvedSatellitePaths)" + Outputs="@(_AssemblyDSOSource)"> + compressedAssembliesInfo, string assemblyStoreApkName) { string sourcePath; - AssemblyCompression.AssemblyData compressedAssembly = null; - string compressedOutputDir = Path.GetFullPath (Path.Combine (Path.GetDirectoryName (ApkOutputPath), "..", "lz4")); AssemblyStoreGenerator storeGenerator; if (UseAssemblyStore) { @@ -391,6 +389,12 @@ void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary "GADNSF"; + [Required] + public string SourcesOutputDirectory { get; set; } + + [Required] + public string CompressedAssembliesOutputDirectory { get; set; } + + [Required] + public string[] SupportedAbis { get; set; } + + [Required] + public ITaskItem[] Assemblies { get; set; } + + [Required] + public bool EnableCompression { get; set; } + public override bool RunTask () { - throw new System.NotImplementedException(); + Dictionary> dsoAssembliesInfo = new (); + AssemblyCompression? assemblyCompressor = null; + if (EnableCompression) { + assemblyCompressor = new AssemblyCompression (Log, CompressedAssembliesOutputDirectory); + } + + foreach (ITaskItem assembly in Assemblies) { + FileInfo fi = new (assembly.ItemSpec); + string inputFile; + bool compressed; + ulong compressedSize; + + if (assemblyCompressor != null) { + (inputFile, compressed) = assemblyCompressor.CompressAssembly (assembly, fi); + + if (!compressed) { + compressedSize = 0; + } else { + var cfi = new FileInfo (inputFile); + compressedSize = (ulong)cfi.Length; + } + } else { + inputFile = assembly.ItemSpec; + compressed = false; + compressedSize = 0; + } + + AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (assembly); + if (!dsoAssembliesInfo.TryGetValue (arch, out List? assemblyList)) { + assemblyList = new List (); + dsoAssembliesInfo.Add (arch, assemblyList); + } + assemblyList.Add (new DSOAssemblyInfo (GetAssemblyName (assembly), inputFile, (ulong)fi.Length, compressedSize)); + } + + return true; + } + + string GetAssemblyName (ITaskItem assembly) + { + if (!assembly.ItemSpec.EndsWith (".resources.dll", StringComparison.OrdinalIgnoreCase)) { + return Path.GetFileName (assembly.ItemSpec); + } + + // It's a satellite assembly, %(DestinationSubDirectory) is the culture prefix + string? destinationSubDir = assembly.GetMetadata ("DestinationSubDirectory"); + if (String.IsNullOrEmpty (destinationSubDir)) { + throw new InvalidOperationException ($"Satellite assembly '{assembly.ItemSpec}' has no culture metadata item"); + } + + string ret = $"{destinationSubDir}{Path.GetFileName (assembly.ItemSpec)}"; + if (!assembly.ItemSpec.EndsWith (ret, StringComparison.OrdinalIgnoreCase)) { + throw new InvalidOperationException ($"Invalid metadata in satellite assembly '{assembly.ItemSpec}', culture metadata ('{destinationSubDir}') doesn't match file path"); + } + + return ret; } } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAbiItems.cs b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAbiItems.cs index 628526fa090..c683c98c6b7 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAbiItems.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAbiItems.cs @@ -16,7 +16,7 @@ public class PrepareAbiItems : AndroidTask const string CompressedAssembliesBase = "compressed_assemblies"; const string JniRemappingBase = "jni_remap"; const string MarshalMethodsBase = "marshal_methods"; - const string AssemblyDSOBase = "assembly_dso"; + public const string AssemblyDSOBase = "assembly_dso"; public override string TaskPrefix => "PAI"; @@ -66,7 +66,7 @@ public override bool RunTask () TaskItem item; foreach (string abi in BuildTargetAbis) { - item = new TaskItem (Path.Combine (NativeSourcesDir, $"{baseName}.{abi}.ll")); + item = new TaskItem (Path.Combine (NativeSourcesDir, MonoAndroidHelper.MakeNativeAssemblyFileName (baseName, abi))); item.SetMetadata ("abi", abi); sources.Add (item); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs index 1972dba0d85..000de2863b9 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs @@ -1,21 +1,25 @@ using System; using System.Buffers; +using System.Collections.Generic; using System.IO; using K4os.Compression.LZ4; +using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; namespace Xamarin.Android.Tasks { class AssemblyCompression { - public enum CompressionResult + enum CompressionResult { Success, InputTooBig, EncodingFailed, } - public sealed class AssemblyData + sealed class AssemblyData { public string SourcePath { get; internal set; } public uint DescriptorIndex { get; internal set; } @@ -47,15 +51,25 @@ public void SetData (string sourcePath, uint descriptorIndex) // two - it should be more than enough for most needs. //public const ulong InputAssemblySizeLimit = 60 * 1024 * 1024; - static readonly ArrayPool bytePool = ArrayPool.Shared; + readonly ArrayPool bytePool = ArrayPool.Shared; + readonly TaskLoggingHelper log; + readonly string compressedOutputDir; - public static CompressionResult Compress (AssemblyData data, string outputDirectory) + public AssemblyCompression (TaskLoggingHelper log, string compressedOutputDir) { - if (data == null) + this.log = log; + this.compressedOutputDir = compressedOutputDir; + } + + CompressionResult Compress (AssemblyData data, string outputDirectory) + { + if (data == null) { throw new ArgumentNullException (nameof (data)); + } - if (String.IsNullOrEmpty (outputDirectory)) + if (String.IsNullOrEmpty (outputDirectory)) { throw new ArgumentException ("must not be null or empty", nameof (outputDirectory)); + } Directory.CreateDirectory (outputDirectory); @@ -79,8 +93,9 @@ public static CompressionResult Compress (AssemblyData data, string outputDirect destBytes = bytePool.Rent (LZ4Codec.MaximumOutputSize (sourceBytes.Length)); int encodedLength = LZ4Codec.Encode (sourceBytes, 0, checked((int)fi.Length), destBytes, 0, destBytes.Length, LZ4Level.L09_HC); - if (encodedLength < 0) + if (encodedLength < 0) { return CompressionResult.EncodingFailed; + } data.DestinationSize = (uint)encodedLength; using (var fs = File.Open (data.DestinationPath, FileMode.Create, FileAccess.Write, FileShare.Read)) { @@ -94,13 +109,112 @@ public static CompressionResult Compress (AssemblyData data, string outputDirect } } } finally { - if (sourceBytes != null) + if (sourceBytes != null) { bytePool.Return (sourceBytes); - if (destBytes != null) + } + if (destBytes != null) { bytePool.Return (destBytes); + } } return CompressionResult.Success; } + + public (string outputPath, bool compressed) CompressAssembly (ITaskItem assembly, FileInfo inputInfo) + { + if (Boolean.TryParse (assembly.GetMetadata ("AndroidSkipCompression"), out bool value) && value) { + log.LogDebugMessage ($"Skipping compression of {assembly.ItemSpec} due to 'AndroidSkipCompression' == 'true' "); + return (assembly.ItemSpec, false); + } + + if (!inputInfo.Exists) { + throw new InvalidOperationException ($"File '{assembly.ItemSpec}' does not exist"); + } + + string assemblyOutputDir; + string? subDirectory = assembly.GetMetadata ("DestinationSubDirectory"); + if (!String.IsNullOrEmpty (subDirectory)) { + assemblyOutputDir = Path.Combine (compressedOutputDir, subDirectory); + } else { + assemblyOutputDir = compressedOutputDir; + } + string outputPath = Path.Combine (assemblyOutputDir, $"{Path.GetFileName (assembly.ItemSpec)}.lz4"); + Directory.CreateDirectory (assemblyOutputDir); + + byte[]? sourceBytes = null; + byte[]? destBytes = null; + try { + int inputLength = checked((int)inputInfo.Length); + sourceBytes = bytePool.Rent (inputLength); + using (var fs = File.Open (assembly.ItemSpec, FileMode.Open, FileAccess.Read, FileShare.Read)) { + fs.Read (sourceBytes, 0, inputLength); + } + + destBytes = bytePool.Rent (LZ4Codec.MaximumOutputSize (sourceBytes.Length)); + int encodedLength = LZ4Codec.Encode (sourceBytes, 0, inputLength, destBytes, 0, destBytes.Length, LZ4Level.L09_HC); + if (encodedLength < 0) { + log.LogMessage ($"Failed to compress {assembly.ItemSpec}"); + return (assembly.ItemSpec, false); + } + + using (var fs = File.Open (outputPath, FileMode.Create, FileAccess.Write, FileShare.Read)) { + using (var bw = new BinaryWriter (fs)) { + bw.Write (destBytes, 0, encodedLength); + bw.Flush (); + } + } + } finally { + if (sourceBytes != null) { + bytePool.Return (sourceBytes); + } + if (destBytes != null) { + bytePool.Return (destBytes); + } + } + + return (outputPath, true); + } + + public string CompressAssembly (ITaskItem assembly, IDictionary compressedAssembliesInfo) + { + if (bool.TryParse (assembly.GetMetadata ("AndroidSkipCompression"), out bool value) && value) { + log.LogDebugMessage ($"Skipping compression of {assembly.ItemSpec} due to 'AndroidSkipCompression' == 'true' "); + return assembly.ItemSpec; + } + + var key = CompressedAssemblyInfo.GetDictionaryKey (assembly); + if (compressedAssembliesInfo.TryGetValue (key, out CompressedAssemblyInfo info) && info != null) { + AssemblyData compressedAssembly = new AssemblyData (assembly.ItemSpec, info.DescriptorIndex); + + string assemblyOutputDir; + string subDirectory = assembly.GetMetadata ("DestinationSubDirectory"); + if (!String.IsNullOrEmpty (subDirectory)) + assemblyOutputDir = Path.Combine (compressedOutputDir, subDirectory); + else + assemblyOutputDir = compressedOutputDir; + CompressionResult result = Compress (compressedAssembly, assemblyOutputDir); + if (result != CompressionResult.Success) { + switch (result) { + case CompressionResult.EncodingFailed: + log.LogMessage ($"Failed to compress {assembly.ItemSpec}"); + break; + + case CompressionResult.InputTooBig: + log.LogMessage ($"Input assembly {assembly.ItemSpec} exceeds maximum input size"); + break; + + default: + log.LogMessage ($"Unknown error compressing {assembly.ItemSpec}"); + break; + } + return assembly.ItemSpec; + } + return compressedAssembly.DestinationPath; + } else { + log.LogDebugMessage ($"Assembly missing from {nameof (CompressedAssemblyInfo)}: {key}"); + } + + return assembly.ItemSpec; + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssemblyInfo.cs b/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssemblyInfo.cs index b3f775a96b7..0923982d025 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssemblyInfo.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssemblyInfo.cs @@ -37,4 +37,39 @@ public static string GetDictionaryKey (ITaskItem assembly) return Path.Combine (subDirectory, Path.GetFileName (assembly.ItemSpec)); } } + + class DSOAssemblyInfo + { + /// + /// Size of the loadable assembly data (after decompression, if compression is enabled). + /// + public ulong DataSize { get; } + + /// + /// Size of the compressed assembly data or `0` if assembly is uncompressed. + /// + public ulong CompressedDataSize { get; } + + public string InputFile { get; } + + /// + /// Name of the assembly, including culture prefix if it's a satellite assembly. Must include the + /// extension. + /// + public string Name { get; } + + /// + /// is the original assembly name, including culture prefix (e.g. `en_US/`) if it is a + /// satellite assembly. should be the full path to the input file. + /// gives the original file size, while specifies + /// data size after compression, or `0` if file isn't compressed. + /// + public DSOAssemblyInfo (string name, string inputFile, ulong dataSize, ulong compressedDataSize) + { + Name = name; + InputFile = inputFile; + DataSize = dataSize; + CompressedDataSize = compressedDataSize; + } + } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs index d72b347d5e9..2bb74996a6e 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs @@ -643,5 +643,7 @@ public static string GetNativeLibsRootDirectoryPath (string androidBinUtilsDirec string relPath = GetToolsRootDirectoryRelativePath (androidBinUtilsDirectory); return Path.GetFullPath (Path.Combine (androidBinUtilsDirectory, relPath, "lib")); } + + public static string MakeNativeAssemblyFileName (string baseName, string abi) => $"{baseName}.{abi}.ll"; } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs index 8d2595b07d6..a1081b3f35e 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs @@ -447,17 +447,15 @@ bool ShouldSkipInJavaToManaged (TypeDefinition td) return td.IsInterface || td.HasGenericParameters; } - string GetOutputFilePath (string baseFileName, string abi) => $"{baseFileName}.{abi}.ll"; - void GenerateNativeAssembly (LLVMIR.LlvmIrComposer composer, LLVMIR.LlvmIrModule typeMapModule, string baseFileName) { if (targetArch != AndroidTargetArch.None) { - DoWriteNativeAssembly (targetArch, GetOutputFilePath (baseFileName, ArchToAbi (targetArch))); + DoWriteNativeAssembly (targetArch, MonoAndroidHelper.MakeNativeAssemblyFileName (baseFileName, ArchToAbi (targetArch))); return; } foreach (string abi in supportedAbis) { - DoWriteNativeAssembly (MonoAndroidHelper.AbiToTargetArch (abi), GetOutputFilePath (baseFileName, abi)); + DoWriteNativeAssembly (MonoAndroidHelper.AbiToTargetArch (abi), MonoAndroidHelper.MakeNativeAssemblyFileName (baseFileName, abi)); } void DoWriteNativeAssembly (AndroidTargetArch arch, string outputFile) From cbda14eba1afb7367c10223cbf1e564ba3039e9b Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 4 Sep 2023 14:05:26 +0200 Subject: [PATCH 38/71] Stub DSO generator task --- .../Utilities/AssemblyDSOGenerator.cs | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs new file mode 100644 index 00000000000..299e5721517 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs @@ -0,0 +1,21 @@ +using System.Collections.Generic; + +using Xamarin.Android.Tasks.LLVMIR; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +class AssemblyDSOGenerator : LlvmIrComposer +{ + Dictionary> assemblies; + + public AssemblyDSOGenerator (Dictionary> dsoAssemblies) + { + assemblies = dsoAssemblies; + } + + protected override void Construct (LlvmIrModule module) + { + throw new System.NotImplementedException(); + } +} From caab88143b9fb26bfbe287ac0e330908f3f5dfbf Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 8 Sep 2023 21:04:01 +0200 Subject: [PATCH 39/71] Finally settled on the assembly DSO structures --- .../GenerateAssemblyDsoNativeSourceFiles.cs | 21 +++++- .../Utilities/AssemblyDSOGenerator.cs | 71 ++++++++++++++++++- ...ySet.cs => RidAgnosticInputAssemblySet.cs} | 0 src/monodroid/jni/application_dso_stub.cc | 41 +++++++++++ src/monodroid/jni/xamarin-app.hh | 39 ++++++++++ 5 files changed, 169 insertions(+), 3 deletions(-) rename src/Xamarin.Android.Build.Tasks/Utilities/{RidAgnosticeInputAssemblySet.cs => RidAgnosticInputAssemblySet.cs} (100%) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAssemblyDsoNativeSourceFiles.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAssemblyDsoNativeSourceFiles.cs index 5a1f8ea9ddf..9dc02e48368 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAssemblyDsoNativeSourceFiles.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAssemblyDsoNativeSourceFiles.cs @@ -33,14 +33,20 @@ public override bool RunTask () AssemblyCompression? assemblyCompressor = null; if (EnableCompression) { assemblyCompressor = new AssemblyCompression (Log, CompressedAssembliesOutputDirectory); + Log.LogDebugMessage ("Assembly compression ENABLED"); + } else { + Log.LogDebugMessage ("Assembly compression DISABLED"); } + ulong assemblyDataSize = 0; + Log.LogDebugMessage ("Processing the input assemblies"); foreach (ITaskItem assembly in Assemblies) { FileInfo fi = new (assembly.ItemSpec); string inputFile; bool compressed; ulong compressedSize; + Log.LogDebugMessage ($" Input: {assembly.ItemSpec}"); if (assemblyCompressor != null) { (inputFile, compressed) = assemblyCompressor.CompressAssembly (assembly, fi); @@ -55,16 +61,27 @@ public override bool RunTask () compressed = false; compressedSize = 0; } - + assemblyDataSize += compressedSize == 0 ? (ulong)fi.Length : compressedSize; + Log.LogDebugMessage ($" will include from: {inputFile} (compressed? {compressed}; compressedSize == {compressedSize}"); AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (assembly); if (!dsoAssembliesInfo.TryGetValue (arch, out List? assemblyList)) { assemblyList = new List (); dsoAssembliesInfo.Add (arch, assemblyList); } assemblyList.Add (new DSOAssemblyInfo (GetAssemblyName (assembly), inputFile, (ulong)fi.Length, compressedSize)); + Log.LogDebugMessage ($" added to list with name: {assemblyList[assemblyList.Count - 1].Name}"); + } + + Log.LogDebugMessage ($"Size of assembly data to stash: {assemblyDataSize}"); + Log.LogDebugMessage ($"Number of architectures to stash into DSOs: {dsoAssembliesInfo.Count}"); + foreach (var kvp in dsoAssembliesInfo) { + Log.LogDebugMessage ($" {kvp.Key}: {kvp.Value.Count} assemblies"); } - return true; + var generator = new AssemblyDSOGenerator (dsoAssembliesInfo); + LLVMIR.LlvmIrModule module = generator.Construct (); + + return !Log.HasLoggedErrors; } string GetAssemblyName (ITaskItem assembly) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs index 299e5721517..512f14b5c46 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using Xamarin.Android.Tasks.LLVMIR; @@ -7,7 +8,35 @@ namespace Xamarin.Android.Tasks; class AssemblyDSOGenerator : LlvmIrComposer { + // Must be identical to AssemblyEntry in src/monodroid/jni/xamarin-app.hh + sealed class AssemblyEntry + { + // offset into the `xa_input_assembly_data` array + public uint input_data_offset; + + // number of bytes data of this assembly occupies + public uint input_data_size; + + // offset into the `xa_uncompressed_assembly_data` array where the uncompressed + // assembly data (if any) lives. + public uint uncompressed_data_offset; + + // Size of the uncompressed data. 0 if assembly wasn't compressed. + public uint uncompressed_data_size; + } + + // Must be identical to AssemblyIndexEntry in src/monodroid/jni/xamarin-app.hh + sealed class AssemblyIndexEntry + { + ulong name_hash; + + // Index into the `xa_assemblies` descriptor array + uint index; + } + Dictionary> assemblies; + StructureInfo? assemblyEntryStructureInfo; + StructureInfo? assemblyIndexEntryStructureInfo; public AssemblyDSOGenerator (Dictionary> dsoAssemblies) { @@ -16,6 +45,46 @@ public AssemblyDSOGenerator (Dictionary protected override void Construct (LlvmIrModule module) { - throw new System.NotImplementedException(); + MapStructures (module); + + if (assemblies.Count == 0) { + ConstructEmptyModule (); + return; + } + + int expectedCount = -1; + foreach (var kvp in assemblies) { + AndroidTargetArch arch = kvp.Key; + List infos = kvp.Value; + + if (expectedCount < 0) { + expectedCount = infos.Count; + } + + if (infos.Count != expectedCount) { + throw new InvalidOperationException ($"Collection of assemblies for architecture {arch} has a different number of entries ({infos.Count}) than expected ({expectedCount})"); + } + + if (infos.Count == 0) { + continue; + } + } + + if (expectedCount <= 0) { + ConstructEmptyModule (); + } + + throw new NotImplementedException (); + } + + void ConstructEmptyModule () + { + throw new NotImplementedException (); + } + + void MapStructures (LlvmIrModule module) + { + assemblyEntryStructureInfo = module.MapStructure (); + assemblyIndexEntryStructureInfo = module.MapStructure (); } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/RidAgnosticeInputAssemblySet.cs b/src/Xamarin.Android.Build.Tasks/Utilities/RidAgnosticInputAssemblySet.cs similarity index 100% rename from src/Xamarin.Android.Build.Tasks/Utilities/RidAgnosticeInputAssemblySet.cs rename to src/Xamarin.Android.Build.Tasks/Utilities/RidAgnosticInputAssemblySet.cs diff --git a/src/monodroid/jni/application_dso_stub.cc b/src/monodroid/jni/application_dso_stub.cc index 97f554e4c25..a8d4c7a67a7 100644 --- a/src/monodroid/jni/application_dso_stub.cc +++ b/src/monodroid/jni/application_dso_stub.cc @@ -202,6 +202,47 @@ const MarshalMethodName mm_method_names[] = { }, }; +alignas(4096) const uint8_t xa_input_assembly_data[InputAssemblyDataSize] = { + 0x04, 0x00, 0x30, 0x26, 0xfe, 0xfb, 0x37, 0xf4, 0xb7, 0x19, 0x0f, 0xdc, 0xad, 0xb5, 0x3c, 0x82, + 0xf4, 0xd9, 0x64, 0xe3, 0x56, 0x95, 0x7a, 0xef, 0x0b, 0x79, 0xbe, 0x28, 0x2b, 0x2a, 0x31, 0x54, + 0xf1, 0x2a, 0x76, 0xf9, 0x84, 0x5a, 0x5e, 0x0c, 0x11, 0x30, 0xaf, 0x5d, 0xb1, 0xff, 0x0f, 0x48, +}; + +alignas(4096) uint8_t xa_uncompressed_assembly_data[UncompressedAssemblyDataSize] = { }; + +const AssemblyEntry xa_assemblies[AssemblyCount] = { + { + .input_data_offset = 0, + .input_data_size = 256, + .uncompressed_data_offset = 0, + .uncompressed_data_size = 0, + }, + + { + .input_data_offset = 256, + .input_data_size = 768, + .uncompressed_data_offset = 0, + .uncompressed_data_size = 2048, + }, +}; + +const AssemblyIndexEntry xa_assembly_index[AssemblyCount] = { + { + .name_hash = 11111u, + .index = 0, + }, + + { + .name_hash = 22222u, + .index = 1, + }, +}; + +const char xa_assembly_names[AssemblyCount][AssemblyNameLength] = { + "Assembly1.dll", + "AnotherAssembly2.dll", +}; + void xamarin_app_init ([[maybe_unused]] JNIEnv *env, [[maybe_unused]] get_function_pointer_fn fn) noexcept { // Dummy diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index d50fc39a683..ca081218805 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -111,6 +111,45 @@ struct XamarinAndroidBundledAssembly final char *name; }; +struct AssemblyEntry +{ + // offset into the `xa_input_assembly_data` array + uint32_t input_data_offset; + + // number of bytes data of this assembly occupies + uint32_t input_data_size; + + // offset into the `xa_uncompressed_assembly_data` array where the uncompressed + // assembly data (if any) lives. + uint32_t uncompressed_data_offset; + + // Size of the uncompressed data. 0 if assembly wasn't compressed. + uint32_t uncompressed_data_size; +}; + +struct AssemblyIndexEntry +{ + xamarin::android::hash_t name_hash; + + // Index into the `xa_assemblies` descriptor array + uint32_t index; +}; + +constexpr size_t InputAssemblyDataSize = 1024; +constexpr size_t UncompressedAssemblyDataSize = 2048; +constexpr size_t AssemblyCount = 2; +constexpr size_t AssemblyNameLength = 26; // including the terminating NUL + +MONO_API MONO_API_EXPORT const uint8_t xa_input_assembly_data[InputAssemblyDataSize]; + +// All the compressed assemblies are uncompressed into this array, with offsets in `xa_assemblies` +// pointing to the place where they start +MONO_API MONO_API_EXPORT uint8_t xa_uncompressed_assembly_data[UncompressedAssemblyDataSize]; + +MONO_API MONO_API_EXPORT const AssemblyEntry xa_assemblies[AssemblyCount]; +MONO_API MONO_API_EXPORT const AssemblyIndexEntry xa_assembly_index[AssemblyCount]; +MONO_API MONO_API_EXPORT const char xa_assembly_names[AssemblyCount][AssemblyNameLength]; + // // Assembly store format // From f793a0c8348df2e113ef88babf0a3c404e40b49f Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 11 Sep 2023 23:10:34 +0200 Subject: [PATCH 40/71] A couple of steps forward --- .../GenerateAssemblyDsoNativeSourceFiles.cs | 6 +- .../Utilities/AssemblyDSOGenerator.cs | 58 +++++++++++++++++-- .../Utilities/CompressedAssemblyInfo.cs | 6 +- 3 files changed, 60 insertions(+), 10 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAssemblyDsoNativeSourceFiles.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAssemblyDsoNativeSourceFiles.cs index 9dc02e48368..1dd85f4b7b7 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAssemblyDsoNativeSourceFiles.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAssemblyDsoNativeSourceFiles.cs @@ -44,7 +44,7 @@ public override bool RunTask () FileInfo fi = new (assembly.ItemSpec); string inputFile; bool compressed; - ulong compressedSize; + uint compressedSize; Log.LogDebugMessage ($" Input: {assembly.ItemSpec}"); if (assemblyCompressor != null) { @@ -54,7 +54,7 @@ public override bool RunTask () compressedSize = 0; } else { var cfi = new FileInfo (inputFile); - compressedSize = (ulong)cfi.Length; + compressedSize = (uint)cfi.Length; } } else { inputFile = assembly.ItemSpec; @@ -68,7 +68,7 @@ public override bool RunTask () assemblyList = new List (); dsoAssembliesInfo.Add (arch, assemblyList); } - assemblyList.Add (new DSOAssemblyInfo (GetAssemblyName (assembly), inputFile, (ulong)fi.Length, compressedSize)); + assemblyList.Add (new DSOAssemblyInfo (GetAssemblyName (assembly), inputFile, (uint)fi.Length, compressedSize)); Log.LogDebugMessage ($" added to list with name: {assemblyList[assemblyList.Count - 1].Name}"); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs index 512f14b5c46..2adf587876e 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs @@ -34,13 +34,36 @@ sealed class AssemblyIndexEntry uint index; } - Dictionary> assemblies; + // Members with underscores correspond to the native fields we output. + sealed class ArchState + { + public readonly List xa_input_assembly_data = new List (); + public ulong xa_uncompressed_assembly_data_length = 0; + public readonly List xa_assemblies; + public readonly List xa_assembly_index; + public ulong assemblyNameLength = 0; + public readonly List xa_assembly_names = new List (); + + public ArchState (int assemblyCount) + { + if (assemblyCount < 0) { + throw new ArgumentException ("must not be a negative number", nameof (assemblyCount)); + } + + xa_assemblies = new List (assemblyCount); + xa_assembly_index = new List (assemblyCount); + } + } + + readonly Dictionary> assemblies; + readonly Dictionary assemblyArchStates; StructureInfo? assemblyEntryStructureInfo; StructureInfo? assemblyIndexEntryStructureInfo; public AssemblyDSOGenerator (Dictionary> dsoAssemblies) { assemblies = dsoAssemblies; + assemblyArchStates = new Dictionary (); } protected override void Construct (LlvmIrModule module) @@ -65,9 +88,7 @@ protected override void Construct (LlvmIrModule module) throw new InvalidOperationException ($"Collection of assemblies for architecture {arch} has a different number of entries ({infos.Count}) than expected ({expectedCount})"); } - if (infos.Count == 0) { - continue; - } + AddAssemblyData (arch, infos); } if (expectedCount <= 0) { @@ -77,6 +98,35 @@ protected override void Construct (LlvmIrModule module) throw new NotImplementedException (); } + void AddAssemblyData (AndroidTargetArch arch, List infos) + { + if (infos.Count == 0) { + return; + } + + if (!assemblyArchStates.TryGetValue (arch, out ArchState archState)) { + archState = new ArchState (infos.Count); + assemblyArchStates.Add (arch, archState); + } + + ulong uncompressed_offset = 0; + foreach (DSOAssemblyInfo info in infos) { + var entry = new AssemblyEntry { + input_data_offset = (uint)archState.xa_input_assembly_data.Count, + input_data_size = info.CompressedDataSize == 0 ? (uint)info.DataSize : (uint)info.CompressedDataSize, + uncompressed_data_size = info.CompressedDataSize == 0 ? 0 : (uint)info.DataSize, + uncompressed_data_offset = (uint)uncompressed_offset, + }; + + uncompressed_offset += entry.uncompressed_data_offset; + + // This is way, way more than Google Play Store supports now, but we won't limit ourselves more than we have to + if (uncompressed_offset > UInt32.MaxValue) { + throw new InvalidOperationException ($"Excessive amount of uncompressed data, exceeding the maximum by {uncompressed_offset - UInt32.MaxValue}"); + } + } + } + void ConstructEmptyModule () { throw new NotImplementedException (); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssemblyInfo.cs b/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssemblyInfo.cs index 0923982d025..04233196a25 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssemblyInfo.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssemblyInfo.cs @@ -43,12 +43,12 @@ class DSOAssemblyInfo /// /// Size of the loadable assembly data (after decompression, if compression is enabled). /// - public ulong DataSize { get; } + public uint DataSize { get; } /// /// Size of the compressed assembly data or `0` if assembly is uncompressed. /// - public ulong CompressedDataSize { get; } + public uint CompressedDataSize { get; } public string InputFile { get; } @@ -64,7 +64,7 @@ class DSOAssemblyInfo /// gives the original file size, while specifies /// data size after compression, or `0` if file isn't compressed. /// - public DSOAssemblyInfo (string name, string inputFile, ulong dataSize, ulong compressedDataSize) + public DSOAssemblyInfo (string name, string inputFile, uint dataSize, uint compressedDataSize) { Name = name; InputFile = inputFile; From 6295d31ceefe38e4a74c8b738c88f51da090e556 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 12 Sep 2023 21:58:21 +0200 Subject: [PATCH 41/71] Moving forward --- .../GenerateAssemblyDsoNativeSourceFiles.cs | 29 ++++- .../Utilities/AssemblyDSOGenerator.cs | 119 +++++++++++++++--- .../LlvmIrGenerator/LlvmIrComposer.cs | 7 +- src/monodroid/jni/application_dso_stub.cc | 7 ++ src/monodroid/jni/xamarin-app.hh | 17 ++- 5 files changed, 153 insertions(+), 26 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAssemblyDsoNativeSourceFiles.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAssemblyDsoNativeSourceFiles.cs index 1dd85f4b7b7..4395227c311 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAssemblyDsoNativeSourceFiles.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAssemblyDsoNativeSourceFiles.cs @@ -38,7 +38,8 @@ public override bool RunTask () Log.LogDebugMessage ("Assembly compression DISABLED"); } - ulong assemblyDataSize = 0; + ulong inputAssemblyDataSize = 0; + ulong uncompressedAssemblyDataSize = 0; Log.LogDebugMessage ("Processing the input assemblies"); foreach (ITaskItem assembly in Assemblies) { FileInfo fi = new (assembly.ItemSpec); @@ -61,7 +62,9 @@ public override bool RunTask () compressed = false; compressedSize = 0; } - assemblyDataSize += compressedSize == 0 ? (ulong)fi.Length : compressedSize; + inputAssemblyDataSize += compressedSize == 0 ? (ulong)fi.Length : compressedSize; + uncompressedAssemblyDataSize += (ulong)fi.Length; + Log.LogDebugMessage ($" will include from: {inputFile} (compressed? {compressed}; compressedSize == {compressedSize}"); AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (assembly); if (!dsoAssembliesInfo.TryGetValue (arch, out List? assemblyList)) { @@ -72,15 +75,33 @@ public override bool RunTask () Log.LogDebugMessage ($" added to list with name: {assemblyList[assemblyList.Count - 1].Name}"); } - Log.LogDebugMessage ($"Size of assembly data to stash: {assemblyDataSize}"); + Log.LogDebugMessage ($"Size of assembly data to stash: {inputAssemblyDataSize}"); Log.LogDebugMessage ($"Number of architectures to stash into DSOs: {dsoAssembliesInfo.Count}"); foreach (var kvp in dsoAssembliesInfo) { Log.LogDebugMessage ($" {kvp.Key}: {kvp.Value.Count} assemblies"); } - var generator = new AssemblyDSOGenerator (dsoAssembliesInfo); + var generator = new AssemblyDSOGenerator (dsoAssembliesInfo, inputAssemblyDataSize, uncompressedAssemblyDataSize); LLVMIR.LlvmIrModule module = generator.Construct (); + foreach (string abi in SupportedAbis) { + string targetAbi = abi.ToLowerInvariant (); + string outputAsmFilePath = Path.Combine (SourcesOutputDirectory, $"{PrepareAbiItems.AssemblyDSOBase}.{targetAbi}.ll"); + + using var sw = MemoryStreamPool.Shared.CreateStreamWriter (); + try { + generator.Generate (module, GeneratePackageManagerJava.GetAndroidTargetArchForAbi (abi), sw, outputAsmFilePath); + } catch { + throw; + } finally { + sw.Flush (); + } + + if (Files.CopyIfStreamChanged (sw.BaseStream, outputAsmFilePath)) { + Log.LogDebugMessage ($"File {outputAsmFilePath} was (re)generated"); + } + } + return !Log.HasLoggedErrors; } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs index 2adf587876e..140c329e3d1 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs @@ -1,5 +1,7 @@ using System; +using System.Buffers; using System.Collections.Generic; +using System.IO; using Xamarin.Android.Tasks.LLVMIR; using Xamarin.Android.Tools; @@ -11,6 +13,9 @@ class AssemblyDSOGenerator : LlvmIrComposer // Must be identical to AssemblyEntry in src/monodroid/jni/xamarin-app.hh sealed class AssemblyEntry { + [NativeAssembler (Ignore = true)] + public byte[] AssemblyData; + // offset into the `xa_input_assembly_data` array public uint input_data_offset; @@ -28,21 +33,30 @@ sealed class AssemblyEntry // Must be identical to AssemblyIndexEntry in src/monodroid/jni/xamarin-app.hh sealed class AssemblyIndexEntry { - ulong name_hash; + [NativeAssembler (Ignore = true)] + public string Name; + + public ulong name_hash; // Index into the `xa_assemblies` descriptor array - uint index; + public uint index; } + // Must be identical to AssembliesConfig in src/monodroid/jni/xamarin-app.hh + sealed class AssembliesConfig + { + public uint input_assembly_data_size; + public uint uncompressed_assembly_data_size; + public uint assembly_name_length; + public uint assembly_count; + }; + // Members with underscores correspond to the native fields we output. sealed class ArchState { - public readonly List xa_input_assembly_data = new List (); - public ulong xa_uncompressed_assembly_data_length = 0; public readonly List xa_assemblies; public readonly List xa_assembly_index; - public ulong assemblyNameLength = 0; - public readonly List xa_assembly_names = new List (); + public readonly AssembliesConfig xa_assemblies_config; public ArchState (int assemblyCount) { @@ -52,18 +66,39 @@ public ArchState (int assemblyCount) xa_assemblies = new List (assemblyCount); xa_assembly_index = new List (assemblyCount); + xa_assemblies_config = new AssembliesConfig { + input_assembly_data_size = 0, + uncompressed_assembly_data_size = 0, + assembly_name_length = 0, + assembly_count = (uint)assemblyCount, + }; } } + readonly ArrayPool bytePool = ArrayPool.Shared; readonly Dictionary> assemblies; readonly Dictionary assemblyArchStates; + readonly uint inputAssemblyDataSize; + readonly uint uncompressedAssemblyDataSize; StructureInfo? assemblyEntryStructureInfo; StructureInfo? assemblyIndexEntryStructureInfo; + StructureInfo? assembliesConfigStructureInfo; - public AssemblyDSOGenerator (Dictionary> dsoAssemblies) + public AssemblyDSOGenerator (Dictionary> dsoAssemblies, ulong inputAssemblyDataSize, ulong uncompressedAssemblyDataSize) { + this.inputAssemblyDataSize = EnsureValidSize (inputAssemblyDataSize, nameof (inputAssemblyDataSize)); + this.uncompressedAssemblyDataSize = EnsureValidSize (uncompressedAssemblyDataSize, nameof (uncompressedAssemblyDataSize)); assemblies = dsoAssemblies; assemblyArchStates = new Dictionary (); + + uint EnsureValidSize (ulong v, string name) + { + if (v > UInt32.MaxValue) { + throw new ArgumentOutOfRangeException (name, "must not exceed UInt32.MaxValue"); + } + + return (uint)v; + } } protected override void Construct (LlvmIrModule module) @@ -93,9 +128,8 @@ protected override void Construct (LlvmIrModule module) if (expectedCount <= 0) { ConstructEmptyModule (); + return; } - - throw new NotImplementedException (); } void AddAssemblyData (AndroidTargetArch arch, List infos) @@ -109,21 +143,73 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) assemblyArchStates.Add (arch, archState); } - ulong uncompressed_offset = 0; + bool is64Bit = arch switch { + AndroidTargetArch.Arm => false, + AndroidTargetArch.X86 => false, + AndroidTargetArch.Arm64 => true, + AndroidTargetArch.X86_64 => true, + _ => throw new NotSupportedException ($"Architecture '{arch}' is not supported") + }; + + ulong inputOffset = 0; + ulong uncompressedOffset = 0; + ulong assemblyNameLength = 0; foreach (DSOAssemblyInfo info in infos) { + uint inputSize = info.CompressedDataSize == 0 ? (uint)info.DataSize : (uint)info.CompressedDataSize; + if (inputSize > Int32.MaxValue) { + throw new InvalidOperationException ($"Assembly {info.InputFile} size exceeds 2GB"); + } + + // We need to read each file into a separate array, as it is (theoretically) possible that all the assemblies data will exceed 2GB, + // which is the limit of we can allocate (or rent, below) in .NET. + // + // We also need to read all the assemblies for all the target ABIs, as it is possible that **all** of them will be different. + // + // All the data will then be concatenated on write time into a single native array. var entry = new AssemblyEntry { - input_data_offset = (uint)archState.xa_input_assembly_data.Count, - input_data_size = info.CompressedDataSize == 0 ? (uint)info.DataSize : (uint)info.CompressedDataSize, + AssemblyData = bytePool.Rent ((int)inputSize), + input_data_offset = (uint)inputOffset, + input_data_size = inputSize, uncompressed_data_size = info.CompressedDataSize == 0 ? 0 : (uint)info.DataSize, - uncompressed_data_offset = (uint)uncompressed_offset, + uncompressed_data_offset = (uint)uncompressedOffset, }; + inputOffset = AddWithCheck (inputOffset, inputSize, UInt32.MaxValue, "Input data too long"); - uncompressed_offset += entry.uncompressed_data_offset; + using (var asmFile = File.Open (info.InputFile, FileMode.Open, FileAccess.Read, FileShare.Read)) { + asmFile.Read (entry.AssemblyData, 0, entry.AssemblyData.Length); + } // This is way, way more than Google Play Store supports now, but we won't limit ourselves more than we have to - if (uncompressed_offset > UInt32.MaxValue) { - throw new InvalidOperationException ($"Excessive amount of uncompressed data, exceeding the maximum by {uncompressed_offset - UInt32.MaxValue}"); + uncompressedOffset = AddWithCheck (uncompressedOffset, entry.uncompressed_data_offset, UInt32.MaxValue, "Compressed data too long"); + archState.xa_assemblies.Add (entry); + + byte[] nameBytes = StringToBytes (info.Name); + if ((ulong)nameBytes.Length > assemblyNameLength) { + assemblyNameLength = (ulong)nameBytes.Length; } + + var index = new AssemblyIndexEntry { + Name = info.Name, + name_hash = GetXxHash (nameBytes, is64Bit), + index = (uint)archState.xa_assemblies.Count - 1, + }; + archState.xa_assembly_index.Add (index); + } + + archState.xa_assemblies_config.input_assembly_data_size = (uint)inputOffset; + archState.xa_assemblies_config.uncompressed_assembly_data_size = (uint)uncompressedOffset; + + // Must include the terminating NUL + archState.xa_assemblies_config.assembly_name_length = (uint)assemblyNameLength + 1; + + ulong AddWithCheck (ulong lhs, ulong rhs, ulong maxValue, string errorMessage) + { + ulong v = lhs + rhs; + if (v > maxValue) { + throw new InvalidOperationException ($"{errorMessage}, exceeding the maximum by {uncompressedOffset - maxValue}"); + } + + return v; } } @@ -136,5 +222,6 @@ void MapStructures (LlvmIrModule module) { assemblyEntryStructureInfo = module.MapStructure (); assemblyIndexEntryStructureInfo = module.MapStructure (); + assembliesConfigStructureInfo = module.MapStructure (); } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs index 8db94269f32..d37476a0b63 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs @@ -34,9 +34,12 @@ public void Generate (LlvmIrModule module, AndroidTargetArch arch, StreamWriter output.Flush (); } - public static ulong GetXxHash (string str, bool is64Bit) + public static byte[] StringToBytes (string str) => Encoding.UTF8.GetBytes (str); + + public static ulong GetXxHash (string str, bool is64Bit) => GetXxHash (StringToBytes (str), is64Bit); + + public static ulong GetXxHash (byte[] stringBytes, bool is64Bit) { - byte[] stringBytes = Encoding.UTF8.GetBytes (str); if (is64Bit) { return XxHash64.HashToUInt64 (stringBytes); } diff --git a/src/monodroid/jni/application_dso_stub.cc b/src/monodroid/jni/application_dso_stub.cc index a8d4c7a67a7..5f46ab026ae 100644 --- a/src/monodroid/jni/application_dso_stub.cc +++ b/src/monodroid/jni/application_dso_stub.cc @@ -326,3 +326,10 @@ const JniRemappingTypeReplacementEntry jni_remapping_type_replacements[] = { .replacement = "another/replacement/java/type", }, }; + +const AssembliesConfig xa_assemblies_config = { + .input_assembly_data_size = InputAssemblyDataSize, + .uncompressed_assembly_data_size = UncompressedAssemblyDataSize, + .assembly_name_length = AssemblyNameLength, + .assembly_count = AssemblyCount, +}; diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index ca081218805..fd129d42c2a 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -135,11 +135,20 @@ struct AssemblyIndexEntry uint32_t index; }; -constexpr size_t InputAssemblyDataSize = 1024; -constexpr size_t UncompressedAssemblyDataSize = 2048; -constexpr size_t AssemblyCount = 2; -constexpr size_t AssemblyNameLength = 26; // including the terminating NUL +constexpr uint32_t InputAssemblyDataSize = 1024; +constexpr uint32_t UncompressedAssemblyDataSize = 2048; +constexpr uint32_t AssemblyCount = 2; +constexpr uint32_t AssemblyNameLength = 26; // including the terminating NUL +struct AssembliesConfig +{ + uint32_t input_assembly_data_size; + uint32_t uncompressed_assembly_data_size; + uint32_t assembly_name_length; + uint32_t assembly_count; +}; + +MONO_API MONO_API_EXPORT const AssembliesConfig xa_assemblies_config; MONO_API MONO_API_EXPORT const uint8_t xa_input_assembly_data[InputAssemblyDataSize]; // All the compressed assemblies are uncompressed into this array, with offsets in `xa_assemblies` From b4fae1af55770d9ab9a1fa44b83a7d258e9b04a9 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 13 Sep 2023 22:32:01 +0200 Subject: [PATCH 42/71] Beginnings of "streamed" array writing Also: - use hexadecimal notation for unsigned integers - remove some superfluous comments from the generated code --- ...pplicationConfigNativeAssemblyGenerator.cs | 2 +- .../Utilities/AssemblyDSOGenerator.cs | 10 + .../LlvmIrGenerator/LlvmIrGenerator.cs | 238 +++++++++++++----- .../LlvmIrGenerator/LlvmIrVariable.cs | 54 ++++ .../MarshalMethodsNativeAssemblyGenerator.cs | 2 +- ...peMappingReleaseNativeAssemblyGenerator.cs | 2 +- 6 files changed, 248 insertions(+), 60 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs index 2dcbac7b0e4..15a4e41a63f 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs @@ -28,7 +28,7 @@ public override string GetComment (object data, string fieldName) { var dso_entry = EnsureType (data); if (String.Compare ("hash", fieldName, StringComparison.Ordinal) == 0) { - return $" hash 0x{dso_entry.hash:x}, from name: {dso_entry.HashedName}"; + return $" hash, from name: {dso_entry.HashedName}"; } if (String.Compare ("name", fieldName, StringComparison.Ordinal) == 0) { diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs index 140c329e3d1..051ebf911b1 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs @@ -75,6 +75,8 @@ public ArchState (int assemblyCount) } } + const string XAUncompressedAssemblyDataVarName = "xa_uncompressed_assembly_data"; + readonly ArrayPool bytePool = ArrayPool.Shared; readonly Dictionary> assemblies; readonly Dictionary assemblyArchStates; @@ -130,6 +132,14 @@ protected override void Construct (LlvmIrModule module) ConstructEmptyModule (); return; } + + var xa_uncompressed_assembly_data = new LlvmIrGlobalVariable (typeof(byte[]), XAUncompressedAssemblyDataVarName) { + Alignment = 4096, // align to page boundary, may make access slightly faster + ArrayItemCount = uncompressedAssemblyDataSize, + Options = LlvmIrVariableOptions.GlobalWritable, + ZeroInitializeArray = true, + }; + module.Add (xa_uncompressed_assembly_data); } void AddAssemblyData (AndroidTargetArch arch, List infos) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs index bb81b03db0e..3152ea968f6 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs @@ -80,31 +80,56 @@ sealed class BasicType public readonly string Name; public readonly ulong Size; public readonly bool IsNumeric; + public readonly bool IsUnsigned; + public readonly bool PreferHex; + public readonly string HexFormat; - public BasicType (string name, ulong size, bool isNumeric = true) + public BasicType (string name, ulong size, bool isNumeric = true, bool isUnsigned = false, bool? preferHex = null) { Name = name; Size = size; IsNumeric = isNumeric; + IsUnsigned = isUnsigned; + + // If hex preference isn't specified, we determine whether the type wants to be represented in + // the hexadecimal notation based on signedness. Unsigned types will be represented in hexadecimal, + // but signed types will remain decimal, as it's easier for humans to see the actual value of the + // variable, given this note from LLVM IR manual: + // + // Note that hexadecimal integers are sign extended from the number of active bits, i.e. the bit width minus the number of leading zeros. So ‘s0x0001’ of type ‘i16’ will be -1, not 1. + // + // See: https://llvm.org/docs/LangRef.html#simple-constants + // + if (preferHex.HasValue) { + PreferHex = preferHex.Value; + } else { + PreferHex = isUnsigned; + } + if (!PreferHex) { + HexFormat = String.Empty; + return; + } + + HexFormat = $"x{size * 2}"; } } public const string IRPointerType = "ptr"; static readonly Dictionary basicTypeMap = new Dictionary { - { typeof (bool), new ("i8", 1, isNumeric: false) }, - { typeof (byte), new ("i8", 1) }, - { typeof (char), new ("i16", 2) }, + { typeof (bool), new ("i1", 1, isNumeric: false, isUnsigned: true, preferHex: false) }, + { typeof (byte), new ("i8", 1, isUnsigned: true) }, + { typeof (char), new ("i16", 2, isUnsigned: true, preferHex: false) }, { typeof (sbyte), new ("i8", 1) }, { typeof (short), new ("i16", 2) }, - { typeof (ushort), new ("i16", 2) }, + { typeof (ushort), new ("i16", 2, isUnsigned: true) }, { typeof (int), new ("i32", 4) }, - { typeof (uint), new ("i32", 4) }, + { typeof (uint), new ("i32", 4, isUnsigned: true) }, { typeof (long), new ("i64", 8) }, - { typeof (ulong), new ("i64", 8) }, + { typeof (ulong), new ("i64", 8, isUnsigned: true) }, { typeof (float), new ("float", 4) }, { typeof (double), new ("double", 8) }, - { typeof (void), new ("void", 0, isNumeric: false) }, + { typeof (void), new ("void", 0, isNumeric: false, preferHex: false) }, }; public string FilePath { get; } @@ -240,7 +265,9 @@ void WriteGlobalVariable (GeneratorWriteContext context, LlvmIrGlobalVariable va context.Output.Write (", align "); ulong alignment; - if (typeInfo.IsAggregate) { + if (variable.Alignment.HasValue) { + alignment = variable.Alignment.Value; + } else if (typeInfo.IsAggregate) { ulong count = GetAggregateValueElementCount (variable); alignment = (ulong)target.GetAggregateAlignment ((int)typeInfo.MaxFieldAlignment, count * typeInfo.Size); } else if (typeInfo.IsStructure) { @@ -449,6 +476,11 @@ ulong GetStructureMaxFieldAlignment (StructureInfo si) void WriteValue (GeneratorWriteContext context, Type valueType, LlvmIrVariable variable) { + if (variable is LlvmIrGlobalVariable globalVariable && globalVariable.ArrayDataProvider != null) { + WriteStreamedArrayValue (context, globalVariable, globalVariable.ArrayDataProvider); + return; + } + if (variable.Type.IsArray ()) { bool zeroInitialize = false; if (variable is LlvmIrGlobalVariable gv) { @@ -549,6 +581,27 @@ bool WriteNativePointerValue (GeneratorWriteContext context, StructureInstance s return false; } + string ToHex (BasicType basicTypeDesc, Type type, object? value) + { + const char prefixSigned = 's'; + const char prefixUnsigned = 'u'; + + string hex; + if (type == typeof(byte)) { + hex = ((byte)value).ToString (basicTypeDesc.HexFormat, CultureInfo.InvariantCulture); + } else if (type == typeof(ushort)) { + hex = ((ushort)value).ToString (basicTypeDesc.HexFormat, CultureInfo.InvariantCulture); + } else if (type == typeof(uint)) { + hex = ((uint)value).ToString (basicTypeDesc.HexFormat, CultureInfo.InvariantCulture); + } else if (type == typeof(ulong)) { + hex = ((ulong)value).ToString (basicTypeDesc.HexFormat, CultureInfo.InvariantCulture); + } else { + throw new NotImplementedException ($"Conversion to hexadecimal from type {type} is not implemented"); + }; + + return $"{(basicTypeDesc.IsUnsigned ? prefixUnsigned : prefixSigned)}0x{hex}"; + } + void WriteValue (GeneratorWriteContext context, Type type, object? value) { if (value is LlvmIrVariable variableRef) { @@ -556,14 +609,19 @@ void WriteValue (GeneratorWriteContext context, Type type, object? value) return; } - if (IsNumeric (type)) { - context.Output.Write (MonoAndroidHelper.CultureInvariantToString (value)); - return; - } + bool isBasic = basicTypeMap.TryGetValue (type, out BasicType basicTypeDesc); + if (isBasic) { + if (basicTypeDesc.IsNumeric) { + context.Output.Write ( + basicTypeDesc.PreferHex ? ToHex (basicTypeDesc, type, value) : MonoAndroidHelper.CultureInvariantToString (value) + ); + return; + } - if (type == typeof(bool)) { - context.Output.Write ((bool)value ? '1' : '0'); - return; + if (type == typeof(bool)) { + context.Output.Write ((bool)value ? "true" : "false"); + return; + } } if (IsStructureInstance (type)) { @@ -628,9 +686,6 @@ void WriteStructureValue (GeneratorWriteContext context, StructureInstance? inst sb.Append (MapManagedTypeToNative (smi)); sb.Append (' '); sb.Append (smi.Info.Name); - if (value != null && smi.MemberType.IsPrimitive && smi.MemberType != typeof(bool)) { - sb.Append ($" (0x{value:x})"); - } comment = sb.ToString (); } WriteCommentLine (context, comment); @@ -641,41 +696,33 @@ void WriteStructureValue (GeneratorWriteContext context, StructureInstance? inst context.Output.Write ('}'); } - void WriteArrayValue (GeneratorWriteContext context, LlvmIrVariable variable) + void WriteArrayValueStart (GeneratorWriteContext context) { - ICollection entries; - if (variable.Type.ImplementsInterface (typeof(IDictionary))) { - var list = new List (); - foreach (var kvp in (IDictionary)variable.Value) { - list.Add (kvp.Key); - list.Add (kvp.Value); - } - entries = list; - } else { - entries = (ICollection)variable.Value; - } - - if (entries.Count == 0) { - context.Output.Write ("zeroinitializer"); - return; - } - context.Output.WriteLine ('['); context.IncreaseIndent (); + } - Type elementType = variable.Type.GetArrayElementType (); - bool writeIndices = (variable.WriteOptions & LlvmIrVariableWriteOptions.ArrayWriteIndexComments) == LlvmIrVariableWriteOptions.ArrayWriteIndexComments; - ulong counter = 0; - string? prevItemComment = null; - uint stride; + void WriteArrayValueEnd (GeneratorWriteContext context) + { + context.DecreaseIndent (); + context.Output.Write (']'); + } + uint GetArrayStride (LlvmIrVariable variable) + { if ((variable.WriteOptions & LlvmIrVariableWriteOptions.ArrayFormatInRows) == LlvmIrVariableWriteOptions.ArrayFormatInRows) { - stride = variable.ArrayStride > 0 ? variable.ArrayStride : 1; - } else { - stride = 1; + return variable.ArrayStride > 0 ? variable.ArrayStride : 1; } + return 1; + } + + void WriteArrayEntries (GeneratorWriteContext context, LlvmIrVariable variable, ICollection entries, Type elementType, uint stride, bool writeIndices) + { bool first = true; + bool ignoreComments = stride > 1; + string? prevItemComment = null; + ulong counter = 0; // TODO: implement output in rows foreach (object entry in entries) { @@ -686,13 +733,15 @@ void WriteArrayValue (GeneratorWriteContext context, LlvmIrVariable variable) first = false; } - prevItemComment = null; - if (variable.GetArrayItemCommentCallback != null) { - prevItemComment = variable.GetArrayItemCommentCallback (variable, target, counter, entry, variable.GetArrayItemCommentCallbackCallerState); - } + if (!ignoreComments) { + prevItemComment = null; + if (variable.GetArrayItemCommentCallback != null) { + prevItemComment = variable.GetArrayItemCommentCallback (variable, target, counter, entry, variable.GetArrayItemCommentCallbackCallerState); + } - if (writeIndices && String.IsNullOrEmpty (prevItemComment)) { - prevItemComment = $" {counter}"; + if (writeIndices && String.IsNullOrEmpty (prevItemComment)) { + prevItemComment = $" {counter}"; + } } counter++; @@ -704,12 +753,9 @@ void WriteArrayValue (GeneratorWriteContext context, LlvmIrVariable variable) } WritePrevItemCommentOrNewline (); - context.DecreaseIndent (); - context.Output.Write (']'); - void WritePrevItemCommentOrNewline () { - if (!String.IsNullOrEmpty (prevItemComment)) { + if (!ignoreComments && !String.IsNullOrEmpty (prevItemComment)) { context.Output.Write (' '); WriteCommentLine (context, prevItemComment); } else { @@ -718,6 +764,86 @@ void WritePrevItemCommentOrNewline () } } + bool ArrayWantsToWriteIndices (LlvmIrVariable variable) => (variable.WriteOptions & LlvmIrVariableWriteOptions.ArrayWriteIndexComments) == LlvmIrVariableWriteOptions.ArrayWriteIndexComments; + + void WriteStreamedArrayValue (GeneratorWriteContext context, LlvmIrGlobalVariable variable, LlvmIrStreamedArrayDataProvider dataProvider) + { + ulong itemCount = 0; + bool first = true; + + WriteArrayValueStart (context); + while (true) { + (LlvmIrStreamedArrayDataProviderState state, ICollection? data) = dataProvider.GetData (); + if (state == LlvmIrStreamedArrayDataProviderState.Finished) { + break; + } + + if (data == null || data.Count == 0) { + throw new InvalidOperationException ("Data must be provided for streamed arrays"); + } + + string comment; + if (state == LlvmIrStreamedArrayDataProviderState.StartSection) { + if (first) { + first = false; + } else { + context.Output.WriteLine (); + } + comment = dataProvider.GetSectionStartComment (); + } else { + comment = String.Empty; + } + + if (comment.Length > 0) { + WriteCommentLine (context, comment); + } + + WriteArrayEntries ( + context, + variable, + data, + dataProvider.ArrayMemberType, + GetArrayStride (variable), + writeIndices: ArrayWantsToWriteIndices (variable) + ); + + } + WriteArrayValueEnd (context); + } + + void WriteArrayValue (GeneratorWriteContext context, LlvmIrVariable variable) + { + ICollection entries; + if (variable.Type.ImplementsInterface (typeof(IDictionary))) { + var list = new List (); + foreach (var kvp in (IDictionary)variable.Value) { + list.Add (kvp.Key); + list.Add (kvp.Value); + } + entries = list; + } else { + entries = (ICollection)variable.Value; + } + + if (entries.Count == 0) { + context.Output.Write ("zeroinitializer"); + return; + } + + WriteArrayValueStart (context); + + WriteArrayEntries ( + context, + variable, + entries, + variable.Type.GetArrayElementType (), + GetArrayStride (variable), + writeIndices: ArrayWantsToWriteIndices (variable) + ); + + WriteArrayValueEnd (context); + } + void WriteLinkage (GeneratorWriteContext context, LlvmIrLinkage linkage) { if (linkage == LlvmIrLinkage.Default) { @@ -1232,8 +1358,6 @@ static string MapManagedTypeToNative (StructureMemberInfo smi) return $"{nativeType}*"; } - static bool IsNumeric (Type type) => basicTypeMap.TryGetValue (type, out BasicType typeDesc) && typeDesc.IsNumeric; - object? GetTypedMemberValue (GeneratorWriteContext context, StructureInfo info, StructureMemberInfo smi, StructureInstance instance, Type expectedType, object? defaultValue = null) { object? value = smi.GetValue (instance.Obj); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs index 7f741490ce0..e676107b9ed 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs @@ -1,4 +1,5 @@ using System; +using System.Collections; using System.Globalization; namespace Xamarin.Android.Tasks.LLVMIR; @@ -146,6 +147,35 @@ public void AssignNumber (ulong n) } } +enum LlvmIrStreamedArrayDataProviderState +{ + Finished, + StartSection, +} + +abstract class LlvmIrStreamedArrayDataProvider +{ + /// + /// Type of every member of the array returned by . Generator will check + /// every member type against this property, allowing also derived types. + /// + public Type ArrayMemberType { get; } + + protected LlvmIrStreamedArrayDataProvider (Type arrayMemberType) + { + ArrayMemberType = arrayMemberType; + } + + /// + /// Whenever returns the + /// generator will call this method to obtain the new section comment, if any, to be output before the actual + /// data. Returning `String.Empty` prevents the comment from being added. + /// + public virtual string GetSectionStartComment () => String.Empty; + + public abstract (LlvmIrStreamedArrayDataProviderState status, ICollection? data) GetData (); +} + class LlvmIrGlobalVariable : LlvmIrVariable { /// @@ -162,9 +192,33 @@ class LlvmIrGlobalVariable : LlvmIrVariable /// public virtual LlvmIrVariableOptions? Options { get; set; } + /// + /// If set to `true`, initialize the array with a shortcut zero-initializer statement. Useful when pre-allocating + /// space for runtime use that won't be filled in with any data at the build time. + /// public bool ZeroInitializeArray { get; set; } + + /// + /// Specify number of items in an array. Used in cases when we want to pre-allocate an array without giving it any + /// value, thus making it impossible for the generator to discover the number of items automatically. This is useful + /// when using . This property is used **only** if the variable + /// is `null`. + /// public ulong ArrayItemCount { get; set; } + /// + /// If set, it will override any automatically calculated alignment for this variable + /// + public ulong? Alignment { get; set; } + + /// + /// If set, the provider will be called to obtain all the data to be placed in an array variable. The total amount + /// of data that will be returned by the provider **must** be specified in the property, + /// in order for the generator to properly declare the variable. The generator will verify that the amount of data + /// is exactly that much and throw an exception otherwise. + /// + public LlvmIrStreamedArrayDataProvider? ArrayDataProvider { get; set; } + /// /// Constructs a local variable. is translated to one of the LLVM IR first class types (see /// https://llvm.org/docs/LangRef.html#t-firstclass) only if it's an integral or floating point type. In all other cases it diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs index f87cf118971..f0c2ece5e6b 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs @@ -1066,7 +1066,7 @@ void UpdateAssemblyImageCacheHashes (LlvmIrVariable variable, LlvmIrModuleTarget i = acs.Hashes32[v32].index; } - return $" {index}: {name} => 0x{value:x} => {i}"; + return $" {index}: {name} => {i}"; } void UpdateAssemblyImageCacheIndices (LlvmIrVariable variable, LlvmIrModuleTarget target, object? callerState) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs index a6e4fd465df..5c889365ddb 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs @@ -258,7 +258,7 @@ uint GetJavaEntryIndex (TypeMapJava javaEntry) throw new InvalidOperationException ("Internal error: construction state expected but not found"); } - return $" {index}: 0x{value:x} => {cs.JavaMap[(int)index].Instance.JavaName}"; + return $" {index}: {cs.JavaMap[(int)index].Instance.JavaName}"; } void GenerateAndSortJavaHashes (LlvmIrVariable variable, LlvmIrModuleTarget target, object? callerState) From 2a9cb28b8d44d11bd539140b7fd604d320505ceb Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 14 Sep 2023 23:23:20 +0200 Subject: [PATCH 43/71] First streamed array done --- .../Utilities/AssemblyDSOGenerator.cs | 136 +++++++++++++++++- .../LlvmIrGenerator/LlvmIrComposer.cs | 5 + .../LlvmIrGenerator/LlvmIrGenerator.cs | 73 ++++++---- .../LlvmIrGenerator/LlvmIrVariable.cs | 33 +++-- 4 files changed, 206 insertions(+), 41 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs index 051ebf911b1..f9e58dff4d4 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs @@ -1,5 +1,5 @@ using System; -using System.Buffers; +using System.Collections; using System.Collections.Generic; using System.IO; @@ -14,7 +14,10 @@ class AssemblyDSOGenerator : LlvmIrComposer sealed class AssemblyEntry { [NativeAssembler (Ignore = true)] - public byte[] AssemblyData; + public byte[]? AssemblyData; + + [NativeAssembler (Ignore = true)] + public string InputFilePath; // offset into the `xa_input_assembly_data` array public uint input_data_offset; @@ -75,9 +78,110 @@ public ArchState (int assemblyCount) } } + sealed class AssemblyInputDataArrayProvider : LlvmIrStreamedArrayDataProvider + { + sealed class DataState + { + public int Index = 0; + public string Comment = String.Empty; + public ulong TotalDataSize = 0; + } + + Dictionary assemblyArchStates; + Dictionary dataStates; + + public AssemblyInputDataArrayProvider (Type arrayElementType, Dictionary assemblyArchStates) + : base (arrayElementType) + { + this.assemblyArchStates = assemblyArchStates; + + dataStates = new Dictionary (); + foreach (var kvp in assemblyArchStates) { + dataStates.Add (kvp.Key, new DataState ()); + } + } + + public override (LlvmIrStreamedArrayDataProviderState status, ICollection data) GetData (LlvmIrModuleTarget target) + { + ArchState archState = GetArchState (target); + DataState dataState = GetDataState (target); + int index = dataState.Index++; + if (index >= archState.xa_assemblies.Count) { + throw new InvalidOperationException ("Internal error: no more data left"); + } + + AssemblyEntry entry = archState.xa_assemblies[index]; + AssemblyIndexEntry indexEntry = archState.xa_assembly_index[index]; + string compressed = entry.uncompressed_data_size == 0 ? "no" : "yes"; + + dataState.Comment = $" Assembly: {indexEntry.Name} ({entry.InputFilePath}); Data size: {entry.AssemblyData.Length}; compressed: {compressed}"; + // Each assembly is a new "section" + return ( + index == archState.xa_assemblies.Count - 1 ? LlvmIrStreamedArrayDataProviderState.LastSection : LlvmIrStreamedArrayDataProviderState.NextSection, + EnsureValidAssemblyData (entry) + ); + } + + public override ulong GetTotalDataSize (LlvmIrModuleTarget target) + { + DataState dataState = GetDataState (target); + if (dataState.TotalDataSize > 0) { + return dataState.TotalDataSize; + } + + ArchState archState = GetArchState (target); + ulong totalSize = 0; + foreach (AssemblyEntry entry in archState.xa_assemblies) { + byte[] data = EnsureValidAssemblyData (entry); + totalSize += (ulong)data.Length; + } + + return dataState.TotalDataSize = totalSize; + } + + public override string GetSectionStartComment (LlvmIrModuleTarget target) + { + DataState dataState = GetDataState (target); + string ret = dataState.Comment; + dataState.Comment = String.Empty; + return ret; + } + + DataState GetDataState (LlvmIrModuleTarget target) + { + if (!dataStates.TryGetValue (target.TargetArch, out DataState dataState)) { + throw new InvalidOperationException ($"Internal error: data state for ABI {target.TargetArch} not available"); + } + + return dataState; + } + + ArchState GetArchState (LlvmIrModuleTarget target) + { + if (!assemblyArchStates.TryGetValue (target.TargetArch, out ArchState archState)) { + throw new InvalidOperationException ($"Internal error: architecture state for ABI {target.TargetArch} not available"); + } + + return archState; + } + + byte[] EnsureValidAssemblyData (AssemblyEntry entry) + { + if (entry.AssemblyData == null) { + throw new InvalidOperationException ("Internal error: assembly data must be present"); + } + + if (entry.AssemblyData.Length == 0) { + throw new InvalidOperationException ("Internal error: assembly data must not be empty"); + } + + return entry.AssemblyData; + } + } + const string XAUncompressedAssemblyDataVarName = "xa_uncompressed_assembly_data"; + const string XAInputAssemblyDataVarName = "xa_input_assembly_data"; - readonly ArrayPool bytePool = ArrayPool.Shared; readonly Dictionary> assemblies; readonly Dictionary assemblyArchStates; readonly uint inputAssemblyDataSize; @@ -140,6 +244,26 @@ protected override void Construct (LlvmIrModule module) ZeroInitializeArray = true, }; module.Add (xa_uncompressed_assembly_data); + + var xa_input_assembly_data = new LlvmIrGlobalVariable (typeof(byte[]), XAInputAssemblyDataVarName) { + Alignment = 4096, + ArrayDataProvider = new AssemblyInputDataArrayProvider (typeof(byte), assemblyArchStates), + ArrayStride = 16, + Options = LlvmIrVariableOptions.GlobalConstant, + WriteOptions = LlvmIrVariableWriteOptions.ArrayFormatInRows, + }; + module.Add (xa_input_assembly_data); + } + + protected override void CleanupAfterGeneration (AndroidTargetArch arch) + { + if (!assemblyArchStates.TryGetValue (arch, out ArchState archState)) { + throw new InvalidOperationException ($"Internal error: data for ABI {arch} not available"); + } + + foreach (AssemblyEntry entry in archState.xa_assemblies) { + entry.AssemblyData = null; // Help the GC a bit + } } void AddAssemblyData (AndroidTargetArch arch, List infos) @@ -166,6 +290,7 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) ulong assemblyNameLength = 0; foreach (DSOAssemblyInfo info in infos) { uint inputSize = info.CompressedDataSize == 0 ? (uint)info.DataSize : (uint)info.CompressedDataSize; + if (inputSize > Int32.MaxValue) { throw new InvalidOperationException ($"Assembly {info.InputFile} size exceeds 2GB"); } @@ -177,7 +302,10 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) // // All the data will then be concatenated on write time into a single native array. var entry = new AssemblyEntry { - AssemblyData = bytePool.Rent ((int)inputSize), + // We can't use the byte pool here, even though it would be more efficient, because the generator expects an ICollection, + // which it then iterates on, and the rented arrays can (and frequently will) be bigger than the requested size. + AssemblyData = new byte[inputSize], + InputFilePath = info.InputFile, input_data_offset = (uint)inputOffset, input_data_size = inputSize, uncompressed_data_size = info.CompressedDataSize == 0 ? 0 : (uint)info.DataSize, diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs index d37476a0b63..2baa5c0fcc6 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs @@ -32,8 +32,13 @@ public void Generate (LlvmIrModule module, AndroidTargetArch arch, StreamWriter LlvmIrGenerator generator = LlvmIrGenerator.Create (arch, fileName); generator.Generate (output, module); output.Flush (); + + CleanupAfterGeneration (arch); } + protected virtual void CleanupAfterGeneration (AndroidTargetArch arch) + {} + public static byte[] StringToBytes (string str) => Encoding.UTF8.GetBytes (str); public static ulong GetXxHash (string str, bool is64Bit) => GetXxHash (StringToBytes (str), is64Bit); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs index 3152ea968f6..c761f5e09a4 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs @@ -268,7 +268,7 @@ void WriteGlobalVariable (GeneratorWriteContext context, LlvmIrGlobalVariable va if (variable.Alignment.HasValue) { alignment = variable.Alignment.Value; } else if (typeInfo.IsAggregate) { - ulong count = GetAggregateValueElementCount (variable); + ulong count = GetAggregateValueElementCount (context, variable); alignment = (ulong)target.GetAggregateAlignment ((int)typeInfo.MaxFieldAlignment, count * typeInfo.Size); } else if (typeInfo.IsStructure) { alignment = (ulong)target.GetAggregateAlignment ((int)typeInfo.MaxFieldAlignment, typeInfo.Size); @@ -317,9 +317,9 @@ void WriteTypeAndValue (GeneratorWriteContext context, LlvmIrVariable variable, WriteValue (context, valueType, variable); } - ulong GetAggregateValueElementCount (LlvmIrVariable variable) => GetAggregateValueElementCount (variable.Type, variable.Value, variable as LlvmIrGlobalVariable); + ulong GetAggregateValueElementCount (GeneratorWriteContext context, LlvmIrVariable variable) => GetAggregateValueElementCount (context, variable.Type, variable.Value, variable as LlvmIrGlobalVariable); - ulong GetAggregateValueElementCount (Type type, object? value, LlvmIrGlobalVariable? globalVariable = null) + ulong GetAggregateValueElementCount (GeneratorWriteContext context, Type type, object? value, LlvmIrGlobalVariable? globalVariable = null) { if (!type.IsArray ()) { throw new InvalidOperationException ($"Internal error: unknown type {type} when trying to determine aggregate type element count"); @@ -327,6 +327,9 @@ ulong GetAggregateValueElementCount (Type type, object? value, LlvmIrGlobalVaria if (value == null) { if (globalVariable != null) { + if (globalVariable.ArrayDataProvider != null) { + return globalVariable.ArrayDataProvider.GetTotalDataSize (context.Target); + } return globalVariable.ArrayItemCount; } return 0; @@ -413,7 +416,7 @@ void WriteType (GeneratorWriteContext context, Type type, object? value, out Llv if (type.IsArray ()) { Type elementType = type.GetArrayElementType (); - ulong elementCount = GetAggregateValueElementCount (type, value, globalVariable); + ulong elementCount = GetAggregateValueElementCount (context, type, value, globalVariable); WriteArrayType (context, elementType, elementCount, out typeInfo); return; @@ -486,7 +489,7 @@ void WriteValue (GeneratorWriteContext context, Type valueType, LlvmIrVariable v if (variable is LlvmIrGlobalVariable gv) { zeroInitialize = gv.ZeroInitializeArray || variable.Value == null; } else { - zeroInitialize = GetAggregateValueElementCount (variable) == 0; + zeroInitialize = GetAggregateValueElementCount (context, variable) == 0; } if (zeroInitialize) { @@ -708,7 +711,7 @@ void WriteArrayValueEnd (GeneratorWriteContext context) context.Output.Write (']'); } - uint GetArrayStride (LlvmIrVariable variable) + uint GetArrayStride (LlvmIrVariable variable) { if ((variable.WriteOptions & LlvmIrVariableWriteOptions.ArrayFormatInRows) == LlvmIrVariableWriteOptions.ArrayFormatInRows) { return variable.ArrayStride > 0 ? variable.ArrayStride : 1; @@ -717,19 +720,24 @@ uint GetArrayStride (LlvmIrVariable variable) return 1; } - void WriteArrayEntries (GeneratorWriteContext context, LlvmIrVariable variable, ICollection entries, Type elementType, uint stride, bool writeIndices) + void WriteArrayEntries (GeneratorWriteContext context, LlvmIrVariable variable, ICollection entries, Type elementType, uint stride, bool writeIndices, bool terminateWithComma = false) { bool first = true; bool ignoreComments = stride > 1; string? prevItemComment = null; ulong counter = 0; - // TODO: implement output in rows foreach (object entry in entries) { if (!first) { context.Output.Write (','); - WritePrevItemCommentOrNewline (); + if (stride == 1 || counter % stride == 0) { + WritePrevItemCommentOrNewline (); + context.Output.Write (context.CurrentIndent); + } else { + context.Output.Write (' '); + } } else { + context.Output.Write (context.CurrentIndent); first = false; } @@ -745,12 +753,19 @@ void WriteArrayEntries (GeneratorWriteContext context, LlvmIrVariable variable, } counter++; - context.Output.Write (context.CurrentIndent); WriteType (context, elementType, entry, out _); context.Output.Write (' '); WriteValue (context, elementType, entry); } + + if (terminateWithComma) { + if (!ignoreComments) { + context.Output.WriteLine (); // must put comma outside the comment + context.Output.Write (context.CurrentIndent); + } + context.Output.Write (','); + } WritePrevItemCommentOrNewline (); void WritePrevItemCommentOrNewline () @@ -768,45 +783,49 @@ void WritePrevItemCommentOrNewline () void WriteStreamedArrayValue (GeneratorWriteContext context, LlvmIrGlobalVariable variable, LlvmIrStreamedArrayDataProvider dataProvider) { - ulong itemCount = 0; + ulong dataSizeSoFar = 0; + ulong totalDataSize = dataProvider.GetTotalDataSize (context.Target); bool first = true; WriteArrayValueStart (context); while (true) { - (LlvmIrStreamedArrayDataProviderState state, ICollection? data) = dataProvider.GetData (); - if (state == LlvmIrStreamedArrayDataProviderState.Finished) { - break; + (LlvmIrStreamedArrayDataProviderState state, ICollection data) = dataProvider.GetData (context.Target); + if (data.Count == 0) { + throw new InvalidOperationException ("Data must be provided for streamed arrays"); } - if (data == null || data.Count == 0) { - throw new InvalidOperationException ("Data must be provided for streamed arrays"); + dataSizeSoFar += (ulong)data.Count; + if (dataSizeSoFar > totalDataSize) { + throw new InvalidOperationException ($"Data provider {dataProvider} is trying to write more data than declared"); } - string comment; - if (state == LlvmIrStreamedArrayDataProviderState.StartSection) { - if (first) { - first = false; - } else { - context.Output.WriteLine (); - } - comment = dataProvider.GetSectionStartComment (); + if (first) { + first = false; } else { - comment = String.Empty; + context.Output.WriteLine (); } + string comment = dataProvider.GetSectionStartComment (context.Target); if (comment.Length > 0) { + context.Output.Write (context.CurrentIndent); WriteCommentLine (context, comment); } + bool lastSection = state == LlvmIrStreamedArrayDataProviderState.LastSection; WriteArrayEntries ( context, variable, data, - dataProvider.ArrayMemberType, + dataProvider.ArrayElementType, GetArrayStride (variable), - writeIndices: ArrayWantsToWriteIndices (variable) + writeIndices: false, + terminateWithComma: !lastSection ); + if (lastSection) { + break; + } + } WriteArrayValueEnd (context); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs index e676107b9ed..5cddb2ebd65 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs @@ -149,8 +149,8 @@ public void AssignNumber (ulong n) enum LlvmIrStreamedArrayDataProviderState { - Finished, - StartSection, + NextSection, + LastSection, } abstract class LlvmIrStreamedArrayDataProvider @@ -159,21 +159,34 @@ abstract class LlvmIrStreamedArrayDataProvider /// Type of every member of the array returned by . Generator will check /// every member type against this property, allowing also derived types. /// - public Type ArrayMemberType { get; } + public Type ArrayElementType { get; } - protected LlvmIrStreamedArrayDataProvider (Type arrayMemberType) + protected LlvmIrStreamedArrayDataProvider (Type arrayElementType) { - ArrayMemberType = arrayMemberType; + ArrayElementType = arrayElementType; } /// - /// Whenever returns the - /// generator will call this method to obtain the new section comment, if any, to be output before the actual - /// data. Returning `String.Empty` prevents the comment from being added. + /// Whenever returns the generator will call this method to obtain the new section + /// comment, if any, to be output before the actual data. Returning `String.Empty` prevents the comment + /// from being added. /// - public virtual string GetSectionStartComment () => String.Empty; + public virtual string GetSectionStartComment (LlvmIrModuleTarget target) => String.Empty; - public abstract (LlvmIrStreamedArrayDataProviderState status, ICollection? data) GetData (); + /// + /// Provide the next chunk of data for the specified target (ABI). Implementations need to return at least one + /// non-empty collection of data. The returned collection **must** be exactly the size of contained data (e.g. it cannot be + /// a byte array rented from a byte pool, because these can be bigger than requested. When returning the last (or the only) section, + /// must have a value of . + /// Each section may be preceded by a comment, . + /// + public abstract (LlvmIrStreamedArrayDataProviderState status, ICollection data) GetData (LlvmIrModuleTarget target); + + /// + /// Provide the total data size for the specified target (ABI). This needs to be used instead of + /// because a variable instance is created once and shared by all targets, while per-target data sets might have different sizes. + /// + public abstract ulong GetTotalDataSize (LlvmIrModuleTarget target); } class LlvmIrGlobalVariable : LlvmIrVariable From 3ea330a622e2efee036d5a8348819fc09a4bc9f4 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 15 Sep 2023 22:38:45 +0200 Subject: [PATCH 44/71] Better formatting, more ABI differences, almost done on generation --- .../Utilities/ApplicationConfig.cs | 9 + ...pplicationConfigNativeAssemblyGenerator.cs | 1 + .../Utilities/AssemblyDSOGenerator.cs | 191 +++++++++++++++--- .../LlvmIrGenerator/LlvmIrGenerator.cs | 24 ++- .../LlvmIrGenerator/LlvmIrVariable.cs | 9 + .../LlvmIrGenerator/MemberInfoUtilities.cs | 10 + .../NativeAssemblerAttribute.cs | 2 + .../MarshalMethodsNativeAssemblyGenerator.cs | 2 + ...peMappingReleaseNativeAssemblyGenerator.cs | 4 + 9 files changed, 220 insertions(+), 32 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfig.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfig.cs index 25550eb3473..21fb00a63f5 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfig.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfig.cs @@ -42,11 +42,20 @@ sealed class ApplicationConfig public uint bundled_assembly_name_width; public uint number_of_assembly_store_files; public uint number_of_dso_cache_entries; + + [NativeAssembler (NumberFormat = LLVMIR.LlvmIrVariableNumberFormat.Hexadecimal)] public uint android_runtime_jnienv_class_token; + + [NativeAssembler (NumberFormat = LLVMIR.LlvmIrVariableNumberFormat.Hexadecimal)] public uint jnienv_initialize_method_token; + + [NativeAssembler (NumberFormat = LLVMIR.LlvmIrVariableNumberFormat.Hexadecimal)] public uint jnienv_registerjninatives_method_token; + public uint jni_remapping_replacement_type_count; public uint jni_remapping_replacement_method_index_entry_count; + + [NativeAssembler (NumberFormat = LLVMIR.LlvmIrVariableNumberFormat.Hexadecimal)] public uint mono_components_mask; public string android_package_name = String.Empty; } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs index 15a4e41a63f..294cb9cb5a9 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs @@ -235,6 +235,7 @@ protected override void Construct (LlvmIrModule module) var dso_cache = new LlvmIrGlobalVariable (dsoCache, "dso_cache", LlvmIrVariableOptions.GlobalWritable) { Comment = " DSO cache entries", BeforeWriteCallback = HashAndSortDSOCache, + NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal, }; module.Add (dso_cache); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs index f9e58dff4d4..7b6afd00212 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs @@ -34,17 +34,24 @@ sealed class AssemblyEntry } // Must be identical to AssemblyIndexEntry in src/monodroid/jni/xamarin-app.hh - sealed class AssemblyIndexEntry + class AssemblyIndexEntryBase { [NativeAssembler (Ignore = true)] public string Name; - public ulong name_hash; + [NativeAssembler (NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal)] + public T name_hash; // Index into the `xa_assemblies` descriptor array public uint index; } + sealed class AssemblyIndexEntry32 : AssemblyIndexEntryBase + {} + + sealed class AssemblyIndexEntry64 : AssemblyIndexEntryBase + {} + // Must be identical to AssembliesConfig in src/monodroid/jni/xamarin-app.hh sealed class AssembliesConfig { @@ -57,18 +64,34 @@ sealed class AssembliesConfig // Members with underscores correspond to the native fields we output. sealed class ArchState { - public readonly List xa_assemblies; - public readonly List xa_assembly_index; + public readonly List> xa_assemblies; + public readonly List>? xa_assembly_index32; + public readonly List>? xa_assembly_index64; public readonly AssembliesConfig xa_assemblies_config; - public ArchState (int assemblyCount) + public ArchState (int assemblyCount, AndroidTargetArch arch) { if (assemblyCount < 0) { throw new ArgumentException ("must not be a negative number", nameof (assemblyCount)); } - xa_assemblies = new List (assemblyCount); - xa_assembly_index = new List (assemblyCount); + xa_assemblies = new List> (assemblyCount); + + switch (arch) { + case AndroidTargetArch.Arm64: + case AndroidTargetArch.X86_64: + xa_assembly_index64 = new List> (assemblyCount); + break; + + case AndroidTargetArch.Arm: + case AndroidTargetArch.X86: + xa_assembly_index32 = new List> (assemblyCount); + break; + + default: + throw new InvalidOperationException ($"Internal error: architecture {arch} not supported"); + } + xa_assemblies_config = new AssembliesConfig { input_assembly_data_size = 0, uncompressed_assembly_data_size = 0, @@ -110,11 +133,20 @@ public override (LlvmIrStreamedArrayDataProviderState status, ICollection data) throw new InvalidOperationException ("Internal error: no more data left"); } - AssemblyEntry entry = archState.xa_assemblies[index]; - AssemblyIndexEntry indexEntry = archState.xa_assembly_index[index]; + var entry = (AssemblyEntry)archState.xa_assemblies[index].Obj; + string name; + + if (target.TargetArch == AndroidTargetArch.Arm64 || target.TargetArch == AndroidTargetArch.X86_64) { + name = ((AssemblyIndexEntry64)archState.xa_assembly_index64[index].Obj).Name; + } else if (target.TargetArch == AndroidTargetArch.Arm || target.TargetArch == AndroidTargetArch.X86) { + name = ((AssemblyIndexEntry32)archState.xa_assembly_index32[index].Obj).Name; + } else { + throw new InvalidOperationException ($"Internal error: architecture {target.TargetArch} not supported"); + } + string compressed = entry.uncompressed_data_size == 0 ? "no" : "yes"; - dataState.Comment = $" Assembly: {indexEntry.Name} ({entry.InputFilePath}); Data size: {entry.AssemblyData.Length}; compressed: {compressed}"; + dataState.Comment = $" Assembly: {name} ({entry.InputFilePath}); Data size: {entry.AssemblyData.Length}; compressed: {compressed}"; // Each assembly is a new "section" return ( index == archState.xa_assemblies.Count - 1 ? LlvmIrStreamedArrayDataProviderState.LastSection : LlvmIrStreamedArrayDataProviderState.NextSection, @@ -131,7 +163,8 @@ public override ulong GetTotalDataSize (LlvmIrModuleTarget target) ArchState archState = GetArchState (target); ulong totalSize = 0; - foreach (AssemblyEntry entry in archState.xa_assemblies) { + foreach (StructureInstance si in archState.xa_assemblies) { + var entry = (AssemblyEntry)si.Obj; byte[] data = EnsureValidAssemblyData (entry); totalSize += (ulong)data.Length; } @@ -156,14 +189,7 @@ DataState GetDataState (LlvmIrModuleTarget target) return dataState; } - ArchState GetArchState (LlvmIrModuleTarget target) - { - if (!assemblyArchStates.TryGetValue (target.TargetArch, out ArchState archState)) { - throw new InvalidOperationException ($"Internal error: architecture state for ABI {target.TargetArch} not available"); - } - - return archState; - } + ArchState GetArchState (LlvmIrModuleTarget target) => AssemblyDSOGenerator.GetArchState (target, assemblyArchStates); byte[] EnsureValidAssemblyData (AssemblyEntry entry) { @@ -179,15 +205,19 @@ byte[] EnsureValidAssemblyData (AssemblyEntry entry) } } + const string XAAssembliesConfigVarName = "xa_assemblies_config"; + const string XAAssemblyIndexVarName = "xa_assembly_index"; + const string XAAssemblyNamesVarName = "xa_assembly_names"; + const string XAInputAssemblyDataVarName = "xa_input_assembly_data"; const string XAUncompressedAssemblyDataVarName = "xa_uncompressed_assembly_data"; - const string XAInputAssemblyDataVarName = "xa_input_assembly_data"; readonly Dictionary> assemblies; readonly Dictionary assemblyArchStates; readonly uint inputAssemblyDataSize; readonly uint uncompressedAssemblyDataSize; StructureInfo? assemblyEntryStructureInfo; - StructureInfo? assemblyIndexEntryStructureInfo; + StructureInfo? assemblyIndexEntry32StructureInfo; + StructureInfo? assemblyIndexEntry64StructureInfo; StructureInfo? assembliesConfigStructureInfo; public AssemblyDSOGenerator (Dictionary> dsoAssemblies, ulong inputAssemblyDataSize, ulong uncompressedAssemblyDataSize) @@ -237,6 +267,24 @@ protected override void Construct (LlvmIrModule module) return; } + var xa_assemblies_config = new LlvmIrGlobalVariable (typeof(StructureInstance), XAAssembliesConfigVarName) { + BeforeWriteCallback = AssembliesConfigBeforeWrite, + Options = LlvmIrVariableOptions.GlobalConstant, + }; + module.Add (xa_assemblies_config); + + // var xa_assembly_index = new LlvmIrGlobalVariable (typeof(List>), XAAssemblyIndexVarName) { + // BeforeWriteCallback = AssemblyIndexBeforeWrite, + // Options = LlvmIrVariableOptions.GlobalConstant, + // }; + // module.Add (xa_assembly_index); + + var xa_assembly_names = new LlvmIrGlobalVariable (typeof(List), XAAssemblyNamesVarName) { + BeforeWriteCallback = AssemblyNamesBeforeWrite, + Options = LlvmIrVariableOptions.GlobalConstant, + }; + module.Add (xa_assembly_names); + var xa_uncompressed_assembly_data = new LlvmIrGlobalVariable (typeof(byte[]), XAUncompressedAssemblyDataVarName) { Alignment = 4096, // align to page boundary, may make access slightly faster ArrayItemCount = uncompressedAssemblyDataSize, @@ -249,19 +297,79 @@ protected override void Construct (LlvmIrModule module) Alignment = 4096, ArrayDataProvider = new AssemblyInputDataArrayProvider (typeof(byte), assemblyArchStates), ArrayStride = 16, + NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal, Options = LlvmIrVariableOptions.GlobalConstant, WriteOptions = LlvmIrVariableWriteOptions.ArrayFormatInRows, }; module.Add (xa_input_assembly_data); } + void AssemblyNamesBeforeWrite (LlvmIrVariable variable, LlvmIrModuleTarget target, object? state) + { + ArchState archState = GetArchState (target); + var names = new List (); + if (target.TargetArch == AndroidTargetArch.Arm64 || target.TargetArch == AndroidTargetArch.X86_64) { + foreach (StructureInstance e in archState.xa_assembly_index64) { + var entry = (AssemblyIndexEntry64)e.Obj; + names.Add (entry.Name); + } + } else if (target.TargetArch == AndroidTargetArch.Arm || target.TargetArch == AndroidTargetArch.X86) { + foreach (StructureInstance e in archState.xa_assembly_index32) { + var entry = (AssemblyIndexEntry32)e.Obj; + names.Add (entry.Name); + } + } else { + throw new InvalidOperationException ($"Internal error: architecture {target.TargetArch} not supported"); + } + + variable.Value = names; + } + + void AssemblyIndexBeforeWrite (LlvmIrVariable variable, LlvmIrModuleTarget target, object? state) + { + ArchState archState = GetArchState (target); + var gv = (LlvmIrGlobalVariable)variable; + object value; + Type type; + + if (target.TargetArch == AndroidTargetArch.Arm64 || target.TargetArch == AndroidTargetArch.X86_64) { + value = archState.xa_assembly_index64; + type = archState.xa_assembly_index64.GetType (); + } else if (target.TargetArch == AndroidTargetArch.Arm || target.TargetArch == AndroidTargetArch.X86) { + value = archState.xa_assembly_index32; + type = archState.xa_assembly_index32.GetType (); + } else { + throw new InvalidOperationException ($"Internal error: architecture {target.TargetArch} not supported"); + } + + gv.OverrideValueAndType (type, value); + } + + void AssembliesConfigBeforeWrite (LlvmIrVariable variable, LlvmIrModuleTarget target, object? state) + { + ArchState archState = GetArchState (target); + variable.Value = new StructureInstance (assembliesConfigStructureInfo, archState.xa_assemblies_config); + } + + ArchState GetArchState (LlvmIrModuleTarget target) => GetArchState (target, assemblyArchStates); + + static ArchState GetArchState (LlvmIrModuleTarget target, Dictionary archStates) + { + if (!archStates.TryGetValue (target.TargetArch, out ArchState archState)) { + throw new InvalidOperationException ($"Internal error: architecture state for ABI {target.TargetArch} not available"); + } + + return archState; + } + protected override void CleanupAfterGeneration (AndroidTargetArch arch) { if (!assemblyArchStates.TryGetValue (arch, out ArchState archState)) { throw new InvalidOperationException ($"Internal error: data for ABI {arch} not available"); } - foreach (AssemblyEntry entry in archState.xa_assemblies) { + foreach (StructureInstance si in archState.xa_assemblies) { + var entry = (AssemblyEntry)si.Obj; entry.AssemblyData = null; // Help the GC a bit } } @@ -273,7 +381,7 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) } if (!assemblyArchStates.TryGetValue (arch, out ArchState archState)) { - archState = new ArchState (infos.Count); + archState = new ArchState (infos.Count, arch); assemblyArchStates.Add (arch, archState); } @@ -319,21 +427,41 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) // This is way, way more than Google Play Store supports now, but we won't limit ourselves more than we have to uncompressedOffset = AddWithCheck (uncompressedOffset, entry.uncompressed_data_offset, UInt32.MaxValue, "Compressed data too long"); - archState.xa_assemblies.Add (entry); + archState.xa_assemblies.Add (new StructureInstance (assemblyEntryStructureInfo, entry)); byte[] nameBytes = StringToBytes (info.Name); if ((ulong)nameBytes.Length > assemblyNameLength) { assemblyNameLength = (ulong)nameBytes.Length; } - var index = new AssemblyIndexEntry { - Name = info.Name, - name_hash = GetXxHash (nameBytes, is64Bit), - index = (uint)archState.xa_assemblies.Count - 1, - }; - archState.xa_assembly_index.Add (index); + if (is64Bit) { + var indexEntry = new AssemblyIndexEntry64 { + Name = info.Name, + name_hash = GetXxHash (nameBytes, is64Bit), + index = (uint)archState.xa_assemblies.Count - 1, + }; + archState.xa_assembly_index64.Add (new StructureInstance (assemblyIndexEntry64StructureInfo, indexEntry)); + } else { + var indexEntry = new AssemblyIndexEntry32 { + Name = info.Name, + name_hash = (uint)GetXxHash (nameBytes, is64Bit), + index = (uint)archState.xa_assemblies.Count - 1, + }; + archState.xa_assembly_index32.Add (new StructureInstance (assemblyIndexEntry32StructureInfo, indexEntry)); + } + } + + if (is64Bit) { + archState.xa_assembly_index64.Sort ( + (StructureInstance a, StructureInstance b) => ((AssemblyIndexEntry64)a.Obj).name_hash.CompareTo (((AssemblyIndexEntry64)b.Obj).name_hash) + ); + } else { + archState.xa_assembly_index32.Sort ( + (StructureInstance a, StructureInstance b) => ((AssemblyIndexEntry32)a.Obj).name_hash.CompareTo (((AssemblyIndexEntry32)b.Obj).name_hash) + ); } + archState.xa_assemblies_config.assembly_count = (uint)archState.xa_assemblies.Count; archState.xa_assemblies_config.input_assembly_data_size = (uint)inputOffset; archState.xa_assemblies_config.uncompressed_assembly_data_size = (uint)uncompressedOffset; @@ -359,7 +487,8 @@ void ConstructEmptyModule () void MapStructures (LlvmIrModule module) { assemblyEntryStructureInfo = module.MapStructure (); - assemblyIndexEntryStructureInfo = module.MapStructure (); + assemblyIndexEntry32StructureInfo = module.MapStructure (); + assemblyIndexEntry64StructureInfo = module.MapStructure (); assembliesConfigStructureInfo = module.MapStructure (); } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs index c761f5e09a4..f4346f0e08c 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs @@ -29,6 +29,7 @@ sealed class GeneratorWriteContext public readonly LlvmIrMetadataManager MetadataManager; public string CurrentIndent { get; private set; } = String.Empty; public bool InVariableGroup { get; set; } + public LlvmIrVariableNumberFormat NumberFormat { get; set; } = LlvmIrVariableNumberFormat.Default; public GeneratorWriteContext (TextWriter writer, LlvmIrModule module, LlvmIrModuleTarget target, LlvmIrMetadataManager metadataManager) { @@ -216,6 +217,8 @@ void WriteGlobalVariables (GeneratorWriteContext context) } foreach (LlvmIrGlobalVariable gv in context.Module.GlobalVariables) { + context.NumberFormat = gv.NumberFormat; + if (gv is LlvmIrGroupDelimiterVariable groupDelimiter) { if (!context.InVariableGroup && !String.IsNullOrEmpty (groupDelimiter.Comment)) { context.Output.WriteLine (); @@ -615,8 +618,15 @@ void WriteValue (GeneratorWriteContext context, Type type, object? value) bool isBasic = basicTypeMap.TryGetValue (type, out BasicType basicTypeDesc); if (isBasic) { if (basicTypeDesc.IsNumeric) { + bool hex = context.NumberFormat switch { + LlvmIrVariableNumberFormat.Default => basicTypeDesc.PreferHex, + LlvmIrVariableNumberFormat.Decimal => false, + LlvmIrVariableNumberFormat.Hexadecimal => true, + _ => throw new InvalidOperationException ($"Internal error: number format {context.NumberFormat} is unsupported") + }; + context.Output.Write ( - basicTypeDesc.PreferHex ? ToHex (basicTypeDesc, type, value) : MonoAndroidHelper.CultureInvariantToString (value) + hex ? ToHex (basicTypeDesc, type, value) : MonoAndroidHelper.CultureInvariantToString (value) ); return; } @@ -677,8 +687,20 @@ void WriteStructureValue (GeneratorWriteContext context, StructureInstance? inst context.Output.Write (' '); object? value = GetTypedMemberValue (context, info, smi, instance, smi.MemberType); + LlvmIrVariableNumberFormat numberFormat = smi.Info.GetNumberFormat (); + LlvmIrVariableNumberFormat? savedNumberFormat = null; + + if (numberFormat != LlvmIrVariableNumberFormat.Default && numberFormat != context.NumberFormat) { + savedNumberFormat = context.NumberFormat; + context.NumberFormat = numberFormat; + } + WriteValue (context, instance, smi, value); + if (savedNumberFormat.HasValue) { + context.NumberFormat = savedNumberFormat.Value; + } + if (i < lastMember) { context.Output.Write (", "); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs index 5cddb2ebd65..5bb66450a68 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs @@ -12,6 +12,13 @@ enum LlvmIrVariableWriteOptions ArrayFormatInRows = 0x0002, } +enum LlvmIrVariableNumberFormat +{ + Default, + Hexadecimal, + Decimal, +} + abstract class LlvmIrVariable : IEquatable { public abstract bool Global { get; } @@ -30,6 +37,8 @@ abstract class LlvmIrVariable : IEquatable public object? Value { get; set; } public string? Comment { get; set; } + public LlvmIrVariableNumberFormat NumberFormat { get; set; } = LlvmIrVariableNumberFormat.Decimal; + /// /// Both global and local variables will want their names to matter in equality checks, but function /// parameters must not take it into account, thus this property. If set to false, diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/MemberInfoUtilities.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/MemberInfoUtilities.cs index 431d92b6229..713b0c5a1a6 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/MemberInfoUtilities.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/MemberInfoUtilities.cs @@ -71,5 +71,15 @@ public static bool InlineArrayNeedsPadding (this MemberInfo mi) return attr.NeedsPadding; } + + public static LlvmIrVariableNumberFormat GetNumberFormat (this MemberInfo mi) + { + var attr = mi.GetCustomAttribute (); + if (attr == null) { + return LlvmIrVariableNumberFormat.Default; + } + + return attr.NumberFormat; + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/NativeAssemblerAttribute.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/NativeAssemblerAttribute.cs index 39fc5e094bb..c239255e1ac 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/NativeAssemblerAttribute.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/NativeAssemblerAttribute.cs @@ -33,6 +33,8 @@ class NativeAssemblerAttribute : Attribute /// size to which the member must be padded is specified by /// public bool NeedsPadding { get; set; } + + public LLVMIR.LlvmIrVariableNumberFormat NumberFormat { get; set; } = LLVMIR.LlvmIrVariableNumberFormat.Default; } [AttributeUsage (AttributeTargets.Class, Inherited = true)] diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs index f0c2ece5e6b..da2f4a07b2a 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs @@ -922,6 +922,7 @@ void AddMarshalMethodNames (LlvmIrModule module, AssemblyCacheState acs) var mm_method_names_variable = new LlvmIrGlobalVariable (mm_method_names, "mm_method_names", LlvmIrVariableOptions.GlobalConstant) { BeforeWriteCallback = UpdateMarshalMethodNameIds, BeforeWriteCallbackCallerState = acs, + NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal, }; module.Add (mm_method_names_variable); @@ -1021,6 +1022,7 @@ void AddAssemblyImageCache (LlvmIrModule module, out AssemblyCacheState acs) BeforeWriteCallbackCallerState = acs, GetArrayItemCommentCallback = GetAssemblyImageCacheItemComment, GetArrayItemCommentCallbackCallerState = acs, + NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal, }; module.Add (assembly_image_cache_hashes); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs index 5c889365ddb..84c0033e8a6 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs @@ -69,6 +69,7 @@ sealed class TypeMapModuleEntry [NativeAssembler (Ignore = true)] public TypeMapJava JavaTypeMapEntry; + [NativeAssembler (NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal)] public uint type_token_id; public uint java_map_index; } @@ -126,6 +127,8 @@ sealed class TypeMapJava public ulong JavaNameHash64; public uint module_index; + + [NativeAssembler (NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal)] public uint type_token_id; public uint java_name_index; } @@ -212,6 +215,7 @@ protected override void Construct (LlvmIrModule module) BeforeWriteCallbackCallerState = cs, GetArrayItemCommentCallback = GetJavaHashesItemComment, GetArrayItemCommentCallbackCallerState = cs, + NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal, }; map_java_hashes.WriteOptions &= ~LlvmIrVariableWriteOptions.ArrayWriteIndexComments; module.Add (map_java_hashes); From b811e299a7ff6b61624e9a6caf123ae3540e5a13 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 18 Sep 2023 22:15:54 +0200 Subject: [PATCH 45/71] Managed side is feature complete --- .../Utilities/AssemblyDSOGenerator.cs | 55 ++++++++++++++++--- .../LlvmIrGenerator/LlvmIrGenerator.cs | 33 ++++++++++- 2 files changed, 76 insertions(+), 12 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs index 7b6afd00212..83e75d6ba0f 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs @@ -39,6 +39,9 @@ class AssemblyIndexEntryBase [NativeAssembler (Ignore = true)] public string Name; + [NativeAssembler (Ignore = true)] + public byte[] NameBytes; + [NativeAssembler (NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal)] public T name_hash; @@ -273,13 +276,14 @@ protected override void Construct (LlvmIrModule module) }; module.Add (xa_assemblies_config); - // var xa_assembly_index = new LlvmIrGlobalVariable (typeof(List>), XAAssemblyIndexVarName) { - // BeforeWriteCallback = AssemblyIndexBeforeWrite, - // Options = LlvmIrVariableOptions.GlobalConstant, - // }; - // module.Add (xa_assembly_index); + var xa_assembly_index = new LlvmIrGlobalVariable (typeof(List>), XAAssemblyIndexVarName) { + BeforeWriteCallback = AssemblyIndexBeforeWrite, + GetArrayItemCommentCallback = AssemblyIndexItemComment, + Options = LlvmIrVariableOptions.GlobalConstant, + }; + module.Add (xa_assembly_index); - var xa_assembly_names = new LlvmIrGlobalVariable (typeof(List), XAAssemblyNamesVarName) { + var xa_assembly_names = new LlvmIrGlobalVariable (typeof(List), XAAssemblyNamesVarName) { BeforeWriteCallback = AssemblyNamesBeforeWrite, Options = LlvmIrVariableOptions.GlobalConstant, }; @@ -304,25 +308,56 @@ protected override void Construct (LlvmIrModule module) module.Add (xa_input_assembly_data); } + string AssemblyIndexItemComment (LlvmIrVariable variable, LlvmIrModuleTarget target, ulong index, object? itemValue, object? state) + { + var value32 = itemValue as StructureInstance; + if (value32 != null) { + return MakeComment (((AssemblyIndexEntry32)value32.Obj).Name); + } + + var value64 = itemValue as StructureInstance; + if (value64 != null) { + return MakeComment (((AssemblyIndexEntry64)value64.Obj).Name); + } + + throw new InvalidOperationException ($"Internal error: assembly index array member has unsupported type '{itemValue?.GetType ()}'"); + + string MakeComment (string name) => $" => {name}"; + } + void AssemblyNamesBeforeWrite (LlvmIrVariable variable, LlvmIrModuleTarget target, object? state) { ArchState archState = GetArchState (target); - var names = new List (); + var names = new List (); + if (target.TargetArch == AndroidTargetArch.Arm64 || target.TargetArch == AndroidTargetArch.X86_64) { foreach (StructureInstance e in archState.xa_assembly_index64) { var entry = (AssemblyIndexEntry64)e.Obj; - names.Add (entry.Name); + names.Add (GetProperlySizedBytes (entry.NameBytes)); } } else if (target.TargetArch == AndroidTargetArch.Arm || target.TargetArch == AndroidTargetArch.X86) { foreach (StructureInstance e in archState.xa_assembly_index32) { var entry = (AssemblyIndexEntry32)e.Obj; - names.Add (entry.Name); + names.Add (GetProperlySizedBytes (entry.NameBytes)); } } else { throw new InvalidOperationException ($"Internal error: architecture {target.TargetArch} not supported"); } variable.Value = names; + + byte[] GetProperlySizedBytes (byte[] inputBytes) + { + if (inputBytes.Length > archState.xa_assemblies_config.assembly_name_length - 1) { + throw new ArgumentOutOfRangeException (nameof (inputBytes), $"Must not exceed {archState.xa_assemblies_config.assembly_name_length - 1} bytes"); + } + + var ret = new byte[archState.xa_assemblies_config.assembly_name_length]; + Array.Clear (ret, 0, ret.Length); + inputBytes.CopyTo (ret, 0); + + return ret; + } } void AssemblyIndexBeforeWrite (LlvmIrVariable variable, LlvmIrModuleTarget target, object? state) @@ -437,6 +472,7 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) if (is64Bit) { var indexEntry = new AssemblyIndexEntry64 { Name = info.Name, + NameBytes = nameBytes, name_hash = GetXxHash (nameBytes, is64Bit), index = (uint)archState.xa_assemblies.Count - 1, }; @@ -444,6 +480,7 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) } else { var indexEntry = new AssemblyIndexEntry32 { Name = info.Name, + NameBytes = nameBytes, name_hash = (uint)GetXxHash (nameBytes, is64Bit), index = (uint)archState.xa_assemblies.Count - 1, }; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs index f4346f0e08c..129a25463b9 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs @@ -516,6 +516,29 @@ void AssertArraySize (StructureInstance si, StructureMemberInfo smi, ulong lengt throw new InvalidOperationException ($"Invalid array size in field '{smi.Info.Name}' of structure '{si.Info.Name}', expected {expectedLength}, found {length}"); } + void WriteInlineArray (GeneratorWriteContext context, byte[] bytes, bool encodeAsASCII) + { + if (encodeAsASCII) { + context.Output.Write ('c'); + context.Output.Write (QuoteString (bytes, bytes.Length, out _, nullTerminated: false)); + return; + } + + string irType = MapToIRType (typeof(byte)); + bool first = true; + context.Output.Write ("[ "); + foreach (byte b in bytes) { + if (!first) { + context.Output.Write (", "); + } else { + first = false; + } + + context.Output.Write ($"{irType} u0x{b:x02}"); + } + context.Output.Write (" ]"); + } + void WriteValue (GeneratorWriteContext context, StructureInstance structInstance, StructureMemberInfo smi, object? value) { if (smi.IsNativePointer) { @@ -533,8 +556,7 @@ void WriteValue (GeneratorWriteContext context, StructureInstance structInstance // Byte arrays are represented in the same way as strings, without the explicit NUL termination byte AssertArraySize (structInstance, smi, length, smi.ArrayElements); - context.Output.Write ('c'); - context.Output.Write (QuoteString (bytes, bytes.Length, out _, nullTerminated: false)); + WriteInlineArray (context, bytes, encodeAsASCII: false); return; } @@ -659,8 +681,13 @@ void WriteValue (GeneratorWriteContext context, Type type, object? value) return; } - if (type.IsInlineArray ()) { + if (type.IsArray) { + if (type == typeof(byte[])) { + WriteInlineArray (context, (byte[])value, encodeAsASCII: true); + return; + } + throw new NotSupportedException ($"Internal error: array of type {type} is unsupported"); } throw new NotSupportedException ($"Internal error: value type '{type}' is unsupported"); From 248298c85ae4656823305c4a98fd37b9355f192e Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 19 Sep 2023 11:06:44 +0200 Subject: [PATCH 46/71] libxamarin-app.so has it all now --- .../Microsoft.Android.Sdk.BuildOrder.targets | 1 - ...soft.Android.Sdk.NativeCompilation.targets | 53 ++++--------------- .../LlvmIrGenerator/LlvmIrGenerator.cs | 36 ++++++++++++- 3 files changed, 44 insertions(+), 46 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.BuildOrder.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.BuildOrder.targets index f9f8f860db9..88c037f3841 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.BuildOrder.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.BuildOrder.targets @@ -67,7 +67,6 @@ properties that determine build ordering. _GenerateEnvironmentFiles; _CompileJava; _CreateApplicationSharedLibraries; - _CreateAssemblySharedLibraries; _CompileDex; $(_AfterCompileDex); _CreateBaseApk; diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets index dd5a7879a10..e0d089286a3 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets @@ -57,7 +57,8 @@ - + %(_AndroidRemapAssemblySource.abi) - - - - - <_NativeAssemblySharedLibrariesTarget Include="@(_AssemblyDSOSource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> + <_NativeAssemblyTarget Include="@(_AssemblyDSOSource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> %(_AssemblyDSOSource.abi) - + @@ -119,25 +116,16 @@ - - - <_AssemblySharedLibrary Include="$(_AndroidApplicationSharedLibraryPath)%(_BuildTargetAbis.Identity)\libxamarin-assemblies.so"> - %(_BuildTargetAbis.Identity) - - - - @@ -182,11 +170,11 @@ - - - - - - - - - - diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs index 129a25463b9..275517689d2 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs @@ -421,7 +421,7 @@ void WriteType (GeneratorWriteContext context, Type type, object? value, out Llv Type elementType = type.GetArrayElementType (); ulong elementCount = GetAggregateValueElementCount (context, type, value, globalVariable); - WriteArrayType (context, elementType, elementCount, out typeInfo); + WriteArrayType (context, elementType, elementCount, globalVariable, out typeInfo); return; } @@ -437,6 +437,11 @@ void WriteType (GeneratorWriteContext context, Type type, object? value, out Llv } void WriteArrayType (GeneratorWriteContext context, Type elementType, ulong elementCount, out LlvmTypeInfo typeInfo) + { + WriteArrayType (context, elementType, elementCount, variable: null, out typeInfo); + } + + void WriteArrayType (GeneratorWriteContext context, Type elementType, ulong elementCount, LlvmIrGlobalVariable? variable, out LlvmTypeInfo typeInfo) { string irType; ulong size; @@ -453,6 +458,35 @@ void WriteArrayType (GeneratorWriteContext context, Type elementType, ulong elem } else { irType = GetIRType (elementType, out size, out isPointer); maxFieldAlignment = size; + + if (elementType.IsArray) { + if (variable == null) { + throw new InvalidOperationException ($"Internal error: array of arrays ({elementType}) requires variable to be defined"); + } + + // For the sake of simpler code, we currently assume that all the element arrays are of the same size, because that's the only scenario + // that we use at this time. + var value = variable.Value as ICollection; + if (value == null) { + throw new InvalidOperationException ($"Internal error: variable '{variable.Name}' of type '{variable.Type}' is required to have a value of type which implements the ICollection interface"); + } + + if (value.Count == 0) { + throw new InvalidOperationException ($"Internal error: variable '{variable.Name}' of type '{variable.Type}' is required to have a value which is a non-empty ICollection"); + } + + Array? firstItem = null; + foreach (object v in value) { + firstItem = (Array)v; + break; + } + + if (firstItem == null) { + throw new InvalidOperationException ($"Internal error: variable '{variable.Name}' of type '{variable.Type}' is required to have a value which is a non-empty ICollection with non-null elements"); + } + + irType = $"[{MonoAndroidHelper.CultureInvariantToString (firstItem.Length)} x {irType}]"; + } } typeInfo = new LlvmTypeInfo ( isPointer: isPointer, From 129044b34e38ff3b7271ec977c544f95c6e3a0e9 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 19 Sep 2023 22:17:27 +0200 Subject: [PATCH 47/71] A handful of improvements * Add hashes for assembly names without extension * Improve code formatting * Make sure we have no hash clashes * Link assembly data into libxamarin-app.so --- .../Utilities/AssemblyDSOGenerator.cs | 48 +++++++++++-- src/monodroid/jni/application_dso_stub.cc | 70 ++++++++++--------- src/monodroid/jni/xamarin-app.hh | 3 + 3 files changed, 83 insertions(+), 38 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs index 83e75d6ba0f..7c5e1cf9a1d 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs @@ -47,6 +47,9 @@ class AssemblyIndexEntryBase // Index into the `xa_assemblies` descriptor array public uint index; + + // whether hashed name had extension + public bool has_extension; } sealed class AssemblyIndexEntry32 : AssemblyIndexEntryBase @@ -428,6 +431,7 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) _ => throw new NotSupportedException ($"Architecture '{arch}' is not supported") }; + var usedHashes = new HashSet (); ulong inputOffset = 0; ulong uncompressedOffset = 0; ulong assemblyNameLength = 0; @@ -468,21 +472,47 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) if ((ulong)nameBytes.Length > assemblyNameLength) { assemblyNameLength = (ulong)nameBytes.Length; } + ulong nameHash = EnsureUniqueHash (GetXxHash (nameBytes, is64Bit), info.Name); + + string nameWithoutExtension = Path.GetFileNameWithoutExtension (info.Name); + byte[] nameWithoutExtensionBytes = StringToBytes (nameWithoutExtension); + ulong nameWithoutExtensionHash = EnsureUniqueHash (GetXxHash (nameWithoutExtensionBytes, is64Bit), nameWithoutExtension); + + uint assemblyIndex = (uint)archState.xa_assemblies.Count - 1; if (is64Bit) { var indexEntry = new AssemblyIndexEntry64 { Name = info.Name, NameBytes = nameBytes, - name_hash = GetXxHash (nameBytes, is64Bit), - index = (uint)archState.xa_assemblies.Count - 1, + name_hash = nameHash, + index = assemblyIndex, + has_extension = true, + }; + archState.xa_assembly_index64.Add (new StructureInstance (assemblyIndexEntry64StructureInfo, indexEntry)); + + indexEntry = new AssemblyIndexEntry64 { + Name = nameWithoutExtension, + NameBytes = nameWithoutExtensionBytes, + name_hash = nameWithoutExtensionHash, + index = assemblyIndex, + has_extension = false, }; archState.xa_assembly_index64.Add (new StructureInstance (assemblyIndexEntry64StructureInfo, indexEntry)); } else { var indexEntry = new AssemblyIndexEntry32 { Name = info.Name, NameBytes = nameBytes, - name_hash = (uint)GetXxHash (nameBytes, is64Bit), - index = (uint)archState.xa_assemblies.Count - 1, + name_hash = (uint)nameHash, + index = assemblyIndex, + }; + archState.xa_assembly_index32.Add (new StructureInstance (assemblyIndexEntry32StructureInfo, indexEntry)); + + indexEntry = new AssemblyIndexEntry32 { + Name = nameWithoutExtension, + NameBytes = nameWithoutExtensionBytes, + name_hash = (uint)nameWithoutExtensionHash, + index = assemblyIndex, + has_extension = false, }; archState.xa_assembly_index32.Add (new StructureInstance (assemblyIndexEntry32StructureInfo, indexEntry)); } @@ -514,6 +544,16 @@ ulong AddWithCheck (ulong lhs, ulong rhs, ulong maxValue, string errorMessage) return v; } + + ulong EnsureUniqueHash (ulong hash, string name) + { + if (usedHashes.Contains (hash)) { + throw new InvalidOperationException ($"Hash 0x{hash:x} for name '{name}' is not unique"); + } + + usedHashes.Add (hash); + return hash; + } } void ConstructEmptyModule () diff --git a/src/monodroid/jni/application_dso_stub.cc b/src/monodroid/jni/application_dso_stub.cc index 5f46ab026ae..9d474e65c39 100644 --- a/src/monodroid/jni/application_dso_stub.cc +++ b/src/monodroid/jni/application_dso_stub.cc @@ -203,44 +203,53 @@ const MarshalMethodName mm_method_names[] = { }; alignas(4096) const uint8_t xa_input_assembly_data[InputAssemblyDataSize] = { - 0x04, 0x00, 0x30, 0x26, 0xfe, 0xfb, 0x37, 0xf4, 0xb7, 0x19, 0x0f, 0xdc, 0xad, 0xb5, 0x3c, 0x82, - 0xf4, 0xd9, 0x64, 0xe3, 0x56, 0x95, 0x7a, 0xef, 0x0b, 0x79, 0xbe, 0x28, 0x2b, 0x2a, 0x31, 0x54, - 0xf1, 0x2a, 0x76, 0xf9, 0x84, 0x5a, 0x5e, 0x0c, 0x11, 0x30, 0xaf, 0x5d, 0xb1, 0xff, 0x0f, 0x48, + 0x04, 0x00, 0x30, 0x26, 0xfe, 0xfb, 0x37, 0xf4, 0xb7, 0x19, 0x0f, 0xdc, 0xad, 0xb5, 0x3c, 0x82, + 0xf4, 0xd9, 0x64, 0xe3, 0x56, 0x95, 0x7a, 0xef, 0x0b, 0x79, 0xbe, 0x28, 0x2b, 0x2a, 0x31, 0x54, + 0xf1, 0x2a, 0x76, 0xf9, 0x84, 0x5a, 0x5e, 0x0c, 0x11, 0x30, 0xaf, 0x5d, 0xb1, 0xff, 0x0f, 0x48, }; alignas(4096) uint8_t xa_uncompressed_assembly_data[UncompressedAssemblyDataSize] = { }; const AssemblyEntry xa_assemblies[AssemblyCount] = { - { - .input_data_offset = 0, - .input_data_size = 256, - .uncompressed_data_offset = 0, - .uncompressed_data_size = 0, - }, - - { - .input_data_offset = 256, - .input_data_size = 768, - .uncompressed_data_offset = 0, - .uncompressed_data_size = 2048, - }, + { + .input_data_offset = 0, + .input_data_size = 256, + .uncompressed_data_offset = 0, + .uncompressed_data_size = 0, + }, + + { + .input_data_offset = 256, + .input_data_size = 768, + .uncompressed_data_offset = 0, + .uncompressed_data_size = 2048, + }, }; const AssemblyIndexEntry xa_assembly_index[AssemblyCount] = { - { - .name_hash = 11111u, - .index = 0, - }, - - { - .name_hash = 22222u, - .index = 1, - }, + { + .name_hash = 11111u, + .index = 0, + .has_extension = true, + }, + + { + .name_hash = 22222u, + .index = 1, + .has_extension = true, + }, }; const char xa_assembly_names[AssemblyCount][AssemblyNameLength] = { - "Assembly1.dll", - "AnotherAssembly2.dll", + "Assembly1.dll", + "AnotherAssembly2.dll", +}; + +const AssembliesConfig xa_assemblies_config = { + .input_assembly_data_size = InputAssemblyDataSize, + .uncompressed_assembly_data_size = UncompressedAssemblyDataSize, + .assembly_name_length = AssemblyNameLength, + .assembly_count = AssemblyCount, }; void xamarin_app_init ([[maybe_unused]] JNIEnv *env, [[maybe_unused]] get_function_pointer_fn fn) noexcept @@ -326,10 +335,3 @@ const JniRemappingTypeReplacementEntry jni_remapping_type_replacements[] = { .replacement = "another/replacement/java/type", }, }; - -const AssembliesConfig xa_assemblies_config = { - .input_assembly_data_size = InputAssemblyDataSize, - .uncompressed_assembly_data_size = UncompressedAssemblyDataSize, - .assembly_name_length = AssemblyNameLength, - .assembly_count = AssemblyCount, -}; diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index fd129d42c2a..95d4099c9ea 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -133,6 +133,9 @@ struct AssemblyIndexEntry // Index into the `xa_assemblies` descriptor array uint32_t index; + + // whether hashed name had extension + bool has_extension; }; constexpr uint32_t InputAssemblyDataSize = 1024; From f2250e717f48f29a735f3330afa0e2e9be73c512 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 20 Sep 2023 23:01:40 +0200 Subject: [PATCH 48/71] Handle satellite assemblies --- .../GenerateAssemblyDsoNativeSourceFiles.cs | 47 ++++++++++++++++--- .../Tasks/GenerateJavaStubs.cs | 8 +++- .../Utilities/AssemblyDSOGenerator.cs | 12 ++++- .../Utilities/MonoAndroidHelper.cs | 19 ++++++++ 4 files changed, 77 insertions(+), 9 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAssemblyDsoNativeSourceFiles.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAssemblyDsoNativeSourceFiles.cs index 4395227c311..3093f1d098d 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAssemblyDsoNativeSourceFiles.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAssemblyDsoNativeSourceFiles.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using Microsoft.Android.Build.Tasks; using Microsoft.Build.Framework; @@ -38,6 +39,7 @@ public override bool RunTask () Log.LogDebugMessage ("Assembly compression DISABLED"); } + var satelliteAssemblies = new List (); ulong inputAssemblyDataSize = 0; ulong uncompressedAssemblyDataSize = 0; Log.LogDebugMessage ("Processing the input assemblies"); @@ -66,13 +68,24 @@ public override bool RunTask () uncompressedAssemblyDataSize += (ulong)fi.Length; Log.LogDebugMessage ($" will include from: {inputFile} (compressed? {compressed}; compressedSize == {compressedSize}"); - AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (assembly); - if (!dsoAssembliesInfo.TryGetValue (arch, out List? assemblyList)) { - assemblyList = new List (); - dsoAssembliesInfo.Add (arch, assemblyList); + AndroidTargetArch arch; + if (!MonoAndroidHelper.IsSatelliteAssembly (assembly)) { + arch = MonoAndroidHelper.GetTargetArch (assembly); + StoreAssembly (arch, assembly, inputFile, fi.Length, compressedSize); + continue; + } + + // Satellite assemblies don't have any ABI, so they need to be added to all supported architectures. + // We will do it after this loop, when all architectures are known. + satelliteAssemblies.Add (MakeAssemblyInfo (assembly, inputFile, fi.Length, compressedSize)); + } + + if (satelliteAssemblies.Count > 0) { + foreach (DSOAssemblyInfo info in satelliteAssemblies) { + foreach (AndroidTargetArch arch in dsoAssembliesInfo.Keys.ToList ()) { + AddAssemblyToList (arch, info); + } } - assemblyList.Add (new DSOAssemblyInfo (GetAssemblyName (assembly), inputFile, (uint)fi.Length, compressedSize)); - Log.LogDebugMessage ($" added to list with name: {assemblyList[assemblyList.Count - 1].Name}"); } Log.LogDebugMessage ($"Size of assembly data to stash: {inputAssemblyDataSize}"); @@ -103,11 +116,31 @@ public override bool RunTask () } return !Log.HasLoggedErrors; + + void StoreAssembly (AndroidTargetArch arch, ITaskItem assembly, string inputFile, long fileLength, uint compressedSize, DSOAssemblyInfo? info = null) + { + AddAssemblyToList (arch, MakeAssemblyInfo (assembly, inputFile, fileLength, compressedSize)); + } + + void AddAssemblyToList (AndroidTargetArch arch, DSOAssemblyInfo info) + { + if (!dsoAssembliesInfo.TryGetValue (arch, out List? assemblyList)) { + assemblyList = new List (); + dsoAssembliesInfo.Add (arch, assemblyList); + } + assemblyList.Add (info); + Log.LogDebugMessage ($" added to arch {arch} with name: {assemblyList[assemblyList.Count - 1].Name}"); + } + + DSOAssemblyInfo MakeAssemblyInfo (ITaskItem assembly, string inputFile, long fileLength, uint compressedSize) + { + return new DSOAssemblyInfo (GetAssemblyName (assembly), inputFile, (uint)fileLength, compressedSize); + } } string GetAssemblyName (ITaskItem assembly) { - if (!assembly.ItemSpec.EndsWith (".resources.dll", StringComparison.OrdinalIgnoreCase)) { + if (!MonoAndroidHelper.IsSatelliteAssembly (assembly)) { return Path.GetFileName (assembly.ItemSpec); } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 0e4c5ace83b..c25d4b3432f 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -313,6 +313,8 @@ T CollectInterestingAssemblies (T assemblies, Func (T assemblies, Func infos) } ulong nameHash = EnsureUniqueHash (GetXxHash (nameBytes, is64Bit), info.Name); - string nameWithoutExtension = Path.GetFileNameWithoutExtension (info.Name); + string nameWithoutExtension; + string? dirName = Path.GetDirectoryName (info.Name); + + if (String.IsNullOrEmpty (dirName)) { + nameWithoutExtension = Path.GetFileNameWithoutExtension (info.Name); + } else { + // Don't use Path.Combine because the `/` separator must remain as such, since it's not a "real" + // directory separator but a culture/name separator. Path.Combine would use `\` on Windows. + nameWithoutExtension = $"{dirName}/{Path.GetFileNameWithoutExtension (info.Name)}"; + } + byte[] nameWithoutExtensionBytes = StringToBytes (nameWithoutExtension); ulong nameWithoutExtensionHash = EnsureUniqueHash (GetXxHash (nameWithoutExtensionBytes, is64Bit), nameWithoutExtension); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs index 2bb74996a6e..9d0ca6a7d3a 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs @@ -645,5 +645,24 @@ public static string GetNativeLibsRootDirectoryPath (string androidBinUtilsDirec } public static string MakeNativeAssemblyFileName (string baseName, string abi) => $"{baseName}.{abi}.ll"; + + public static bool IsSatelliteAssembly (ITaskItem item) + { + return IsNotEmptyMetadataAndMaybeMatches (item, "AssetType", "resources") && IsNotEmptyMetadataAndMaybeMatches (item, "Culture"); + + bool IsNotEmptyMetadataAndMaybeMatches (ITaskItem item, string metadataName, string? expectedValue = null) + { + string? data = item.GetMetadata (metadataName); + if (String.IsNullOrEmpty (data)) { + return false; + } + + if (String.IsNullOrEmpty (expectedValue)) { + return true; + } + + return String.Compare (data, expectedValue, StringComparison.Ordinal) == 0; + } + } } } From e811441b024de3b496f18d4abb4402dff7466042 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 21 Sep 2023 22:45:09 +0200 Subject: [PATCH 49/71] New direction - to per-assembly DSOs Some assemblies will still be contained in libxamarin-app.so, but most of them will end up in their own shared libraries. Te reason is memory usage (so that we don't have to keep the compressed assembly data in memory indefinitely). The per-assembly shared libraries will not be `dlopen`-ed because that call is too expensive (half a millisecond on Pixel 6 Pro, 1.6ms on Nexus 5) while mmap is very cheap, so we will record, at build time, location of the individual assembly's data in its DSO and use that to read the content after mmapping it. This requires some reworking of the current code as the sequence in which the DSOs are generated needs to be done in three steps: * generate the individual assemblies DSO source code * compile the above and link the DSOs * generate code for inclusion in libxamarin-app.so This is because, before the last step, we need to know the location of data in the DSOs, and that data is available only when we have the .so library and can read it with ELFSharp. --- .../Microsoft.Android.Sdk.NativeCompilation.targets | 13 +++++++++++++ .../Tasks/GenerateAssemblyDsoNativeSourceFiles.cs | 5 ++++- .../Utilities/AssemblyDSOGenerator.cs | 12 +++++++++++- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets index e0d089286a3..2137026ccc2 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets @@ -116,8 +116,20 @@ + + + <_DSOFastPathAssembly Include="Java.Interop.dll" /> + <_DSOFastPathAssembly Include="Mono.Android.Runtime.dll" /> + <_DSOFastPathAssembly Include="Mono.Android.dll" /> + <_DSOFastPathAssembly Include="System.Private.CoreLib.dll" /> + <_DSOFastPathAssembly Include="System.Runtime" /> + <_DSOFastPathAssembly Include="$(AndroidApplication)" /> + + + @@ -126,6 +138,7 @@ CompressedAssembliesOutputDirectory="$(_CompressedAssembliesOutputDir)" SupportedAbis="@(_BuildTargetAbis)" Assemblies="@(_ResolvedUserAssemblies);@(_ResolvedFrameworkAssemblies);@(_AndroidResolvedSatellitePaths)" + FastPathAssemblyNames="@(_DSOFastPathAssembly)" EnableCompression="$(AndroidEnableAssemblyCompression)" /> diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAssemblyDsoNativeSourceFiles.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAssemblyDsoNativeSourceFiles.cs index 3093f1d098d..814cc7e9e35 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAssemblyDsoNativeSourceFiles.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAssemblyDsoNativeSourceFiles.cs @@ -28,6 +28,9 @@ public class GenerateAssemblyDsoNativeSourceFiles : AndroidTask [Required] public bool EnableCompression { get; set; } + [Required] + public string[] FastPathAssemblyNames { get; set; } + public override bool RunTask () { Dictionary> dsoAssembliesInfo = new (); @@ -94,7 +97,7 @@ public override bool RunTask () Log.LogDebugMessage ($" {kvp.Key}: {kvp.Value.Count} assemblies"); } - var generator = new AssemblyDSOGenerator (dsoAssembliesInfo, inputAssemblyDataSize, uncompressedAssemblyDataSize); + var generator = new AssemblyDSOGenerator (FastPathAssemblyNames, dsoAssembliesInfo, inputAssemblyDataSize, uncompressedAssemblyDataSize); LLVMIR.LlvmIrModule module = generator.Construct (); foreach (string abi in SupportedAbis) { diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs index 648cf44ce05..455a6b18f4d 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs @@ -219,6 +219,7 @@ byte[] EnsureValidAssemblyData (AssemblyEntry entry) readonly Dictionary> assemblies; readonly Dictionary assemblyArchStates; + readonly HashSet? fastPathAssemblies; readonly uint inputAssemblyDataSize; readonly uint uncompressedAssemblyDataSize; StructureInfo? assemblyEntryStructureInfo; @@ -226,13 +227,22 @@ byte[] EnsureValidAssemblyData (AssemblyEntry entry) StructureInfo? assemblyIndexEntry64StructureInfo; StructureInfo? assembliesConfigStructureInfo; - public AssemblyDSOGenerator (Dictionary> dsoAssemblies, ulong inputAssemblyDataSize, ulong uncompressedAssemblyDataSize) + public AssemblyDSOGenerator (ICollection fastPathAssemblyNames, Dictionary> dsoAssemblies, ulong inputAssemblyDataSize, ulong uncompressedAssemblyDataSize) { this.inputAssemblyDataSize = EnsureValidSize (inputAssemblyDataSize, nameof (inputAssemblyDataSize)); this.uncompressedAssemblyDataSize = EnsureValidSize (uncompressedAssemblyDataSize, nameof (uncompressedAssemblyDataSize)); assemblies = dsoAssemblies; assemblyArchStates = new Dictionary (); + if (fastPathAssemblyNames.Count == 0) { + return; + } + + fastPathAssemblies = new HashSet (StringComparer.OrdinalIgnoreCase); + foreach (string asmName in fastPathAssemblyNames) { + fastPathAssemblies.Add (asmName); + } + uint EnsureValidSize (ulong v, string name) { if (v > UInt32.MaxValue) { From eac51f2ddc0665b8960f9155c34e50f5113dd8d5 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 22 Sep 2023 20:01:07 +0200 Subject: [PATCH 50/71] Progress towards per-assembly DSOs --- ...soft.Android.Sdk.NativeCompilation.targets | 109 ++++++++++++++---- .../GenerateAssemblyDsoNativeSourceFiles.cs | 3 + .../PrepareStandaloneAssemblyDSOItems.cs | 59 ++++++++++ 3 files changed, 147 insertions(+), 24 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Tasks/PrepareStandaloneAssemblyDSOItems.cs diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets index 2137026ccc2..7e8d6763d61 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets @@ -6,6 +6,7 @@ + <_AndroidUseAssemblySharedLibraries Condition=" '$(EmbedAssembliesIntoApk)' != 'true' or '$(AndroidIncludeDebugSymbols)' == 'true' ">false @@ -57,16 +58,42 @@ - + + + + + <_DSOFastPathAssembly Include="Java.Interop.dll" /> + <_DSOFastPathAssembly Include="Mono.Android.Runtime.dll" /> + <_DSOFastPathAssembly Include="Mono.Android.dll" /> + <_DSOFastPathAssembly Include="System.Private.CoreLib.dll" /> + <_DSOFastPathAssembly Include="System.Runtime" /> + <_DSOFastPathAssembly Include="$(AssemblyName).dll" /> + + + - + + + + + + @@ -102,7 +129,7 @@ %(_AndroidRemapAssemblySource.abi) - <_NativeAssemblyTarget Include="@(_AssemblyDSOSource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> + <_NativeAssemblyTarget Include="@(_AssemblyDSOSourceApplication->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> %(_AssemblyDSOSource.abi) @@ -116,23 +143,29 @@ - - - <_DSOFastPathAssembly Include="Java.Interop.dll" /> - <_DSOFastPathAssembly Include="Mono.Android.Runtime.dll" /> - <_DSOFastPathAssembly Include="Mono.Android.dll" /> - <_DSOFastPathAssembly Include="System.Private.CoreLib.dll" /> - <_DSOFastPathAssembly Include="System.Runtime" /> - <_DSOFastPathAssembly Include="$(AndroidApplication)" /> - + + + + - - + Outputs="(@_AssemblyDSOSourceStandalone)"> + StandaloneOnly="true" /> - + + + - + + + + + + + + <_CreateApplicationSharedLibrariesDependsOn> + _PrepareAssemblyDSOSources; + _CreateStandaloneAssemblyDSOs; + _CompileApplicationNativeAssemblySources; + _PrepareApplicationSharedLibraryItems + + + > dsoAssembliesInfo = new (); diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareStandaloneAssemblyDSOItems.cs b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareStandaloneAssemblyDSOItems.cs new file mode 100644 index 00000000000..864f01beb5c --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareStandaloneAssemblyDSOItems.cs @@ -0,0 +1,59 @@ +using System; +using System.IO; +using System.Collections.Generic; + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Microsoft.Android.Build.Tasks; + +namespace Xamarin.Android.Tasks; + +public class PrepareStandaloneAssemblyDSOItems : AndroidTask +{ + public override string TaskPrefix => "PSADI"; + + [Required] + public ITaskItem[] Assemblies { get; set; } + + [Required] + public string[] SupportedAbis { get; set; } + + [Required] + public string NativeSourcesDir { get; set; } + + [Required] + public ITaskItem[] FastPathAssemblies { get; set; } + + [Output] + public ITaskItem[] AssemblySources { get; set; } + + public override bool RunTask () + { + var fastAssemblies = new HashSet (StringComparer.OrdinalIgnoreCase); + + foreach (ITaskItem asm in FastPathAssemblies) { + fastAssemblies.Add (Path.GetFileName (asm.ItemSpec)); + } + + var seenAssemblyNames = new HashSet (StringComparer.OrdinalIgnoreCase); + var sources = new List (); + foreach (ITaskItem asm in Assemblies) { + string asmName = Path.GetFileName (asm.ItemSpec); + if (fastAssemblies.Contains (asmName) || seenAssemblyNames.Contains (asmName)) { + continue; + } + seenAssemblyNames.Add (asmName); + + string baseName = Path.GetFileNameWithoutExtension (asmName); + foreach (string abi in SupportedAbis) { + var item = new TaskItem (Path.Combine (NativeSourcesDir, MonoAndroidHelper.MakeNativeAssemblyFileName (baseName, abi))); + item.SetMetadata ("abi", abi); + + sources.Add (item); + } + } + + AssemblySources = sources.ToArray (); + return !Log.HasLoggedErrors; + } +} From aeaab2bf943ae247f6aabc9dded55eab5effa33a Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 25 Sep 2023 22:40:57 +0200 Subject: [PATCH 51/71] More steps towards standalone DSOs --- ...soft.Android.Sdk.NativeCompilation.targets | 62 +++++----- .../BuildAndLinkStandaloneAssemblyDSOs.cs | 75 ++++++++++++ ...enerateAppAssemblyDSONativeSourceFiles.cs} | 89 +++++--------- .../PrepareAssemblyStandaloneDSOAbiItems.cs | 110 ++++++++++++++++++ .../PrepareStandaloneAssemblyDSOItems.cs | 59 ---------- .../Utilities/AssemblyDSOGenerator.cs | 31 ++++- 6 files changed, 264 insertions(+), 162 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Tasks/BuildAndLinkStandaloneAssemblyDSOs.cs rename src/Xamarin.Android.Build.Tasks/Tasks/{GenerateAssemblyDsoNativeSourceFiles.cs => GenerateAppAssemblyDSONativeSourceFiles.cs} (62%) create mode 100644 src/Xamarin.Android.Build.Tasks/Tasks/PrepareAssemblyStandaloneDSOAbiItems.cs delete mode 100644 src/Xamarin.Android.Build.Tasks/Tasks/PrepareStandaloneAssemblyDSOItems.cs diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets index 7e8d6763d61..78a9909e080 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets @@ -2,11 +2,12 @@ - + - + + <_AndroidUseAssemblySharedLibraries Condition=" '$(EmbedAssembliesIntoApk)' != 'true' or '$(AndroidIncludeDebugSymbols)' == 'true' ">false @@ -84,16 +85,16 @@ - - + - - + FastPathAssemblyNames="@(_DSOFastPathAssembly)" + SharedLibraryOutputDir="$(_AndroidApplicationSharedLibraryPath)"> + + @@ -152,28 +153,13 @@ * Pass items produced by the _CreateStandaloneAssemblyDSOs target and use metadata when generating the source to be included in libxamarin-app.so --> - - - - - + EnableCompression="$(AndroidEnableAssemblyCompression)"/> - - - + Condition=" '$(_AndroidUseAssemblySharedLibraries)' == 'true' " + Inputs="@(_ResolvedUserAssemblies);@(_ResolvedFrameworkAssemblies);@(_AndroidResolvedSatellitePaths)" + Outputs="@(_StandaloneAssemblyDSOTarget)"> + + + @@ -226,7 +228,9 @@ CompressedAssembliesOutputDirectory="$(_CompressedAssembliesOutputDir)" TargetSharedLibraries="@(_StandaloneAssemblyDSOTarget)" EnableCompression="$(AndroidEnableAssemblyCompression)" - SharedLibraryOutputDir="$(_AndroidApplicationSharedLibraryPath)%(_BuildTargetAbis.Identity)"> + SharedLibraryOutputDir="$(_AndroidApplicationSharedLibraryPath)%(_BuildTargetAbis.Identity)" + KeepGeneratedSources="$(_AndroidKeepStandaloneAssemblySources)" + AndroidBinUtilsDirectory="$(AndroidBinUtilsDirectory)"> + the main activity's OnCreate. + + Names must include the .dll extension. + --> <_DSOFastPathAssembly Include="Java.Interop.dll" /> <_DSOFastPathAssembly Include="Mono.Android.Runtime.dll" /> <_DSOFastPathAssembly Include="Mono.Android.dll" /> <_DSOFastPathAssembly Include="System.Private.CoreLib.dll" /> - <_DSOFastPathAssembly Include="System.Runtime" /> + <_DSOFastPathAssembly Include="System.Runtime.dll" /> <_DSOFastPathAssembly Include="$(AssemblyName).dll" /> @@ -133,7 +136,7 @@ <_NativeAssemblyTarget Include="@(_AssemblyDSOSourceApplication->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> - %(_AssemblyDSOSource.abi) + %(_AssemblyDSOSourceApplication.abi) @@ -161,6 +164,7 @@ SupportedAbis="@(_BuildTargetAbis)" Assemblies="@(_ResolvedUserAssemblies);@(_ResolvedFrameworkAssemblies);@(_AndroidResolvedSatellitePaths)" FastPathAssemblyNames="@(_DSOFastPathAssembly)" + StandaloneAssemblyDSOs="@(_StandaloneAssemblyDSO)" EnableCompression="$(AndroidEnableAssemblyCompression)"/> @@ -228,7 +232,6 @@ CompressedAssembliesOutputDirectory="$(_CompressedAssembliesOutputDir)" TargetSharedLibraries="@(_StandaloneAssemblyDSOTarget)" EnableCompression="$(AndroidEnableAssemblyCompression)" - SharedLibraryOutputDir="$(_AndroidApplicationSharedLibraryPath)%(_BuildTargetAbis.Identity)" KeepGeneratedSources="$(_AndroidKeepStandaloneAssemblySources)" AndroidBinUtilsDirectory="$(AndroidBinUtilsDirectory)" ParallelBuildsNumber="$(_AndroidNativeParallelBuildsNumber)"> diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildAndLinkStandaloneAssemblyDSOs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildAndLinkStandaloneAssemblyDSOs.cs index 97fbb3d71b5..83d82e06856 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildAndLinkStandaloneAssemblyDSOs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildAndLinkStandaloneAssemblyDSOs.cs @@ -58,15 +58,23 @@ public LocalDSOAssemblyInfo (TargetDSO targetDSO, ITaskItem sharedLibraryItem, s } } - public override string TaskPrefix => "BALSAD"; + public static class DSOMetadata + { + public const string Compressed = "Compressed"; + public const string DataSize = "DataSize"; + public const string DataSymbolOffset = "DataSymbolOffset"; + public const string InputAssemblyPath = "InputAssemblyPath"; + public const string OriginalAssemblyPath = "OriginalAssemblyPath"; + public const string SatelliteAssemblyCulture = "SatelliteAssemblyCulture"; + public const string UncompressedDataSize = "UncompressedDataSize"; + } const uint DefaultParallelBuilds = 8; - [Required] - public ITaskItem[] TargetSharedLibraries { get; set; } + public override string TaskPrefix => "BALSAD"; [Required] - public string SharedLibraryOutputDir { get; set; } + public ITaskItem[] TargetSharedLibraries { get; set; } [Required] public string AndroidBinUtilsDirectory { get; set; } @@ -92,14 +100,14 @@ protected override void Generate () DSOAssemblyInfo dsoInfo = AddAssembly (dso, dsoItem, assemblies); dsoItem.SetMetadata ("Abi", dso.Abi); - dsoItem.SetMetadata ("DataSize", MonoAndroidHelper.CultureInvariantToString (dsoInfo.CompressedDataSize == 0 ? dsoInfo.DataSize : dsoInfo.CompressedDataSize)); - dsoItem.SetMetadata ("UncompressedDataSize", MonoAndroidHelper.CultureInvariantToString (dsoInfo.DataSize)); - dsoItem.SetMetadata ("Compressed", dsoInfo.CompressedDataSize == 0 ? "false" : "true"); - dsoItem.SetMetadata ("OriginalAssemblyPath", dso.OriginalAssemblyPath); - dsoItem.SetMetadata ("InputAssemblyPath", dsoInfo.InputFile); + dsoItem.SetMetadata (DSOMetadata.DataSize, MonoAndroidHelper.CultureInvariantToString (dsoInfo.CompressedDataSize == 0 ? dsoInfo.DataSize : dsoInfo.CompressedDataSize)); + dsoItem.SetMetadata (DSOMetadata.UncompressedDataSize, MonoAndroidHelper.CultureInvariantToString (dsoInfo.DataSize)); + dsoItem.SetMetadata (DSOMetadata.Compressed, dsoInfo.CompressedDataSize == 0 ? "false" : "true"); + dsoItem.SetMetadata (DSOMetadata.OriginalAssemblyPath, dso.OriginalAssemblyPath); + dsoItem.SetMetadata (DSOMetadata.InputAssemblyPath, dsoInfo.InputFile); if (!String.IsNullOrEmpty (dso.Culture)) { - dsoItem.SetMetadata ("SatelliteAssemblyCulture", dso.Culture); + dsoItem.SetMetadata (DSOMetadata.SatelliteAssemblyCulture, dso.Culture); } sharedLibraries.Add (dsoItem); @@ -225,7 +233,7 @@ void DoCompileAndLink (NativeCompilationHelper.AssemblerConfig config, Dictionar return; } Log.LogDebugMessage ($"Shared library '{linkerConfig.OutputFile}' has symbol '{AssemblyDSOGenerator.XAInputAssemblyDataVarName}' at offset {dsoInfo.XAInputAssemblyDataOffset}"); - dsoInfo.SharedLibraryItem.SetMetadata ("DataSymbolOffset", MonoAndroidHelper.CultureInvariantToString (dsoInfo.XAInputAssemblyDataOffset)); + dsoInfo.SharedLibraryItem.SetMetadata (DSOMetadata.DataSymbolOffset, MonoAndroidHelper.CultureInvariantToString (dsoInfo.XAInputAssemblyDataOffset)); ulong expectedSize; if (dsoInfo.CompressedDataSize == 0) { diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAppAssemblyDSONativeSourceFiles.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAppAssemblyDSONativeSourceFiles.cs index 8768c7679f8..756ce9350e7 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAppAssemblyDSONativeSourceFiles.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAppAssemblyDSONativeSourceFiles.cs @@ -22,23 +22,37 @@ public class GenerateAppAssemblyDSONativeSourceFiles : AssemblyNativeSourceGener [Required] public string[] FastPathAssemblyNames { get; set; } + [Required] + public ITaskItem[] StandaloneAssemblyDSOs { get; set; } + protected override void Generate () { Dictionary> dsoAssembliesInfo = new (); var satelliteAssemblies = new List (); + + // The figure here includes only the "fast path" assemblies, that is those which end up in libxamarin-app.so ulong inputAssemblyDataSize = 0; + + // This variable, however, keeps the total size of all the assemblies, both from libxamarin-app.so and from + // their respective .so shared libraries. ulong uncompressedAssemblyDataSize = 0; - Log.LogDebugMessage ("Processing the input assemblies"); + AndroidTargetArch arch; + var fastPathAssemblies = new HashSet (FastPathAssemblyNames, StringComparer.OrdinalIgnoreCase); + + Log.LogDebugMessage ("Processing input assemblies"); foreach (ITaskItem assembly in Assemblies) { + if (!fastPathAssemblies.Contains (Path.GetFileName (assembly.ItemSpec))) { + continue; + } + CompressionResult cres = Compress (assembly); string inputFile = cres.OutputFile; - Log.LogDebugMessage ($" Input: {assembly.ItemSpec}"); + Log.LogDebugMessage ($" Fast path input: {assembly.ItemSpec}"); inputAssemblyDataSize += cres.Compressed ? (ulong)cres.InputFileInfo.Length : cres.CompressedSize; uncompressedAssemblyDataSize += (ulong)cres.InputFileInfo.Length; - AndroidTargetArch arch; if (!MonoAndroidHelper.IsSatelliteAssembly (assembly)) { arch = MonoAndroidHelper.GetTargetArch (assembly); StoreAssembly (arch, assembly, inputFile, cres.InputFileInfo.Length, cres.CompressedSize); @@ -50,10 +64,19 @@ protected override void Generate () satelliteAssemblies.Add (MakeAssemblyInfo (assembly, inputFile, cres.InputFileInfo.Length, cres.CompressedSize)); } + foreach (ITaskItem dsoItem in StandaloneAssemblyDSOs) { + arch = MonoAndroidHelper.GetTargetArch (dsoItem); + DSOAssemblyInfo info = MakeStandaloneAssemblyInfo (dsoItem); + + Log.LogDebugMessage ($" Standalone input: {info.Name}"); + uncompressedAssemblyDataSize += info.DataSize; + AddAssemblyToList (arch, info); + } + if (satelliteAssemblies.Count > 0) { foreach (DSOAssemblyInfo info in satelliteAssemblies) { - foreach (AndroidTargetArch arch in dsoAssembliesInfo.Keys.ToList ()) { - AddAssemblyToList (arch, info); + foreach (AndroidTargetArch dsoArch in dsoAssembliesInfo.Keys.ToList ()) { + AddAssemblyToList (dsoArch, info); } } } @@ -82,4 +105,52 @@ void AddAssemblyToList (AndroidTargetArch arch, DSOAssemblyInfo info) Log.LogDebugMessage ($" added to arch {arch} with name: {assemblyList[assemblyList.Count - 1].Name}"); } } + + DSOAssemblyInfo MakeStandaloneAssemblyInfo (ITaskItem dsoItem) + { + string name = Path.GetFileName (GetRequiredMetadata (BuildAndLinkStandaloneAssemblyDSOs.DSOMetadata.OriginalAssemblyPath)); + string? cultureName = dsoItem.GetMetadata (BuildAndLinkStandaloneAssemblyDSOs.DSOMetadata.SatelliteAssemblyCulture); + + if (!String.IsNullOrEmpty (cultureName)) { + name = $"{cultureName}/{name}"; + } + + string inputFile = GetRequiredMetadata (BuildAndLinkStandaloneAssemblyDSOs.DSOMetadata.InputAssemblyPath); + string compressed = GetRequiredMetadata (BuildAndLinkStandaloneAssemblyDSOs.DSOMetadata.Compressed); + if (!Boolean.TryParse (compressed, out bool isCompressed)) { + throw new InvalidOperationException ($"Internal error: unable to parse '{compressed}' as a boolean value, from the '{BuildAndLinkStandaloneAssemblyDSOs.DSOMetadata.Compressed}' metadata of item {dsoItem}"); + } + + uint dataSize = GetUintFromRequiredMetadata (BuildAndLinkStandaloneAssemblyDSOs.DSOMetadata.DataSize); + uint compressedDataSize; + if (!isCompressed) { + compressedDataSize = 0; + } else { + compressedDataSize = dataSize; + dataSize = GetUintFromRequiredMetadata (BuildAndLinkStandaloneAssemblyDSOs.DSOMetadata.UncompressedDataSize); + + } + + return new DSOAssemblyInfo (name, inputFile, dataSize, compressedDataSize, isStandalone: true); + + string GetRequiredMetadata (string name) + { + string ret = dsoItem.GetMetadata (name); + if (String.IsNullOrEmpty (ret)) { + throw new InvalidOperationException ($"Internal error: item {dsoItem} doesn't contain required metadata item '{name}' or its value is an empty string"); + } + + return ret; + } + + uint GetUintFromRequiredMetadata (string name) + { + string metadata = GetRequiredMetadata (name); + if (!UInt32.TryParse (metadata, out uint value)) { + throw new InvalidOperationException ($"Internal error: unable to parse '{metadata}' as a UInt32 value, from the '{name}' metadata of item {dsoItem}"); + } + + return value; + } + } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.Classes.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.Classes.cs index 343c9563423..fea6c56a7e6 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.Classes.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.Classes.cs @@ -17,6 +17,13 @@ class StandaloneAssemblyEntry [NativeAssembler (Ignore = true)] public string InputFilePath; + + // If `true`, we have an instance of a standalone assembly being processed when + // generating libxamarin-app.so sources. This is necessary because we still need + // all the assembly information (name, size etc) to generated indexes etc. However, + // in this case assembly data will not be needed as it's already in its own DSO + [NativeAssembler (Ignore = true)] + public bool IsStandalone; } // Must be identical to AssemblyEntry in src/monodroid/jni/xamarin-app.hh @@ -53,6 +60,9 @@ class AssemblyIndexEntryBase // whether hashed name had extension public bool has_extension; + + // whether assembly data lives in a separate DSO + public bool is_standalone; } sealed class AssemblyIndexEntry32 : AssemblyIndexEntryBase @@ -130,12 +140,14 @@ protected byte[] EnsureValidAssemblyData (StandaloneAssemblyEntry? entry) throw new ArgumentNullException (nameof (entry)); } - if (entry.AssemblyData == null) { - throw new InvalidOperationException ("Internal error: assembly data must be present"); - } + if (!entry.IsStandalone) { + if (entry.AssemblyData == null) { + throw new InvalidOperationException ("Internal error: assembly data must be present"); + } - if (entry.AssemblyData.Length == 0) { - throw new InvalidOperationException ("Internal error: assembly data must not be empty"); + if (entry.AssemblyData.Length == 0) { + throw new InvalidOperationException ("Internal error: assembly data must not be empty"); + } } return entry.AssemblyData; @@ -185,7 +197,7 @@ public AssemblyInputDataArrayProvider (Type arrayElementType, Dictionary si in archState.xa_assemblies) { var entry = (AssemblyEntry)si.Obj; + if (entry.IsStandalone) { + continue; + } + byte[] data = EnsureValidAssemblyData (entry); totalSize += (ulong)data.Length; } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs index 58dc77124ac..e7c81434fd6 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs @@ -11,6 +11,7 @@ namespace Xamarin.Android.Tasks; partial class AssemblyDSOGenerator : LlvmIrComposer { const string XAAssembliesConfigVarName = "xa_assemblies_config"; + const string XAAssembliesVarName = "xa_assemblies"; const string XAAssemblyIndexVarName = "xa_assembly_index"; const string XAAssemblyNamesVarName = "xa_assembly_names"; public const string XAInputAssemblyDataVarName = "xa_input_assembly_data"; @@ -91,6 +92,8 @@ void ConstructStandalone (LlvmIrModule module) module.Add (xa_input_assembly_data); } + // TODO: need to store information about an assembly being standalone. Also, we need to store the shared library name somewhere (probably a separate array, at the same + // index as the xa_assemblies_index indicates void ConstructFastPath (LlvmIrModule module) { MapStructures (module); @@ -127,6 +130,12 @@ void ConstructFastPath (LlvmIrModule module) }; module.Add (xa_assemblies_config); + var xa_assemblies = new LlvmIrGlobalVariable (typeof(List>), XAAssembliesVarName) { + BeforeWriteCallback = AssembliesBeforeWrite, + Options = LlvmIrVariableOptions.GlobalConstant, + }; + module.Add (xa_assemblies); + var xa_assembly_index = new LlvmIrGlobalVariable (typeof(List>), XAAssemblyIndexVarName) { BeforeWriteCallback = AssemblyIndexBeforeWrite, GetArrayItemCommentCallback = AssemblyIndexItemComment, @@ -159,6 +168,12 @@ void ConstructFastPath (LlvmIrModule module) module.Add (xa_input_assembly_data); } + void AssembliesBeforeWrite (LlvmIrVariable variable, LlvmIrModuleTarget target, object? state) + { + ArchState archState = GetArchState (target); + variable.Value = archState.xa_assemblies; + } + string AssemblyIndexItemComment (LlvmIrVariable variable, LlvmIrModuleTarget target, ulong index, object? itemValue, object? state) { var value32 = itemValue as StructureInstance; @@ -301,6 +316,7 @@ void AddStandaloneAssemblyData (AndroidTargetArch arch, DSOAssemblyInfo info) var entry = new StandaloneAssemblyEntry { AssemblyData = new byte[inputSize], InputFilePath = info.InputFile, + IsStandalone = true, }; (ArchState archState, bool _) = GetArchState (arch, 1, entry); @@ -319,6 +335,7 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) ulong uncompressedOffset = 0; ulong assemblyNameLength = 0; foreach (DSOAssemblyInfo info in infos) { + bool isStandalone = info.IsStandalone; uint inputSize = GetInputSize (info); // We need to read each file into a separate array, as it is (theoretically) possible that all the assemblies data will exceed 2GB, @@ -330,7 +347,8 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) var entry = new AssemblyEntry { // We can't use the byte pool here, even though it would be more efficient, because the generator expects an ICollection, // which it then iterates on, and the rented arrays can (and frequently will) be bigger than the requested size. - AssemblyData = new byte[inputSize], + AssemblyData = isStandalone ? null : new byte[inputSize], + IsStandalone = isStandalone, InputFilePath = info.InputFile, input_data_offset = (uint)inputOffset, input_data_size = inputSize, @@ -338,10 +356,12 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) uncompressed_data_offset = (uint)uncompressedOffset, }; inputOffset = AddWithCheck (inputOffset, inputSize, UInt32.MaxValue, "Input data too long"); - ReadAssemblyData (info, entry); + if (!isStandalone) { + ReadAssemblyData (info, entry); + } // This is way, way more than Google Play Store supports now, but we won't limit ourselves more than we have to - uncompressedOffset = AddWithCheck (uncompressedOffset, entry.uncompressed_data_offset, UInt32.MaxValue, "Compressed data too long"); + uncompressedOffset = AddWithCheck (uncompressedOffset, entry.uncompressed_data_size, UInt32.MaxValue, "Assembly data too long"); archState.xa_assemblies.Add (new StructureInstance (assemblyEntryStructureInfo, entry)); byte[] nameBytes = StringToBytes (info.Name); @@ -373,6 +393,7 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) name_hash = nameHash, index = assemblyIndex, has_extension = true, + is_standalone = isStandalone, }; archState.xa_assembly_index64.Add (new StructureInstance (assemblyIndexEntry64StructureInfo, indexEntry)); @@ -382,6 +403,7 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) name_hash = nameWithoutExtensionHash, index = assemblyIndex, has_extension = false, + is_standalone = isStandalone, }; archState.xa_assembly_index64.Add (new StructureInstance (assemblyIndexEntry64StructureInfo, indexEntry)); } else { @@ -390,6 +412,7 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) NameBytes = nameBytes, name_hash = (uint)nameHash, index = assemblyIndex, + is_standalone = isStandalone, }; archState.xa_assembly_index32.Add (new StructureInstance (assemblyIndexEntry32StructureInfo, indexEntry)); @@ -399,6 +422,7 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) name_hash = (uint)nameWithoutExtensionHash, index = assemblyIndex, has_extension = false, + is_standalone = isStandalone, }; archState.xa_assembly_index32.Add (new StructureInstance (assemblyIndexEntry32StructureInfo, indexEntry)); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/DSOAssemblyInfo.cs b/src/Xamarin.Android.Build.Tasks/Utilities/DSOAssemblyInfo.cs index 76ebe0ccab1..de1acb4df24 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/DSOAssemblyInfo.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/DSOAssemblyInfo.cs @@ -12,25 +12,33 @@ class DSOAssemblyInfo /// public uint CompressedDataSize { get; } + /// + /// The file data comes from, either the original assembly or its compressed copy + /// public string InputFile { get; } /// - /// Name of the assembly, including culture prefix if it's a satellite assembly. Must include the - /// extension. + /// Name of the assembly, including culture prefix if it's a satellite assembly. Must include the extension. /// public string Name { get; } + /// + /// Indicates whether assembly data is stored in a standalone shared library. + /// + public bool IsStandalone { get; } + /// /// is the original assembly name, including culture prefix (e.g. `en_US/`) if it is a /// satellite assembly. should be the full path to the input file. /// gives the original file size, while specifies /// data size after compression, or `0` if file isn't compressed. /// - public DSOAssemblyInfo (string name, string inputFile, uint dataSize, uint compressedDataSize) + public DSOAssemblyInfo (string name, string inputFile, uint dataSize, uint compressedDataSize, bool isStandalone = false) { Name = name; InputFile = inputFile; DataSize = dataSize; CompressedDataSize = compressedDataSize; + IsStandalone = isStandalone; } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs index 275517689d2..c3102f3a5c3 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs @@ -803,43 +803,45 @@ uint GetArrayStride (LlvmIrVariable variable) return 1; } - void WriteArrayEntries (GeneratorWriteContext context, LlvmIrVariable variable, ICollection entries, Type elementType, uint stride, bool writeIndices, bool terminateWithComma = false) + void WriteArrayEntries (GeneratorWriteContext context, LlvmIrVariable variable, ICollection? entries, Type elementType, uint stride, bool writeIndices, bool terminateWithComma = false) { bool first = true; bool ignoreComments = stride > 1; string? prevItemComment = null; ulong counter = 0; - foreach (object entry in entries) { - if (!first) { - context.Output.Write (','); - if (stride == 1 || counter % stride == 0) { - WritePrevItemCommentOrNewline (); - context.Output.Write (context.CurrentIndent); + if (entries != null) { + foreach (object entry in entries) { + if (!first) { + context.Output.Write (','); + if (stride == 1 || counter % stride == 0) { + WritePrevItemCommentOrNewline (); + context.Output.Write (context.CurrentIndent); + } else { + context.Output.Write (' '); + } } else { - context.Output.Write (' '); + context.Output.Write (context.CurrentIndent); + first = false; } - } else { - context.Output.Write (context.CurrentIndent); - first = false; - } - if (!ignoreComments) { - prevItemComment = null; - if (variable.GetArrayItemCommentCallback != null) { - prevItemComment = variable.GetArrayItemCommentCallback (variable, target, counter, entry, variable.GetArrayItemCommentCallbackCallerState); - } + if (!ignoreComments) { + prevItemComment = null; + if (variable.GetArrayItemCommentCallback != null) { + prevItemComment = variable.GetArrayItemCommentCallback (variable, target, counter, entry, variable.GetArrayItemCommentCallbackCallerState); + } - if (writeIndices && String.IsNullOrEmpty (prevItemComment)) { - prevItemComment = $" {counter}"; + if (writeIndices && String.IsNullOrEmpty (prevItemComment)) { + prevItemComment = $" {counter}"; + } } - } - counter++; - WriteType (context, elementType, entry, out _); + counter++; + WriteType (context, elementType, entry, out _); - context.Output.Write (' '); - WriteValue (context, elementType, entry); + context.Output.Write (' '); + WriteValue (context, elementType, entry); + } } if (terminateWithComma) { @@ -872,29 +874,36 @@ void WriteStreamedArrayValue (GeneratorWriteContext context, LlvmIrGlobalVariabl WriteArrayValueStart (context); while (true) { - (LlvmIrStreamedArrayDataProviderState state, ICollection data) = dataProvider.GetData (context.Target); - if (data.Count == 0) { - throw new InvalidOperationException ("Data must be provided for streamed arrays"); + (LlvmIrStreamedArrayDataProviderState state, ICollection? data) = dataProvider.GetData (context.Target); + if (state == LlvmIrStreamedArrayDataProviderState.NextSectionNoData) { + continue; } - dataSizeSoFar += (ulong)data.Count; - if (dataSizeSoFar > totalDataSize) { - throw new InvalidOperationException ($"Data provider {dataProvider} is trying to write more data than declared"); - } + bool mustHaveData = state != LlvmIrStreamedArrayDataProviderState.LastSectionNoData; + if (mustHaveData) { + if (data.Count == 0) { + throw new InvalidOperationException ("Data must be provided for streamed arrays"); + } - if (first) { - first = false; - } else { - context.Output.WriteLine (); - } - string comment = dataProvider.GetSectionStartComment (context.Target); + dataSizeSoFar += (ulong)data.Count; + if (dataSizeSoFar > totalDataSize) { + throw new InvalidOperationException ($"Data provider {dataProvider} is trying to write more data than declared"); + } - if (comment.Length > 0) { - context.Output.Write (context.CurrentIndent); - WriteCommentLine (context, comment); + if (first) { + first = false; + } else { + context.Output.WriteLine (); + } + string comment = dataProvider.GetSectionStartComment (context.Target); + + if (comment.Length > 0) { + context.Output.Write (context.CurrentIndent); + WriteCommentLine (context, comment); + } } - bool lastSection = state == LlvmIrStreamedArrayDataProviderState.LastSection; + bool lastSection = state == LlvmIrStreamedArrayDataProviderState.LastSection || state == LlvmIrStreamedArrayDataProviderState.LastSectionNoData; WriteArrayEntries ( context, variable, diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs index 5bb66450a68..49524f0bf6c 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs @@ -160,6 +160,8 @@ enum LlvmIrStreamedArrayDataProviderState { NextSection, LastSection, + NextSectionNoData, + LastSectionNoData, } abstract class LlvmIrStreamedArrayDataProvider diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index 95d4099c9ea..9ec39da00c3 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -113,29 +113,32 @@ struct XamarinAndroidBundledAssembly final struct AssemblyEntry { - // offset into the `xa_input_assembly_data` array - uint32_t input_data_offset; + // offset into the `xa_input_assembly_data` array + uint32_t input_data_offset; - // number of bytes data of this assembly occupies - uint32_t input_data_size; + // number of bytes data of this assembly occupies + uint32_t input_data_size; - // offset into the `xa_uncompressed_assembly_data` array where the uncompressed - // assembly data (if any) lives. - uint32_t uncompressed_data_offset; + // offset into the `xa_uncompressed_assembly_data` array where the uncompressed + // assembly data (if any) lives. + uint32_t uncompressed_data_offset; - // Size of the uncompressed data. 0 if assembly wasn't compressed. - uint32_t uncompressed_data_size; + // Size of the uncompressed data. 0 if assembly wasn't compressed. + uint32_t uncompressed_data_size; }; struct AssemblyIndexEntry { - xamarin::android::hash_t name_hash; + xamarin::android::hash_t name_hash; - // Index into the `xa_assemblies` descriptor array - uint32_t index; + // Index into the `xa_assemblies` descriptor array + uint32_t index; - // whether hashed name had extension - bool has_extension; + // whether hashed name had extension + bool has_extension; + + // whether assembly data lives in a separate DSO + bool is_standalone; }; constexpr uint32_t InputAssemblyDataSize = 1024; From 00a3530817b88913e99ca960d177da94f1a9a624 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 4 Oct 2023 20:53:11 +0200 Subject: [PATCH 60/71] Code generation appears to be done, packaging done Removed all the sources related to the previous assembly compression implementation --- ...soft.Android.Sdk.NativeCompilation.targets | 45 +----- .../Tasks/BuildApk.cs | 39 +---- ...GenerateAppAssemblyDSONativeSourceFiles.cs | 2 +- ...teCompressedAssembliesNativeSourceFiles.cs | 101 ------------ .../Tasks/PrepareAbiItems.cs | 3 - .../Utilities/AssemblyCompression.cs | 42 ----- .../Utilities/AssemblyDSOGenerator.Classes.cs | 11 +- .../Utilities/AssemblyDSOGenerator.cs | 93 +++++++---- ...ressedAssembliesNativeAssemblyGenerator.cs | 146 ------------------ .../Utilities/CompressedAssemblyInfo.cs | 40 ----- .../Utilities/DSOAssemblyInfo.cs | 11 +- .../Xamarin.Android.Common.targets | 2 +- src/monodroid/jni/application_dso_stub.cc | 11 +- src/monodroid/jni/embedded-assemblies.cc | 54 +------ src/monodroid/jni/xamarin-app.hh | 25 +-- 15 files changed, 103 insertions(+), 522 deletions(-) delete mode 100644 src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssembliesNativeAssemblyGenerator.cs delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssemblyInfo.cs diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets index 79900b32d32..b00c0052b47 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets @@ -1,6 +1,5 @@ - @@ -50,15 +49,6 @@ Mode="environment"> - - - - @@ -123,10 +113,6 @@ %(_EnvironmentAssemblySource.abi) - <_NativeAssemblyTarget Include="@(_CompressedAssembliesAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> - %(_CompressedAssembliesAssemblySource.abi) - - <_NativeAssemblyTarget Include="@(_MarshalMethodsAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> %(_MarshalMethodsAssemblySource.abi) @@ -192,27 +178,13 @@ /> - - - - - - - - + + + diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index 631bd5154a0..22c80a85148 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -83,8 +83,6 @@ public class BuildApk : AndroidTask public string TlsProvider { get; set; } public string UncompressedFileExtensions { get; set; } - // Make it required after https://github.com/xamarin/monodroid/pull/1094 is merged - //[Required] public bool EnableCompression { get; set; } public bool IncludeWrapSh { get; set; } @@ -130,7 +128,7 @@ protected virtual void FixupArchive (ZipArchiveEx zip) { } List excludePatterns = new List (); - void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOutputPath, bool debug, bool compress, IDictionary compressedAssembliesInfo, string assemblyStoreApkName) + void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOutputPath, bool debug, string assemblyStoreApkName) { ArchiveFileList files = new ArchiveFileList (); bool refresh = true; @@ -197,7 +195,7 @@ void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOut } if (EmbedAssemblies) { - AddAssemblies (apk, debug, compress, compressedAssembliesInfo, assemblyStoreApkName); + AddAssemblies (apk, debug, assemblyStoreApkName); apk.Flush (); } @@ -312,18 +310,8 @@ public override bool RunTask () } bool debug = _Debug; - bool compress = !debug && EnableCompression; - IDictionary compressedAssembliesInfo = null; - if (compress) { - string key = CompressedAssemblyInfo.GetKey (ProjectFullPath); - Log.LogDebugMessage ($"Retrieving assembly compression info with key '{key}'"); - compressedAssembliesInfo = BuildEngine4.UnregisterTaskObjectAssemblyLocal> (key, RegisteredTaskObjectLifetime.Build); - if (compressedAssembliesInfo == null) - throw new InvalidOperationException ($"Assembly compression info not found for key '{key}'. Compression will not be performed."); - } - - ExecuteWithAbi (SupportedAbis, ApkInputPath, ApkOutputPath, debug, compress, compressedAssembliesInfo, assemblyStoreApkName: null); + ExecuteWithAbi (SupportedAbis, ApkInputPath, ApkOutputPath, debug, assemblyStoreApkName: null); outputFiles.Add (ApkOutputPath); if (CreatePackagePerAbi && SupportedAbis.Length > 1) { foreach (var abi in SupportedAbis) { @@ -332,7 +320,7 @@ public override bool RunTask () var apk = Path.GetFileNameWithoutExtension (ApkOutputPath); ExecuteWithAbi (new [] { abi }, String.Format ("{0}-{1}", ApkInputPath, abi), Path.Combine (path, String.Format ("{0}-{1}.apk", apk, abi)), - debug, compress, compressedAssembliesInfo, assemblyStoreApkName: abi); + debug, assemblyStoreApkName: abi); outputFiles.Add (Path.Combine (path, String.Format ("{0}-{1}.apk", apk, abi))); } } @@ -362,7 +350,7 @@ static Regex FileGlobToRegEx (string fileGlob, RegexOptions options) return new Regex (sb.ToString (), options); } - void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary compressedAssembliesInfo, string assemblyStoreApkName) + void AddAssemblies (ZipArchiveEx apk, bool debug, string assemblyStoreApkName) { string sourcePath; AssemblyStoreGenerator storeGenerator; @@ -389,12 +377,6 @@ void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary "GCANSF"; - - [Required] - public ITaskItem[] ResolvedAssemblies { get; set; } - - [Required] - public string [] SupportedAbis { get; set; } - - [Required] - public string EnvironmentOutputDirectory { get; set; } - - [Required] - public bool Debug { get; set; } - - [Required] - public bool EnableCompression { get; set; } - - [Required] - public string ProjectFullPath { get; set; } - - public override bool RunTask () - { - GenerateCompressedAssemblySources (); - return !Log.HasLoggedErrors; - } - - void GenerateCompressedAssemblySources () - { - if (Debug || !EnableCompression) { - Generate (null); - return; - } - - var assemblies = new SortedDictionary (StringComparer.Ordinal); - foreach (ITaskItem assembly in ResolvedAssemblies) { - if (bool.TryParse (assembly.GetMetadata ("AndroidSkipAddToPackage"), out bool value) && value) { - continue; - } - - var assemblyKey = CompressedAssemblyInfo.GetDictionaryKey (assembly); - if (assemblies.ContainsKey (assemblyKey)) { - Log.LogDebugMessage ($"Skipping duplicate assembly: {assembly.ItemSpec}"); - continue; - } - - var fi = new FileInfo (assembly.ItemSpec); - if (!fi.Exists) { - Log.LogError ($"Assembly {assembly.ItemSpec} does not exist"); - continue; - } - - assemblies.Add (assemblyKey, new CompressedAssemblyInfo (checked((uint)fi.Length))); - } - - uint index = 0; - foreach (var kvp in assemblies) { - kvp.Value.DescriptorIndex = index++; - } - - string key = CompressedAssemblyInfo.GetKey (ProjectFullPath); - Log.LogDebugMessage ($"Storing compression assemblies info with key '{key}'"); - BuildEngine4.RegisterTaskObjectAssemblyLocal (key, assemblies, RegisteredTaskObjectLifetime.Build); - Generate (assemblies); - - void Generate (IDictionary dict) - { - var composer = new CompressedAssembliesNativeAssemblyGenerator (dict); - LLVMIR.LlvmIrModule compressedAssemblies = composer.Construct (); - - foreach (string abi in SupportedAbis) { - string baseAsmFilePath = Path.Combine (EnvironmentOutputDirectory, $"compressed_assemblies.{abi.ToLowerInvariant ()}"); - string llvmIrFilePath = $"{baseAsmFilePath}.ll"; - - using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) { - try { - composer.Generate (compressedAssemblies, MonoAndroidHelper.AbiToTargetArch (abi), sw, llvmIrFilePath); - } catch { - throw; - } finally { - sw.Flush (); - } - - if (Files.CopyIfStreamChanged (sw.BaseStream, llvmIrFilePath)) { - Log.LogDebugMessage ($"File {llvmIrFilePath} was regenerated"); - } - } - } - } - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAbiItems.cs b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAbiItems.cs index c683c98c6b7..31b10540bc4 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAbiItems.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAbiItems.cs @@ -13,7 +13,6 @@ public class PrepareAbiItems : AndroidTask const string ArmV7a = "armeabi-v7a"; const string TypeMapBase = "typemaps"; const string EnvBase = "environment"; - const string CompressedAssembliesBase = "compressed_assemblies"; const string JniRemappingBase = "jni_remap"; const string MarshalMethodsBase = "marshal_methods"; public const string AssemblyDSOBase = "assembly_dso"; @@ -51,8 +50,6 @@ public override bool RunTask () baseName = TypeMapBase; } else if (String.Compare ("environment", Mode, StringComparison.OrdinalIgnoreCase) == 0) { baseName = EnvBase; - } else if (String.Compare ("compressed", Mode, StringComparison.OrdinalIgnoreCase) == 0) { - baseName = CompressedAssembliesBase; } else if (String.Compare ("jniremap", Mode, StringComparison.OrdinalIgnoreCase) == 0) { baseName = JniRemappingBase; } else if (String.Compare ("marshal_methods", Mode, StringComparison.OrdinalIgnoreCase) == 0) { diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs index f482a5c1f58..d0c89eaab8c 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs @@ -178,47 +178,5 @@ CompressionResult Compress (AssemblyData data, string outputDirectory) return (outputPath, true); } - - public string CompressAssembly (ITaskItem assembly, IDictionary compressedAssembliesInfo) - { - if (bool.TryParse (assembly.GetMetadata ("AndroidSkipCompression"), out bool value) && value) { - log.LogDebugMessage ($"Skipping compression of {assembly.ItemSpec} due to 'AndroidSkipCompression' == 'true' "); - return assembly.ItemSpec; - } - - var key = CompressedAssemblyInfo.GetDictionaryKey (assembly); - if (compressedAssembliesInfo.TryGetValue (key, out CompressedAssemblyInfo info) && info != null) { - AssemblyData compressedAssembly = new AssemblyData (assembly.ItemSpec, info.DescriptorIndex); - - string assemblyOutputDir; - string subDirectory = assembly.GetMetadata ("DestinationSubDirectory"); - if (!String.IsNullOrEmpty (subDirectory)) - assemblyOutputDir = Path.Combine (compressedOutputDir, subDirectory); - else - assemblyOutputDir = compressedOutputDir; - CompressionResult result = Compress (compressedAssembly, assemblyOutputDir); - if (result != CompressionResult.Success) { - switch (result) { - case CompressionResult.EncodingFailed: - log.LogMessage ($"Failed to compress {assembly.ItemSpec}"); - break; - - case CompressionResult.InputTooBig: - log.LogMessage ($"Input assembly {assembly.ItemSpec} exceeds maximum input size"); - break; - - default: - log.LogMessage ($"Unknown error compressing {assembly.ItemSpec}"); - break; - } - return assembly.ItemSpec; - } - return compressedAssembly.DestinationPath; - } else { - log.LogDebugMessage ($"Assembly missing from {nameof (CompressedAssemblyInfo)}: {key}"); - } - - return assembly.ItemSpec; - } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.Classes.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.Classes.cs index fea6c56a7e6..51aa0a0b15b 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.Classes.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.Classes.cs @@ -29,6 +29,9 @@ class StandaloneAssemblyEntry // Must be identical to AssemblyEntry in src/monodroid/jni/xamarin-app.hh sealed class AssemblyEntry : StandaloneAssemblyEntry { + [NativeAssembler (Ignore = true)] + public string Name; + // offset into the `xa_input_assembly_data` array public uint input_data_offset; @@ -49,9 +52,6 @@ class AssemblyIndexEntryBase [NativeAssembler (Ignore = true)] public string Name; - [NativeAssembler (Ignore = true)] - public byte[] NameBytes; - [NativeAssembler (NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal)] public T name_hash; @@ -78,6 +78,7 @@ sealed class AssembliesConfig public uint uncompressed_assembly_data_size; public uint assembly_name_length; public uint assembly_count; + public uint shared_library_name_length; }; // Members with underscores correspond to the native fields we output. @@ -86,6 +87,8 @@ sealed class ArchState public readonly List> xa_assemblies; public readonly List>? xa_assembly_index32; public readonly List>? xa_assembly_index64; + public readonly List xa_assembly_names; + public readonly List xa_assembly_dso_names; public readonly AssembliesConfig xa_assemblies_config; public readonly StandaloneAssemblyEntry? StandaloneAssembly; @@ -97,6 +100,8 @@ public ArchState (int assemblyCount, AndroidTargetArch arch, StandaloneAssemblyE StandaloneAssembly = standaloneAssembly; xa_assemblies = new List> (assemblyCount); + xa_assembly_names = new List (assemblyCount); + xa_assembly_dso_names = new List (assemblyCount); switch (arch) { case AndroidTargetArch.Arm64: diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs index e7c81434fd6..8e9fe04c457 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs @@ -12,6 +12,7 @@ partial class AssemblyDSOGenerator : LlvmIrComposer { const string XAAssembliesConfigVarName = "xa_assemblies_config"; const string XAAssembliesVarName = "xa_assemblies"; + const string XAAssemblyDSONamesVarName = "xa_assembly_dso_names"; const string XAAssemblyIndexVarName = "xa_assembly_index"; const string XAAssemblyNamesVarName = "xa_assembly_names"; public const string XAInputAssemblyDataVarName = "xa_input_assembly_data"; @@ -92,8 +93,6 @@ void ConstructStandalone (LlvmIrModule module) module.Add (xa_input_assembly_data); } - // TODO: need to store information about an assembly being standalone. Also, we need to store the shared library name somewhere (probably a separate array, at the same - // index as the xa_assemblies_index indicates void ConstructFastPath (LlvmIrModule module) { MapStructures (module); @@ -132,6 +131,7 @@ void ConstructFastPath (LlvmIrModule module) var xa_assemblies = new LlvmIrGlobalVariable (typeof(List>), XAAssembliesVarName) { BeforeWriteCallback = AssembliesBeforeWrite, + GetArrayItemCommentCallback = AssembliesItemComment, Options = LlvmIrVariableOptions.GlobalConstant, }; module.Add (xa_assemblies); @@ -149,6 +149,12 @@ void ConstructFastPath (LlvmIrModule module) }; module.Add (xa_assembly_names); + var xa_assembly_dso_names = new LlvmIrGlobalVariable (typeof(List), XAAssemblyDSONamesVarName) { + BeforeWriteCallback = AssemblyDSONamesBeforeWrite, + Options = LlvmIrVariableOptions.GlobalConstant, + }; + module.Add (xa_assembly_dso_names); + var xa_uncompressed_assembly_data = new LlvmIrGlobalVariable (typeof(byte[]), XAUncompressedAssemblyDataVarName) { Alignment = 4096, // align to page boundary, may make access slightly faster ArrayItemCount = uncompressedAssemblyDataSize, @@ -174,6 +180,18 @@ void AssembliesBeforeWrite (LlvmIrVariable variable, LlvmIrModuleTarget target, variable.Value = archState.xa_assemblies; } + string AssembliesItemComment (LlvmIrVariable variable, LlvmIrModuleTarget target, ulong index, object? itemValue, object? state) + { + ArchState archState = GetArchState (target); + var entry = itemValue as StructureInstance; + + if (entry != null) { + return MakeComment (((AssemblyEntry)entry.Obj).Name); + } + + throw new InvalidOperationException ($"Internal error: assembly array member has unsupported type '{itemValue?.GetType ()}'"); + } + string AssemblyIndexItemComment (LlvmIrVariable variable, LlvmIrModuleTarget target, ulong index, object? itemValue, object? state) { var value32 = itemValue as StructureInstance; @@ -187,45 +205,46 @@ string AssemblyIndexItemComment (LlvmIrVariable variable, LlvmIrModuleTarget tar } throw new InvalidOperationException ($"Internal error: assembly index array member has unsupported type '{itemValue?.GetType ()}'"); - - string MakeComment (string name) => $" => {name}"; } + static string MakeComment (string name) => $" => {name}"; + void AssemblyNamesBeforeWrite (LlvmIrVariable variable, LlvmIrModuleTarget target, object? state) { ArchState archState = GetArchState (target); - var names = new List (); + var names = new List ((int)archState.xa_assemblies_config.assembly_count); - if (target.TargetArch == AndroidTargetArch.Arm64 || target.TargetArch == AndroidTargetArch.X86_64) { - foreach (StructureInstance e in archState.xa_assembly_index64) { - var entry = (AssemblyIndexEntry64)e.Obj; - names.Add (GetProperlySizedBytes (entry.NameBytes)); - } - } else if (target.TargetArch == AndroidTargetArch.Arm || target.TargetArch == AndroidTargetArch.X86) { - foreach (StructureInstance e in archState.xa_assembly_index32) { - var entry = (AssemblyIndexEntry32)e.Obj; - names.Add (GetProperlySizedBytes (entry.NameBytes)); - } - } else { - throw new InvalidOperationException ($"Internal error: architecture {target.TargetArch} not supported"); + foreach (byte[] nameBytes in archState.xa_assembly_names) { + names.Add (GetProperlySizedBytesForNameArray (archState.xa_assemblies_config.assembly_name_length, nameBytes)); } variable.Value = names; + } - byte[] GetProperlySizedBytes (byte[] inputBytes) - { - if (inputBytes.Length > archState.xa_assemblies_config.assembly_name_length - 1) { - throw new ArgumentOutOfRangeException (nameof (inputBytes), $"Must not exceed {archState.xa_assemblies_config.assembly_name_length - 1} bytes"); - } - - var ret = new byte[archState.xa_assemblies_config.assembly_name_length]; - Array.Clear (ret, 0, ret.Length); - inputBytes.CopyTo (ret, 0); + void AssemblyDSONamesBeforeWrite (LlvmIrVariable variable, LlvmIrModuleTarget target, object? state) + { + ArchState archState = GetArchState (target); + var names = new List ((int)archState.xa_assemblies_config.assembly_count); - return ret; + foreach (byte[] nameBytes in archState.xa_assembly_dso_names) { + names.Add (GetProperlySizedBytesForNameArray (archState.xa_assemblies_config.shared_library_name_length, nameBytes)); } + + variable.Value = names; } + static byte[] GetProperlySizedBytesForNameArray (uint requiredSize, byte[] inputBytes) + { + if (inputBytes.Length > requiredSize - 1) { + throw new ArgumentOutOfRangeException (nameof (inputBytes), $"Must not exceed {requiredSize - 1} bytes"); + } + + var ret = new byte[requiredSize]; + Array.Clear (ret, 0, ret.Length); + inputBytes.CopyTo (ret, 0); + + return ret; + } void AssemblyIndexBeforeWrite (LlvmIrVariable variable, LlvmIrModuleTarget target, object? state) { ArchState archState = GetArchState (target); @@ -334,6 +353,7 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) ulong inputOffset = 0; ulong uncompressedOffset = 0; ulong assemblyNameLength = 0; + ulong sharedLibraryNameLength = 0; foreach (DSOAssemblyInfo info in infos) { bool isStandalone = info.IsStandalone; uint inputSize = GetInputSize (info); @@ -349,6 +369,7 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) // which it then iterates on, and the rented arrays can (and frequently will) be bigger than the requested size. AssemblyData = isStandalone ? null : new byte[inputSize], IsStandalone = isStandalone, + Name = info.Name, InputFilePath = info.InputFile, input_data_offset = (uint)inputOffset, input_data_size = inputSize, @@ -365,6 +386,17 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) archState.xa_assemblies.Add (new StructureInstance (assemblyEntryStructureInfo, entry)); byte[] nameBytes = StringToBytes (info.Name); + archState.xa_assembly_names.Add (nameBytes); + + byte[] sharedLibraryNameBytes = isStandalone ? StringToBytes (info.StandaloneDSOName) : Array.Empty (); + archState.xa_assembly_dso_names.Add (sharedLibraryNameBytes); + + if (sharedLibraryNameBytes != null) { + if (sharedLibraryNameLength < (ulong)sharedLibraryNameBytes.Length) { + sharedLibraryNameLength = (ulong)sharedLibraryNameBytes.Length; + } + } + if ((ulong)nameBytes.Length > assemblyNameLength) { assemblyNameLength = (ulong)nameBytes.Length; } @@ -389,7 +421,6 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) if (is64Bit) { var indexEntry = new AssemblyIndexEntry64 { Name = info.Name, - NameBytes = nameBytes, name_hash = nameHash, index = assemblyIndex, has_extension = true, @@ -399,7 +430,6 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) indexEntry = new AssemblyIndexEntry64 { Name = nameWithoutExtension, - NameBytes = nameWithoutExtensionBytes, name_hash = nameWithoutExtensionHash, index = assemblyIndex, has_extension = false, @@ -409,7 +439,6 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) } else { var indexEntry = new AssemblyIndexEntry32 { Name = info.Name, - NameBytes = nameBytes, name_hash = (uint)nameHash, index = assemblyIndex, is_standalone = isStandalone, @@ -418,7 +447,6 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) indexEntry = new AssemblyIndexEntry32 { Name = nameWithoutExtension, - NameBytes = nameWithoutExtensionBytes, name_hash = (uint)nameWithoutExtensionHash, index = assemblyIndex, has_extension = false, @@ -443,7 +471,8 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) archState.xa_assemblies_config.uncompressed_assembly_data_size = (uint)uncompressedOffset; // Must include the terminating NUL - archState.xa_assemblies_config.assembly_name_length = (uint)assemblyNameLength + 1; + archState.xa_assemblies_config.assembly_name_length = (uint)AddWithCheck (assemblyNameLength, 1, UInt32.MaxValue, "Assembly name is too long"); + archState.xa_assemblies_config.shared_library_name_length = (uint)AddWithCheck (sharedLibraryNameLength, 1, UInt32.MaxValue, "Shared library name is too long"); ulong AddWithCheck (ulong lhs, ulong rhs, ulong maxValue, string errorMessage) { diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssembliesNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssembliesNativeAssemblyGenerator.cs deleted file mode 100644 index e0b3740a453..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssembliesNativeAssemblyGenerator.cs +++ /dev/null @@ -1,146 +0,0 @@ -using System; -using System.Collections.Generic; - -using Xamarin.Android.Tasks.LLVMIR; - -namespace Xamarin.Android.Tasks -{ - partial class CompressedAssembliesNativeAssemblyGenerator : LlvmIrComposer - { - const string DescriptorsArraySymbolName = "compressed_assembly_descriptors"; - const string CompressedAssembliesSymbolName = "compressed_assemblies"; - - sealed class CompressedAssemblyDescriptorContextDataProvider : NativeAssemblerStructContextDataProvider - { - public override string? GetPointedToSymbolName (object data, string fieldName) - { - if (String.Compare ("data", fieldName, StringComparison.Ordinal) != 0) { - return null; - } - - var descriptor = EnsureType (data); - return descriptor.BufferSymbolName; - } - } - - // Order of fields and their type must correspond *exactly* to that in - // src/monodroid/jni/xamarin-app.hh CompressedAssemblyDescriptor structure - [NativeAssemblerStructContextDataProvider (typeof (CompressedAssemblyDescriptorContextDataProvider))] - sealed class CompressedAssemblyDescriptor - { - [NativeAssembler (Ignore = true)] - public string BufferSymbolName; - - public uint uncompressed_file_size; - public bool loaded; - - [NativeAssembler (UsesDataProvider = true), NativePointer (PointsToSymbol = "")] - public byte data; - }; - - sealed class CompressedAssembliesContextDataProvider : NativeAssemblerStructContextDataProvider - { - public override ulong GetBufferSize (object data, string fieldName) - { - if (String.Compare ("descriptors", fieldName, StringComparison.Ordinal) != 0) { - return 0; - } - - var cas = EnsureType (data); - return cas.count; - } - } - - // Order of fields and their type must correspond *exactly* to that in - // src/monodroid/jni/xamarin-app.hh CompressedAssemblies structure - [NativeAssemblerStructContextDataProvider (typeof (CompressedAssembliesContextDataProvider))] - sealed class CompressedAssemblies - { - public uint count; - - [NativeAssembler (UsesDataProvider = true), NativePointer (PointsToSymbol = DescriptorsArraySymbolName)] - public CompressedAssemblyDescriptor descriptors; - }; - - IDictionary assemblies; - StructureInfo compressedAssemblyDescriptorStructureInfo; - StructureInfo compressedAssembliesStructureInfo; - - public CompressedAssembliesNativeAssemblyGenerator (IDictionary assemblies) - { - this.assemblies = assemblies; - } - - void InitCompressedAssemblies (out List>? compressedAssemblyDescriptors, - out StructureInstance? compressedAssemblies, - out List? buffers) - { - if (assemblies == null || assemblies.Count == 0) { - compressedAssemblyDescriptors = null; - compressedAssemblies = null; - buffers = null; - return; - } - - ulong counter = 0; - compressedAssemblyDescriptors = new List> (assemblies.Count); - buffers = new List (assemblies.Count); - foreach (var kvp in assemblies) { - string assemblyName = kvp.Key; - CompressedAssemblyInfo info = kvp.Value; - - string bufferName = $"__compressedAssemblyData_{counter++}"; - var descriptor = new CompressedAssemblyDescriptor { - BufferSymbolName = bufferName, - uncompressed_file_size = info.FileSize, - loaded = false, - data = 0 - }; - - var bufferVar = new LlvmIrGlobalVariable (typeof(List), bufferName, LlvmIrVariableOptions.LocalWritable) { - ZeroInitializeArray = true, - ArrayItemCount = descriptor.uncompressed_file_size, - }; - buffers.Add (bufferVar); - - compressedAssemblyDescriptors.Add (new StructureInstance (compressedAssemblyDescriptorStructureInfo, descriptor)); - } - - compressedAssemblies = new StructureInstance (compressedAssembliesStructureInfo, new CompressedAssemblies { count = (uint)assemblies.Count }); - } - - protected override void Construct (LlvmIrModule module) - { - MapStructures (module); - - List>? compressedAssemblyDescriptors; - StructureInstance? compressedAssemblies; - List? buffers; - - InitCompressedAssemblies (out compressedAssemblyDescriptors, out compressedAssemblies, out buffers); - - if (compressedAssemblyDescriptors == null) { - module.AddGlobalVariable ( - typeof(StructureInstance), - CompressedAssembliesSymbolName, - new StructureInstance (compressedAssembliesStructureInfo, new CompressedAssemblies ()) { IsZeroInitialized = true }, - LlvmIrVariableOptions.GlobalWritable - ); - return; - } - - module.AddGlobalVariable (CompressedAssembliesSymbolName, compressedAssemblies, LlvmIrVariableOptions.GlobalWritable); - module.AddGlobalVariable (DescriptorsArraySymbolName, compressedAssemblyDescriptors, LlvmIrVariableOptions.LocalWritable); - - module.Add (new LlvmIrGroupDelimiterVariable ()); - module.Add (buffers); - module.Add (new LlvmIrGroupDelimiterVariable ()); - } - - void MapStructures (LlvmIrModule module) - { - compressedAssemblyDescriptorStructureInfo = module.MapStructure (); - compressedAssembliesStructureInfo = module.MapStructure (); - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssemblyInfo.cs b/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssemblyInfo.cs deleted file mode 100644 index b3f775a96b7..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssemblyInfo.cs +++ /dev/null @@ -1,40 +0,0 @@ -using System; -using System.IO; -using Microsoft.Build.Framework; - -namespace Xamarin.Android.Tasks -{ - class CompressedAssemblyInfo - { - const string CompressedAssembliesInfoKey = "__CompressedAssembliesInfo"; - - public uint FileSize { get; } - public uint DescriptorIndex { get; set; } - - public CompressedAssemblyInfo (uint fileSize) - { - FileSize = fileSize; - DescriptorIndex = 0; - } - - public static string GetKey (string projectFullPath) - { - if (String.IsNullOrEmpty (projectFullPath)) - throw new ArgumentException ("must be a non-empty string", nameof (projectFullPath)); - - return $"{CompressedAssembliesInfoKey}:{projectFullPath}"; - } - - public static string GetDictionaryKey (ITaskItem assembly) - { - // Prefer %(DestinationSubPath) if set - var path = assembly.GetMetadata ("DestinationSubPath"); - if (!string.IsNullOrEmpty (path)) { - return path; - } - // MSBuild sometimes only sets %(DestinationSubDirectory) - var subDirectory = assembly.GetMetadata ("DestinationSubDirectory"); - return Path.Combine (subDirectory, Path.GetFileName (assembly.ItemSpec)); - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/DSOAssemblyInfo.cs b/src/Xamarin.Android.Build.Tasks/Utilities/DSOAssemblyInfo.cs index de1acb4df24..eb94b9a9d47 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/DSOAssemblyInfo.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/DSOAssemblyInfo.cs @@ -1,3 +1,5 @@ +using System; + namespace Xamarin.Android.Tasks; class DSOAssemblyInfo @@ -27,18 +29,25 @@ class DSOAssemblyInfo /// public bool IsStandalone { get; } + public string? StandaloneDSOName { get; } + /// /// is the original assembly name, including culture prefix (e.g. `en_US/`) if it is a /// satellite assembly. should be the full path to the input file. /// gives the original file size, while specifies /// data size after compression, or `0` if file isn't compressed. /// - public DSOAssemblyInfo (string name, string inputFile, uint dataSize, uint compressedDataSize, bool isStandalone = false) + public DSOAssemblyInfo (string name, string inputFile, uint dataSize, uint compressedDataSize, bool isStandalone = false, string? dsoName = null) { + if (isStandalone && String.IsNullOrEmpty (dsoName)) { + throw new ArgumentException ("must not be null or empty for standalone assembly", nameof (dsoName)); + } + Name = name; InputFile = inputFile; DataSize = dataSize; CompressedDataSize = compressedDataSize; IsStandalone = isStandalone; + StandaloneDSOName = isStandalone ? dsoName : null; } } diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index fa26d187c15..e9dd74d4fec 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -1932,7 +1932,7 @@ because xbuild doesn't support framework reference assemblies. EmbedAssemblies="$(EmbedAssembliesIntoApk)" ResolvedUserAssemblies="@(_ShrunkUserAssemblies);@(_AndroidResolvedSatellitePaths)" ResolvedFrameworkAssemblies="@(_ShrunkFrameworkAssemblies)" - FrameworkNativeLibraries="@(FrameworkNativeLibrary)" + FrameworkNativeLibraries="@(FrameworkNativeLibrary);@(_StandaloneAssemblyDSO)" NativeLibraries="@(AndroidNativeLibrary)" ApplicationSharedLibraries="@(_ApplicationSharedLibrary)" AdditionalNativeLibraryReferences="@(_AdditionalNativeLibraryReferences)" diff --git a/src/monodroid/jni/application_dso_stub.cc b/src/monodroid/jni/application_dso_stub.cc index 9d474e65c39..e3dc90f15c8 100644 --- a/src/monodroid/jni/application_dso_stub.cc +++ b/src/monodroid/jni/application_dso_stub.cc @@ -31,11 +31,6 @@ const TypeMapJava map_java[] = {}; const xamarin::android::hash_t map_java_hashes[] = {}; #endif -CompressedAssemblies compressed_assemblies = { - .count = 0, - .descriptors = nullptr, -}; - // // Config settings below **must** be valid for Desktop builds as the default `libxamarin-app.{dll,dylib,so}` is used by // the Designer @@ -51,6 +46,7 @@ const ApplicationConfig application_config = { .jni_add_native_method_registration_attribute_present = false, .have_runtime_config_blob = false, .have_assembly_store = false, + .marshal_methods_enabled = false, .bound_exception_type = 0, // System .package_naming_policy = 0, .environment_variable_count = 0, @@ -245,6 +241,11 @@ const char xa_assembly_names[AssemblyCount][AssemblyNameLength] = { "AnotherAssembly2.dll", }; +const char xa_assembly_dso_names[AssemblyCount][SharedLibraryNameLength] = { + "libxaAssembly1.so", + "libxaAnotherAssembly2.so", +}; + const AssembliesConfig xa_assemblies_config = { .input_assembly_data_size = InputAssemblyDataSize, .uncompressed_assembly_data_size = UncompressedAssemblyDataSize, diff --git a/src/monodroid/jni/embedded-assemblies.cc b/src/monodroid/jni/embedded-assemblies.cc index 82b7607f437..ff53dc7ea48 100644 --- a/src/monodroid/jni/embedded-assemblies.cc +++ b/src/monodroid/jni/embedded-assemblies.cc @@ -85,59 +85,7 @@ force_inline void EmbeddedAssemblies::get_assembly_data (uint8_t *data, uint32_t data_size, [[maybe_unused]] const char *name, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept { #if defined (ANDROID) && defined (HAVE_LZ4) && defined (RELEASE) - auto header = reinterpret_cast(data); - if (header->magic == COMPRESSED_DATA_MAGIC) { - if (XA_UNLIKELY (compressed_assemblies.descriptors == nullptr)) { - log_fatal (LOG_ASSEMBLY, "Compressed assembly found but no descriptor defined"); - Helpers::abort_application (); - } - if (XA_UNLIKELY (header->descriptor_index >= compressed_assemblies.count)) { - log_fatal (LOG_ASSEMBLY, "Invalid compressed assembly descriptor index %u", header->descriptor_index); - Helpers::abort_application (); - } - - CompressedAssemblyDescriptor &cad = compressed_assemblies.descriptors[header->descriptor_index]; - assembly_data_size = data_size - sizeof(CompressedAssemblyHeader); - if (!cad.loaded) { - StartupAwareLock decompress_lock (assembly_decompress_mutex); - - if (cad.loaded) { - set_assembly_data_and_size (reinterpret_cast(cad.data), cad.uncompressed_file_size, assembly_data, assembly_data_size); - return; - } - - if (XA_UNLIKELY (cad.data == nullptr)) { - log_fatal (LOG_ASSEMBLY, "Invalid compressed assembly descriptor at %u: no data", header->descriptor_index); - Helpers::abort_application (); - } - - if (header->uncompressed_length != cad.uncompressed_file_size) { - if (header->uncompressed_length > cad.uncompressed_file_size) { - log_fatal (LOG_ASSEMBLY, "Compressed assembly '%s' is larger than when the application was built (expected at most %u, got %u). Assemblies don't grow just like that!", name, cad.uncompressed_file_size, header->uncompressed_length); - Helpers::abort_application (); - } else { - log_debug (LOG_ASSEMBLY, "Compressed assembly '%s' is smaller than when the application was built. Adjusting accordingly.", name); - } - cad.uncompressed_file_size = header->uncompressed_length; - } - - const char *data_start = reinterpret_cast(data + sizeof(CompressedAssemblyHeader)); - int ret = LZ4_decompress_safe (data_start, reinterpret_cast(cad.data), static_cast(assembly_data_size), static_cast(cad.uncompressed_file_size)); - - if (ret < 0) { - log_fatal (LOG_ASSEMBLY, "Decompression of assembly %s failed with code %d", name, ret); - Helpers::abort_application (); - } - - if (static_cast(ret) != cad.uncompressed_file_size) { - log_debug (LOG_ASSEMBLY, "Decompression of assembly %s yielded a different size (expected %lu, got %u)", name, cad.uncompressed_file_size, static_cast(ret)); - Helpers::abort_application (); - } - cad.loaded = true; - } - - set_assembly_data_and_size (reinterpret_cast(cad.data), cad.uncompressed_file_size, assembly_data, assembly_data_size); - } else + // TODO: implement storing info about standalone assembly DSO offset here #endif { set_assembly_data_and_size (data, data_size, assembly_data, assembly_data_size); diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index 9ec39da00c3..55d24e134db 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -11,7 +11,6 @@ #include "xxhash.hh" static constexpr uint64_t FORMAT_TAG = 0x015E6972616D58; -static constexpr uint32_t COMPRESSED_DATA_MAGIC = 0x5A4C4158; // 'XALZ', little-endian static constexpr uint32_t ASSEMBLY_STORE_MAGIC = 0x41424158; // 'XABA', little-endian static constexpr uint32_t ASSEMBLY_STORE_FORMAT_VERSION = 1; // Increase whenever an incompatible change is made to the // assembly store format @@ -81,26 +80,6 @@ struct TypeMapJava }; #endif -struct CompressedAssemblyHeader -{ - uint32_t magic; // COMPRESSED_DATA_MAGIC - uint32_t descriptor_index; - uint32_t uncompressed_length; -}; - -struct CompressedAssemblyDescriptor -{ - uint32_t uncompressed_file_size; - bool loaded; - uint8_t *data; -}; - -struct CompressedAssemblies -{ - uint32_t count; - CompressedAssemblyDescriptor *descriptors; -}; - struct XamarinAndroidBundledAssembly final { int32_t apk_fd; @@ -145,6 +124,7 @@ constexpr uint32_t InputAssemblyDataSize = 1024; constexpr uint32_t UncompressedAssemblyDataSize = 2048; constexpr uint32_t AssemblyCount = 2; constexpr uint32_t AssemblyNameLength = 26; // including the terminating NUL +constexpr uint32_t SharedLibraryNameLength = 32; // including the terminating NUL struct AssembliesConfig { @@ -152,6 +132,7 @@ struct AssembliesConfig uint32_t uncompressed_assembly_data_size; uint32_t assembly_name_length; uint32_t assembly_count; + uint32_t shared_library_name_length; }; MONO_API MONO_API_EXPORT const AssembliesConfig xa_assemblies_config; @@ -164,6 +145,7 @@ MONO_API MONO_API_EXPORT uint8_t xa_uncompressed_assembly_data[UncompressedAssem MONO_API MONO_API_EXPORT const AssemblyEntry xa_assemblies[AssemblyCount]; MONO_API MONO_API_EXPORT const AssemblyIndexEntry xa_assembly_index[AssemblyCount]; MONO_API MONO_API_EXPORT const char xa_assembly_names[AssemblyCount][AssemblyNameLength]; +MONO_API MONO_API_EXPORT const char xa_assembly_dso_names[AssemblyCount][SharedLibraryNameLength]; // // Assembly store format @@ -341,7 +323,6 @@ MONO_API MONO_API_EXPORT const TypeMapJava map_java[]; MONO_API MONO_API_EXPORT const xamarin::android::hash_t map_java_hashes[]; #endif -MONO_API MONO_API_EXPORT CompressedAssemblies compressed_assemblies; MONO_API MONO_API_EXPORT const ApplicationConfig application_config; MONO_API MONO_API_EXPORT const char* const app_environment_variables[]; MONO_API MONO_API_EXPORT const char* const app_system_properties[]; From 9b552a619dcd591bba7240b1d373f38c47463ef7 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 5 Oct 2023 22:02:48 +0200 Subject: [PATCH 61/71] =?UTF-8?q?Good=20bye,=20assembly=20blobs=20?= =?UTF-8?q?=F0=9F=91=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Still pondering how to make shared libraries with embedded assemblies perform well. --- .../Tasks/BuildApk.cs | 81 +--- .../Tasks/GeneratePackageManagerJava.cs | 7 +- .../PrepareAssemblyStandaloneDSOAbiItems.cs | 9 +- .../Utilities/ApplicationConfig.cs | 1 - ...pplicationConfigNativeAssemblyGenerator.cs | 92 +---- .../Utilities/ArchAssemblyStore.cs | 112 ------ .../Utilities/AssemblyStore.cs | 377 ------------------ .../Utilities/AssemblyStoreAssemblyInfo.cs | 48 --- .../Utilities/AssemblyStoreGenerator.cs | 133 ------ .../Utilities/AssemblyStoreGlobalIndex.cs | 29 -- .../Utilities/AssemblyStoreIndexEntry.cs | 43 -- .../Utilities/CommonAssemblyStore.cs | 36 -- .../Xamarin.Android.Common.targets | 9 +- src/monodroid/jni/application_dso_stub.cc | 34 +- src/monodroid/jni/embedded-assemblies-zip.cc | 70 +--- src/monodroid/jni/embedded-assemblies.cc | 152 +------ src/monodroid/jni/embedded-assemblies.hh | 14 - src/monodroid/jni/monodroid-glue.cc | 2 - src/monodroid/jni/xamarin-app.hh | 86 ---- 19 files changed, 39 insertions(+), 1296 deletions(-) delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyStore.cs delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStore.cs delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGlobalIndex.cs delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreIndexEntry.cs delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyStore.cs diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index 22c80a85148..aa9cf1882f5 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -93,6 +93,8 @@ public class BuildApk : AndroidTask public bool UseAssemblyStore { get; set; } + public bool UseAssemblySharedLibraries { get; set; } + public string ZipFlushFilesLimit { get; set; } public string ZipFlushSizeLimit { get; set; } @@ -352,30 +354,12 @@ static Regex FileGlobToRegEx (string fileGlob, RegexOptions options) void AddAssemblies (ZipArchiveEx apk, bool debug, string assemblyStoreApkName) { - string sourcePath; - AssemblyStoreGenerator storeGenerator; - - if (UseAssemblyStore) { - storeGenerator = new AssemblyStoreGenerator (AssembliesPath, Log); - } else { - storeGenerator = null; + if (UseAssemblySharedLibraries) { + // All the assemblies are in shared libraries + return; } - AssemblyStoreAssemblyInfo storeAssembly = null; - - // - // NOTE - // - // The very first store (ID 0) **must** contain an index of all the assemblies included in the application, even if they - // are included in other APKs than the base one. The ID 0 store **must** be placed in the base assembly - // - - // Currently, all the assembly stores end up in the "base" apk (the APK name is the key in the dictionary below) but the code is ready for the time when we - // partition assemblies into "feature" APKs - const string DefaultBaseApkName = "base"; - if (String.IsNullOrEmpty (assemblyStoreApkName)) { - assemblyStoreApkName = DefaultBaseApkName; - } + string sourcePath; // Add user assemblies AddAssembliesFromCollection (ResolvedUserAssemblies); @@ -383,37 +367,6 @@ void AddAssemblies (ZipArchiveEx apk, bool debug, string assemblyStoreApkName) // Add framework assemblies AddAssembliesFromCollection (ResolvedFrameworkAssemblies); - if (!UseAssemblyStore) { - return; - } - - Dictionary> assemblyStorePaths = storeGenerator.Generate (Path.GetDirectoryName (ApkOutputPath)); - if (assemblyStorePaths == null) { - throw new InvalidOperationException ("Assembly store generator did not generate any stores"); - } - - if (!assemblyStorePaths.TryGetValue (assemblyStoreApkName, out List baseAssemblyStores) || baseAssemblyStores == null || baseAssemblyStores.Count == 0) { - throw new InvalidOperationException ("Assembly store generator didn't generate the required base stores"); - } - - string assemblyStorePrefix = $"{assemblyStoreApkName}_"; - foreach (string assemblyStorePath in baseAssemblyStores) { - string inArchiveName = Path.GetFileName (assemblyStorePath); - - if (inArchiveName.StartsWith (assemblyStorePrefix, StringComparison.Ordinal)) { - inArchiveName = inArchiveName.Substring (assemblyStorePrefix.Length); - } - - CompressionMethod compressionMethod; - if (inArchiveName.EndsWith (".manifest", StringComparison.Ordinal)) { - compressionMethod = CompressionMethod.Default; - } else { - compressionMethod = UncompressedMethod; - } - - AddFileToArchiveIfNewer (apk, assemblyStorePath, AssembliesPath + inArchiveName, compressionMethod); - } - void AddAssembliesFromCollection (ITaskItem[] assemblies) { foreach (ITaskItem assembly in assemblies) { @@ -430,19 +383,11 @@ void AddAssembliesFromCollection (ITaskItem[] assemblies) // Add assembly var assemblyPath = GetAssemblyPath (assembly, frameworkAssembly: false); - if (UseAssemblyStore) { - storeAssembly = new AssemblyStoreAssemblyInfo (sourcePath, assemblyPath, assembly.GetMetadata ("Abi")); - } else { - AddFileToArchiveIfNewer (apk, sourcePath, assemblyPath + Path.GetFileName (assembly.ItemSpec), compressionMethod: UncompressedMethod); - } + AddFileToArchiveIfNewer (apk, sourcePath, assemblyPath + Path.GetFileName (assembly.ItemSpec), compressionMethod: UncompressedMethod); // Try to add config if exists var config = Path.ChangeExtension (assembly.ItemSpec, "dll.config"); - if (UseAssemblyStore) { - storeAssembly.SetConfigPath (config); - } else { - AddAssemblyConfigEntry (apk, assemblyPath, config); - } + AddAssemblyConfigEntry (apk, assemblyPath, config); // Try to add symbols if Debug if (debug) { @@ -454,17 +399,9 @@ void AddAssembliesFromCollection (ITaskItem[] assemblies) } if (!String.IsNullOrEmpty (symbolsPath)) { - if (UseAssemblyStore) { - storeAssembly.SetDebugInfoPath (symbolsPath); - } else { - AddFileToArchiveIfNewer (apk, symbolsPath, assemblyPath + Path.GetFileName (symbols), compressionMethod: UncompressedMethod); - } + AddFileToArchiveIfNewer (apk, symbolsPath, assemblyPath + Path.GetFileName (symbols), compressionMethod: UncompressedMethod); } } - - if (UseAssemblyStore) { - storeGenerator.Add (assemblyStoreApkName, storeAssembly); - } } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index 7690b47fe50..e0a5deca367 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -226,7 +226,7 @@ void AddEnvironment () Encoding assemblyNameEncoding = Encoding.UTF8; Action updateNameWidth = (ITaskItem assembly) => { - if (UseAssemblyStore) { + if (UseAssemblyStore) { // TODO: modify for assemblies embedded in DSOs return; } @@ -249,7 +249,7 @@ void AddEnvironment () uniqueAssemblyNames.Add (assemblyName); } - if (!UseAssemblyStore) { + if (!UseAssemblyStore) { // TODO: modify for assemblies embedded in DSOs assemblyCount++; return; } @@ -296,7 +296,7 @@ void AddEnvironment () GetRequiredTokens (assembly.ItemSpec, out android_runtime_jnienv_class_token, out jnienv_initialize_method_token, out jnienv_registerjninatives_method_token); } - if (!UseAssemblyStore) { + if (!UseAssemblyStore) { // TODO: modify for assemblies embedded in DSOs int abiNameLength = 0; foreach (string abi in SupportedAbis) { if (abi.Length <= abiNameLength) { @@ -372,7 +372,6 @@ void AddEnvironment () // and in the same order. MonoComponents = (MonoComponent)monoComponents, NativeLibraries = uniqueNativeLibraries, - HaveAssemblyStore = UseAssemblyStore, AndroidRuntimeJNIEnvToken = android_runtime_jnienv_class_token, JNIEnvInitializeToken = jnienv_initialize_method_token, JNIEnvRegisterJniNativesToken = jnienv_registerjninatives_method_token, diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAssemblyStandaloneDSOAbiItems.cs b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAssemblyStandaloneDSOAbiItems.cs index edc8bd69449..395008783ba 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAssemblyStandaloneDSOAbiItems.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAssemblyStandaloneDSOAbiItems.cs @@ -42,6 +42,8 @@ List PrepareItems () var fastPathItems = new HashSet (FastPathAssemblyNames, StringComparer.OrdinalIgnoreCase); var seenAbis = new HashSet (StringComparer.Ordinal); var satelliteAssemblies = new List (); + ushort dsoIndexCounter = 0; + var assemblyIndexes = new Dictionary (StringComparer.OrdinalIgnoreCase); foreach (ITaskItem assembly in Assemblies) { if (fastPathItems.Contains (Path.GetFileName (assembly.ItemSpec))) { @@ -79,8 +81,13 @@ List PrepareItems () ITaskItem MakeTaskItem (ITaskItem assembly, string abi, string baseName) { + if (!assemblyIndexes.TryGetValue (baseName, out ushort index)) { + index = dsoIndexCounter++; + assemblyIndexes.Add (baseName, index); + } + // the 'xa' infix is to make it harder to produce library names that clash with 3rd party libraries - string dsoName = $"libxa{baseName}.so"; + string dsoName = $"libxa{baseName}.{index:X04}.so"; var item = new TaskItem (Path.Combine (SharedLibraryOutputDir, abi, dsoName)); item.SetMetadata ("Abi", abi); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfig.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfig.cs index 21fb00a63f5..3997baf2172 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfig.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfig.cs @@ -32,7 +32,6 @@ sealed class ApplicationConfig public bool instant_run_enabled ; public bool jni_add_native_method_registration_attribute_present; public bool have_runtime_config_blob; - public bool have_assemblies_blob; public bool marshal_methods_enabled; public byte bound_stream_io_exception_type; public uint package_naming_policy; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs index 294cb9cb5a9..ad1bae62f48 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs @@ -57,49 +57,6 @@ sealed class DSOCacheEntry public IntPtr handle = IntPtr.Zero; } - // Order of fields and their type must correspond *exactly* to that in - // src/monodroid/jni/xamarin-app.hh AssemblyStoreAssemblyDescriptor structure - sealed class AssemblyStoreAssemblyDescriptor - { - public uint data_offset; - public uint data_size; - - public uint debug_data_offset; - public uint debug_data_size; - - public uint config_data_offset; - public uint config_data_size; - } - - // Order of fields and their type must correspond *exactly* to that in - // src/monodroid/jni/xamarin-app.hh AssemblyStoreSingleAssemblyRuntimeData structure - sealed class AssemblyStoreSingleAssemblyRuntimeData - { - [NativePointer] - public byte image_data; - - [NativePointer] - public byte debug_info_data; - - [NativePointer] - public byte config_data; - - [NativePointer] - public AssemblyStoreAssemblyDescriptor descriptor; - } - - // Order of fields and their type must correspond *exactly* to that in - // src/monodroid/jni/xamarin-app.hh AssemblyStoreRuntimeData structure - sealed class AssemblyStoreRuntimeData - { - [NativePointer] - public byte data_start; - public uint assembly_count; - - [NativePointer] - public AssemblyStoreAssemblyDescriptor assemblies; - } - sealed class XamarinAndroidBundledAssemblyContextDataProvider : NativeAssemblerStructContextDataProvider { public override ulong GetBufferSize (object data, string fieldName) @@ -211,7 +168,6 @@ protected override void Construct (LlvmIrModule module) instant_run_enabled = InstantRunEnabled, jni_add_native_method_registration_attribute_present = JniAddNativeMethodRegistrationAttributePresent, have_runtime_config_blob = HaveRuntimeConfigBlob, - have_assemblies_blob = HaveAssemblyStore, marshal_methods_enabled = MarshalMethodsEnabled, bound_stream_io_exception_type = (byte)BoundExceptionType, package_naming_policy = (uint)PackageNamingPolicy, @@ -239,21 +195,19 @@ protected override void Construct (LlvmIrModule module) }; module.Add (dso_cache); - if (!HaveAssemblyStore) { - xamarinAndroidBundledAssemblies = new List> (NumberOfAssembliesInApk); - - var emptyBundledAssemblyData = new XamarinAndroidBundledAssembly { - apk_fd = -1, - data_offset = 0, - data_size = 0, - data = 0, - name_length = (uint)BundledAssemblyNameWidth, - name = null, - }; + // TODO: don't generate if we're embedding assemblies into DSOs + xamarinAndroidBundledAssemblies = new List> (NumberOfAssembliesInApk); + var emptyBundledAssemblyData = new XamarinAndroidBundledAssembly { + apk_fd = -1, + data_offset = 0, + data_size = 0, + data = 0, + name_length = (uint)BundledAssemblyNameWidth, + name = null, + }; - for (int i = 0; i < NumberOfAssembliesInApk; i++) { - xamarinAndroidBundledAssemblies.Add (new StructureInstance (xamarinAndroidBundledAssemblyStructureInfo, emptyBundledAssemblyData)); - } + for (int i = 0; i < NumberOfAssembliesInApk; i++) { + xamarinAndroidBundledAssemblies.Add (new StructureInstance (xamarinAndroidBundledAssemblyStructureInfo, emptyBundledAssemblyData)); } string bundledBuffersSize = xamarinAndroidBundledAssemblies == null ? "empty (unused when assembly stores are enabled)" : $"{BundledAssemblyNameWidth} bytes long"; @@ -262,25 +216,6 @@ protected override void Construct (LlvmIrModule module) Comment = $" Bundled assembly name buffers, all {bundledBuffersSize}", }; module.Add (bundled_assemblies); - - AddAssemblyStores (module); - } - - void AddAssemblyStores (LlvmIrModule module) - { - ulong itemCount = (ulong)(HaveAssemblyStore ? NumberOfAssembliesInApk : 0); - var assembly_store_bundled_assemblies = new LlvmIrGlobalVariable (typeof(List>), "assembly_store_bundled_assemblies", LlvmIrVariableOptions.GlobalWritable) { - ZeroInitializeArray = true, - ArrayItemCount = itemCount, - }; - module.Add (assembly_store_bundled_assemblies); - - itemCount = (ulong)(HaveAssemblyStore ? NumberOfAssemblyStoresInApks : 0); - var assembly_stores = new LlvmIrGlobalVariable (typeof(List>), "assembly_stores", LlvmIrVariableOptions.GlobalWritable) { - ZeroInitializeArray = true, - ArrayItemCount = itemCount, - }; - module.Add (assembly_stores); } void HashAndSortDSOCache (LlvmIrVariable variable, LlvmIrModuleTarget target, object? state) @@ -372,9 +307,6 @@ void AddNameMutations (string name) void MapStructures (LlvmIrModule module) { applicationConfigStructureInfo = module.MapStructure (); - module.MapStructure (); - assemblyStoreSingleAssemblyRuntimeDataStructureinfo = module.MapStructure (); - assemblyStoreRuntimeDataStructureInfo = module.MapStructure (); xamarinAndroidBundledAssemblyStructureInfo = module.MapStructure (); dsoCacheEntryStructureInfo = module.MapStructure (); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyStore.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyStore.cs deleted file mode 100644 index a5b5811b7de..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyStore.cs +++ /dev/null @@ -1,112 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; - -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; - -namespace Xamarin.Android.Tasks -{ - class ArchAssemblyStore : AssemblyStore - { - readonly Dictionary> assemblies; - HashSet seenArchAssemblyNames; - - public ArchAssemblyStore (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log, uint id, AssemblyStoreGlobalIndex globalIndexCounter) - : base (apkName, archiveAssembliesPrefix, log, id, globalIndexCounter) - { - assemblies = new Dictionary> (StringComparer.OrdinalIgnoreCase); - } - - public override string WriteIndex (List globalIndex) - { - throw new InvalidOperationException ("Architecture-specific assembly blob cannot contain global assembly index"); - } - - public override void Add (AssemblyStoreAssemblyInfo blobAssembly) - { - if (String.IsNullOrEmpty (blobAssembly.Abi)) { - throw new InvalidOperationException ($"Architecture-agnostic assembly cannot be added to an architecture-specific blob ({blobAssembly.FilesystemAssemblyPath})"); - } - - if (!assemblies.ContainsKey (blobAssembly.Abi)) { - assemblies.Add (blobAssembly.Abi, new List ()); - } - - List blobAssemblies = assemblies[blobAssembly.Abi]; - blobAssemblies.Add (blobAssembly); - - if (seenArchAssemblyNames == null) { - seenArchAssemblyNames = new HashSet (StringComparer.Ordinal); - } - - string assemblyName = GetAssemblyName (blobAssembly); - if (seenArchAssemblyNames.Contains (assemblyName)) { - return; - } - - seenArchAssemblyNames.Add (assemblyName); - } - - public override void Generate (string outputDirectory, List globalIndex, List blobPaths) - { - if (assemblies.Count == 0) { - return; - } - - var assemblyNames = new Dictionary (); - foreach (var kvp in assemblies) { - string abi = kvp.Key; - List archAssemblies = kvp.Value; - - // All the architecture blobs must have assemblies in exactly the same order - archAssemblies.Sort ((AssemblyStoreAssemblyInfo a, AssemblyStoreAssemblyInfo b) => Path.GetFileName (a.FilesystemAssemblyPath).CompareTo (Path.GetFileName (b.FilesystemAssemblyPath))); - if (assemblyNames.Count == 0) { - for (int i = 0; i < archAssemblies.Count; i++) { - AssemblyStoreAssemblyInfo info = archAssemblies[i]; - assemblyNames.Add (i, Path.GetFileName (info.FilesystemAssemblyPath)); - } - continue; - } - - if (archAssemblies.Count != assemblyNames.Count) { - throw new InvalidOperationException ($"Assembly list for ABI '{abi}' has a different number of assemblies than other ABI lists (expected {assemblyNames.Count}, found {archAssemblies.Count}"); - } - - for (int i = 0; i < archAssemblies.Count; i++) { - AssemblyStoreAssemblyInfo info = archAssemblies[i]; - string fileName = Path.GetFileName (info.FilesystemAssemblyPath); - - if (assemblyNames[i] != fileName) { - throw new InvalidOperationException ($"Assembly list for ABI '{abi}' differs from other lists at index {i}. Expected '{assemblyNames[i]}', found '{fileName}'"); - } - } - } - - bool addToGlobalIndex = true; - foreach (var kvp in assemblies) { - string abi = kvp.Key; - List archAssemblies = kvp.Value; - - if (archAssemblies.Count == 0) { - continue; - } - - // Android uses underscores in place of dashes in ABI names, let's follow the convention - string androidAbi = abi.Replace ('-', '_'); - Generate (Path.Combine (outputDirectory, $"{ApkName}_{BlobPrefix}.{androidAbi}{BlobExtension}"), archAssemblies, globalIndex, blobPaths, addToGlobalIndex); - - // NOTE: not thread safe! The counter must grow monotonically but we also don't want to use different index values for the architecture-specific - // assemblies with the same names, that would only waste space in the generated `libxamarin-app.so`. To use the same index values for the same - // assemblies in different architectures we need to move the counter back here. - GlobalIndexCounter.Subtract ((uint)archAssemblies.Count); - - if (addToGlobalIndex) { - // We want the architecture-specific assemblies to be added to the global index only once - addToGlobalIndex = false; - } - } - - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStore.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStore.cs deleted file mode 100644 index 378a9ee8c46..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStore.cs +++ /dev/null @@ -1,377 +0,0 @@ -using System; -using System.Buffers; -using System.Collections.Generic; -using System.IO; -using System.Text; - -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; - -namespace Xamarin.Android.Tasks -{ - abstract class AssemblyStore - { - // The two constants below must match their counterparts in src/monodroid/jni/xamarin-app.hh - const uint BlobMagic = 0x41424158; // 'XABA', little-endian, must match the BUNDLED_ASSEMBLIES_BLOB_MAGIC native constant - const uint BlobVersion = 1; // Must match the BUNDLED_ASSEMBLIES_BLOB_VERSION native constant - - // MUST be equal to the size of the BlobBundledAssembly struct in src/monodroid/jni/xamarin-app.hh - const uint BlobBundledAssemblyNativeStructSize = 6 * sizeof (uint); - - // MUST be equal to the size of the BlobHashEntry struct in src/monodroid/jni/xamarin-app.hh - const uint BlobHashEntryNativeStructSize = sizeof (ulong) + (3 * sizeof (uint)); - - // MUST be equal to the size of the BundledAssemblyBlobHeader struct in src/monodroid/jni/xamarin-app.hh - const uint BlobHeaderNativeStructSize = sizeof (uint) * 5; - - protected const string BlobPrefix = "assemblies"; - protected const string BlobExtension = ".blob"; - - static readonly ArrayPool bytePool = ArrayPool.Shared; - - string archiveAssembliesPrefix; - string indexBlobPath; - - protected string ApkName { get; } - protected TaskLoggingHelper Log { get; } - protected AssemblyStoreGlobalIndex GlobalIndexCounter { get; } - - public uint ID { get; } - public bool IsIndexStore => ID == 0; - - protected AssemblyStore (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log, uint id, AssemblyStoreGlobalIndex globalIndexCounter) - { - if (String.IsNullOrEmpty (archiveAssembliesPrefix)) { - throw new ArgumentException ("must not be null or empty", nameof (archiveAssembliesPrefix)); - } - - if (String.IsNullOrEmpty (apkName)) { - throw new ArgumentException ("must not be null or empty", nameof (apkName)); - } - - GlobalIndexCounter = globalIndexCounter ?? throw new ArgumentNullException (nameof (globalIndexCounter)); - ID = id; - - this.archiveAssembliesPrefix = archiveAssembliesPrefix; - ApkName = apkName; - Log = log; - } - - public abstract void Add (AssemblyStoreAssemblyInfo blobAssembly); - public abstract void Generate (string outputDirectory, List globalIndex, List blobPaths); - - public virtual string WriteIndex (List globalIndex) - { - if (!IsIndexStore) { - throw new InvalidOperationException ("Assembly index may be written only to blob with index 0"); - } - - if (String.IsNullOrEmpty (indexBlobPath)) { - throw new InvalidOperationException ("Index blob path not set, was Generate called properly?"); - } - - if (globalIndex == null) { - throw new ArgumentNullException (nameof (globalIndex)); - } - - string indexBlobHeaderPath = $"{indexBlobPath}.hdr"; - string indexBlobManifestPath = Path.ChangeExtension (indexBlobPath, "manifest"); - - using (var hfs = File.Open (indexBlobHeaderPath, FileMode.Create, FileAccess.Write, FileShare.None)) { - using (var writer = new BinaryWriter (hfs, Encoding.UTF8, leaveOpen: true)) { - WriteIndex (writer, indexBlobManifestPath, globalIndex); - writer.Flush (); - } - - using (var ifs = File.Open (indexBlobPath, FileMode.Open, FileAccess.Read, FileShare.Read)) { - ifs.CopyTo (hfs); - hfs.Flush (); - } - } - - File.Delete (indexBlobPath); - File.Move (indexBlobHeaderPath, indexBlobPath); - - return indexBlobManifestPath; - } - - void WriteIndex (BinaryWriter blobWriter, string manifestPath, List globalIndex) - { - using (var manifest = File.Open (manifestPath, FileMode.Create, FileAccess.Write)) { - using (var manifestWriter = new StreamWriter (manifest, new UTF8Encoding (false))) { - WriteIndex (blobWriter, manifestWriter, globalIndex); - manifestWriter.Flush (); - } - } - } - - void WriteIndex (BinaryWriter blobWriter, StreamWriter manifestWriter, List globalIndex) - { - uint localEntryCount = 0; - var localAssemblies = new List (); - - manifestWriter.WriteLine ("Hash 32 Hash 64 Blob ID Blob idx Name"); - - var seenHashes32 = new HashSet (); - var seenHashes64 = new HashSet (); - bool haveDuplicates = false; - foreach (AssemblyStoreIndexEntry assembly in globalIndex) { - if (assembly.StoreID == ID) { - localEntryCount++; - localAssemblies.Add (assembly); - } - - if (WarnAboutDuplicateHash ("32", assembly.Name, assembly.NameHash32, seenHashes32) || - WarnAboutDuplicateHash ("64", assembly.Name, assembly.NameHash64, seenHashes64)) { - haveDuplicates = true; - } - - manifestWriter.WriteLine ($"0x{assembly.NameHash32:x08} 0x{assembly.NameHash64:x016} {assembly.StoreID:d03} {assembly.LocalBlobIndex:d04} {assembly.Name}"); - } - - if (haveDuplicates) { - throw new InvalidOperationException ("Duplicate assemblies encountered"); - } - - uint globalAssemblyCount = (uint)globalIndex.Count; - - blobWriter.Seek (0, SeekOrigin.Begin); - WriteBlobHeader (blobWriter, localEntryCount, globalAssemblyCount); - - // Header and two tables of the same size, each for 32 and 64-bit hashes - uint offsetFixup = BlobHeaderNativeStructSize + (BlobHashEntryNativeStructSize * globalAssemblyCount * 2); - - WriteAssemblyDescriptors (blobWriter, localAssemblies, CalculateOffsetFixup ((uint)localAssemblies.Count, offsetFixup)); - - var sortedIndex = new List (globalIndex); - sortedIndex.Sort ((AssemblyStoreIndexEntry a, AssemblyStoreIndexEntry b) => a.NameHash32.CompareTo (b.NameHash32)); - foreach (AssemblyStoreIndexEntry entry in sortedIndex) { - WriteHash (entry, entry.NameHash32); - } - - sortedIndex.Sort ((AssemblyStoreIndexEntry a, AssemblyStoreIndexEntry b) => a.NameHash64.CompareTo (b.NameHash64)); - foreach (AssemblyStoreIndexEntry entry in sortedIndex) { - WriteHash (entry, entry.NameHash64); - } - - void WriteHash (AssemblyStoreIndexEntry entry, ulong hash) - { - blobWriter.Write (hash); - blobWriter.Write (entry.MappingIndex); - blobWriter.Write (entry.LocalBlobIndex); - blobWriter.Write (entry.StoreID); - } - - bool WarnAboutDuplicateHash (string bitness, string assemblyName, ulong hash, HashSet seenHashes) - { - if (seenHashes.Contains (hash)) { - Log.LogMessage (MessageImportance.High, $"Duplicate {bitness}-bit hash 0x{hash} encountered for assembly {assemblyName}"); - return true; - } - - seenHashes.Add (hash); - return false; - } - } - - protected string GetAssemblyName (AssemblyStoreAssemblyInfo assembly) - { - string assemblyName = Path.GetFileNameWithoutExtension (assembly.FilesystemAssemblyPath); - if (assemblyName.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) { - assemblyName = Path.GetFileNameWithoutExtension (assemblyName); - } - - return assemblyName; - } - - protected void Generate (string outputFilePath, List assemblies, List globalIndex, List blobPaths, bool addToGlobalIndex = true) - { - if (globalIndex == null) { - throw new ArgumentNullException (nameof (globalIndex)); - } - - if (blobPaths == null) { - throw new ArgumentNullException (nameof (blobPaths)); - } - - if (IsIndexStore) { - indexBlobPath = outputFilePath; - } - - blobPaths.Add (outputFilePath); - Log.LogMessage (MessageImportance.Low, $"AssemblyBlobGenerator: generating blob: {outputFilePath}"); - - using (var fs = File.Open (outputFilePath, FileMode.Create, FileAccess.Write, FileShare.Read)) { - using (var writer = new BinaryWriter (fs, Encoding.UTF8)) { - Generate (writer, assemblies, globalIndex, addToGlobalIndex); - writer.Flush (); - } - } - } - - void Generate (BinaryWriter writer, List assemblies, List globalIndex, bool addToGlobalIndex) - { - var localAssemblies = new List (); - - if (!IsIndexStore) { - // Index blob's header and data before the assemblies is handled in WriteIndex in a slightly different - // way. - uint nbytes = BlobHeaderNativeStructSize + (BlobBundledAssemblyNativeStructSize * (uint)assemblies.Count); - var zeros = bytePool.Rent ((int)nbytes); - writer.Write (zeros, 0, (int)nbytes); - bytePool.Return (zeros); - } - - foreach (AssemblyStoreAssemblyInfo assembly in assemblies) { - string assemblyName = GetAssemblyName (assembly); - string archivePath = assembly.ArchiveAssemblyPath; - if (archivePath.StartsWith (archiveAssembliesPrefix, StringComparison.OrdinalIgnoreCase)) { - archivePath = archivePath.Substring (archiveAssembliesPrefix.Length); - } - - if (!String.IsNullOrEmpty (assembly.Abi)) { - string abiPath = $"{assembly.Abi}/"; - if (archivePath.StartsWith (abiPath, StringComparison.Ordinal)) { - archivePath = archivePath.Substring (abiPath.Length); - } - } - - if (!String.IsNullOrEmpty (archivePath)) { - if (archivePath.EndsWith ("/", StringComparison.Ordinal)) { - assemblyName = $"{archivePath}{assemblyName}"; - } else { - assemblyName = $"{archivePath}/{assemblyName}"; - } - } - - AssemblyStoreIndexEntry entry = WriteAssembly (writer, assembly, assemblyName, (uint)localAssemblies.Count); - if (addToGlobalIndex) { - globalIndex.Add (entry); - } - localAssemblies.Add (entry); - } - - writer.Flush (); - - if (IsIndexStore) { - return; - } - - writer.Seek (0, SeekOrigin.Begin); - WriteBlobHeader (writer, (uint)localAssemblies.Count); - WriteAssemblyDescriptors (writer, localAssemblies); - } - - uint CalculateOffsetFixup (uint localAssemblyCount, uint extraOffset = 0) - { - return (BlobBundledAssemblyNativeStructSize * (uint)localAssemblyCount) + extraOffset; - } - - void WriteBlobHeader (BinaryWriter writer, uint localEntryCount, uint globalEntryCount = 0) - { - // Header, must be identical to the BundledAssemblyBlobHeader structure in src/monodroid/jni/xamarin-app.hh - writer.Write (BlobMagic); // magic - writer.Write (BlobVersion); // version - writer.Write (localEntryCount); // local_entry_count - writer.Write (globalEntryCount); // global_entry_count - writer.Write ((uint)ID); // blob_id - } - - void WriteAssemblyDescriptors (BinaryWriter writer, List assemblies, uint offsetFixup = 0) - { - // Each assembly must be identical to the BlobBundledAssembly structure in src/monodroid/jni/xamarin-app.hh - - foreach (AssemblyStoreIndexEntry assembly in assemblies) { - AdjustOffsets (assembly, offsetFixup); - - writer.Write (assembly.DataOffset); - writer.Write (assembly.DataSize); - - writer.Write (assembly.DebugDataOffset); - writer.Write (assembly.DebugDataSize); - - writer.Write (assembly.ConfigDataOffset); - writer.Write (assembly.ConfigDataSize); - } - } - - void AdjustOffsets (AssemblyStoreIndexEntry assembly, uint offsetFixup) - { - if (offsetFixup == 0) { - return; - } - - assembly.DataOffset += offsetFixup; - - if (assembly.DebugDataOffset > 0) { - assembly.DebugDataOffset += offsetFixup; - } - - if (assembly.ConfigDataOffset > 0) { - assembly.ConfigDataOffset += offsetFixup; - } - } - - AssemblyStoreIndexEntry WriteAssembly (BinaryWriter writer, AssemblyStoreAssemblyInfo assembly, string assemblyName, uint localBlobIndex) - { - uint offset; - uint size; - - (offset, size) = WriteFile (assembly.FilesystemAssemblyPath, true); - - // NOTE: globalAssemblIndex++ is not thread safe but it **must** increase monotonically (see also ArchAssemblyStore.Generate for a special case) - var ret = new AssemblyStoreIndexEntry (assemblyName, ID, GlobalIndexCounter.Increment (), localBlobIndex) { - DataOffset = offset, - DataSize = size, - }; - - (offset, size) = WriteFile (assembly.DebugInfoPath, required: false); - if (offset != 0 && size != 0) { - ret.DebugDataOffset = offset; - ret.DebugDataSize = size; - } - - // Config files must end with \0 (nul) - (offset, size) = WriteFile (assembly.ConfigPath, required: false, appendNul: true); - if (offset != 0 && size != 0) { - ret.ConfigDataOffset = offset; - ret.ConfigDataSize = size; - } - - return ret; - - (uint offset, uint size) WriteFile (string filePath, bool required, bool appendNul = false) - { - if (!File.Exists (filePath)) { - if (required) { - throw new InvalidOperationException ($"Required file '{filePath}' not found"); - } - - return (0, 0); - } - - var fi = new FileInfo (filePath); - if (fi.Length == 0) { - return (0, 0); - } - - if (fi.Length > UInt32.MaxValue || writer.BaseStream.Position + fi.Length > UInt32.MaxValue) { - throw new InvalidOperationException ($"Writing assembly '{filePath}' to assembly blob would exceed the maximum allowed data size."); - } - - uint offset = (uint)writer.BaseStream.Position; - using (var fs = File.Open (filePath, FileMode.Open, FileAccess.Read, FileShare.Read)) { - fs.CopyTo (writer.BaseStream); - } - - uint length = (uint)fi.Length; - if (appendNul) { - length++; - writer.Write ((byte)0); - } - - return (offset, length); - } - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs deleted file mode 100644 index c5c166fb787..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs +++ /dev/null @@ -1,48 +0,0 @@ -using System; -using System.IO; - -namespace Xamarin.Android.Tasks -{ - class AssemblyStoreAssemblyInfo - { - public string FilesystemAssemblyPath { get; } - public string ArchiveAssemblyPath { get; } - public string DebugInfoPath { get; private set; } - public string ConfigPath { get; private set; } - public string Abi { get; } - - public AssemblyStoreAssemblyInfo (string filesystemAssemblyPath, string archiveAssemblyPath, string abi) - { - if (String.IsNullOrEmpty (filesystemAssemblyPath)) { - throw new ArgumentException ("must not be null or empty", nameof (filesystemAssemblyPath)); - } - - if (String.IsNullOrEmpty (archiveAssemblyPath)) { - throw new ArgumentException ("must not be null or empty", nameof (archiveAssemblyPath)); - } - - FilesystemAssemblyPath = filesystemAssemblyPath; - ArchiveAssemblyPath = archiveAssemblyPath; - Abi = abi; - } - - public void SetDebugInfoPath (string path) - { - DebugInfoPath = GetExistingPath (path); - } - - public void SetConfigPath (string path) - { - ConfigPath = GetExistingPath (path); - } - - string GetExistingPath (string path) - { - if (String.IsNullOrEmpty (path) || !File.Exists (path)) { - return String.Empty; - } - - return path; - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs deleted file mode 100644 index d60af903bc9..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs +++ /dev/null @@ -1,133 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Text; - -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; - -namespace Xamarin.Android.Tasks -{ - class AssemblyStoreGenerator - { - sealed class Store - { - public AssemblyStore Common; - public AssemblyStore Arch; - } - - readonly string archiveAssembliesPrefix; - readonly TaskLoggingHelper log; - - // NOTE: when/if we have parallel BuildApk these should become concurrent collections - readonly Dictionary stores = new Dictionary (StringComparer.Ordinal); - - AssemblyStore indexStore; - - // IDs must be counted per AssemblyStoreGenerator instance because it's possible that a single build will create more than one instance of the class and each time - // the stores must be assigned IDs starting from 0, or there will be errors due to "missing" index store - readonly Dictionary apkIds = new Dictionary (StringComparer.Ordinal); - - // Global assembly index must be restarted from 0 for the same reasons as apkIds above and at the same time it must be unique for each assembly added to **any** - // assembly store, thus we need to keep the state here - AssemblyStoreGlobalIndex globalIndexCounter = new AssemblyStoreGlobalIndex (); - - public AssemblyStoreGenerator (string archiveAssembliesPrefix, TaskLoggingHelper log) - { - if (String.IsNullOrEmpty (archiveAssembliesPrefix)) { - throw new ArgumentException ("must not be null or empty", nameof (archiveAssembliesPrefix)); - } - - this.archiveAssembliesPrefix = archiveAssembliesPrefix; - this.log = log; - } - - public void Add (string apkName, AssemblyStoreAssemblyInfo storeAssembly) - { - if (String.IsNullOrEmpty (apkName)) { - throw new ArgumentException ("must not be null or empty", nameof (apkName)); - } - - Store store; - if (!stores.ContainsKey (apkName)) { - store = new Store { - Common = new CommonAssemblyStore (apkName, archiveAssembliesPrefix, log, GetNextStoreID (apkName), globalIndexCounter), - Arch = new ArchAssemblyStore (apkName, archiveAssembliesPrefix, log, GetNextStoreID (apkName), globalIndexCounter) - }; - - stores.Add (apkName, store); - SetIndexStore (store.Common); - SetIndexStore (store.Arch); - } - - store = stores[apkName]; - if (String.IsNullOrEmpty (storeAssembly.Abi)) { - store.Common.Add (storeAssembly); - } else { - store.Arch.Add (storeAssembly); - } - - void SetIndexStore (AssemblyStore b) - { - if (!b.IsIndexStore) { - return; - } - - if (indexStore != null) { - throw new InvalidOperationException ("Index store already set!"); - } - - indexStore = b; - } - } - - uint GetNextStoreID (string apkName) - { - // NOTE: NOT thread safe, if we ever have parallel runs of BuildApk this operation must either be atomic or protected with a lock - if (!apkIds.ContainsKey (apkName)) { - apkIds.Add (apkName, 0); - } - return apkIds[apkName]++; - } - - public Dictionary> Generate (string outputDirectory) - { - if (stores.Count == 0) { - return null; - } - - if (indexStore == null) { - throw new InvalidOperationException ("Index store not found"); - } - - var globalIndex = new List (); - var ret = new Dictionary> (StringComparer.Ordinal); - string indexStoreApkName = null; - foreach (var kvp in stores) { - string apkName = kvp.Key; - Store store = kvp.Value; - - if (!ret.ContainsKey (apkName)) { - ret.Add (apkName, new List ()); - } - - if (store.Common == indexStore || store.Arch == indexStore) { - indexStoreApkName = apkName; - } - - GenerateStore (store.Common, apkName); - GenerateStore (store.Arch, apkName); - } - - string manifestPath = indexStore.WriteIndex (globalIndex); - ret[indexStoreApkName].Add (manifestPath); - - return ret; - - void GenerateStore (AssemblyStore store, string apkName) - { - store.Generate (outputDirectory, globalIndex, ret[apkName]); - } - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGlobalIndex.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGlobalIndex.cs deleted file mode 100644 index 6ce93f11f9d..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGlobalIndex.cs +++ /dev/null @@ -1,29 +0,0 @@ -namespace Xamarin.Android.Tasks -{ - // This class may seem weird, but it's designed with the specific needs of AssemblyStore instances in mind and also prepared for thread-safe use in the future, should the - // need arise - sealed class AssemblyStoreGlobalIndex - { - uint value = 0; - - public uint Value => value; - - /// - /// Increments the counter and returns its previous value - /// - public uint Increment () - { - uint ret = value++; - return ret; - } - - public void Subtract (uint count) - { - if (value < count) { - return; - } - - value -= count; - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreIndexEntry.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreIndexEntry.cs deleted file mode 100644 index 51d2bd8ca77..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreIndexEntry.cs +++ /dev/null @@ -1,43 +0,0 @@ -using System; -using System.IO.Hashing; -using System.Text; - -namespace Xamarin.Android.Tasks -{ - class AssemblyStoreIndexEntry - { - public string Name { get; } - public uint StoreID { get; } - public uint MappingIndex { get; } - public uint LocalBlobIndex { get; } - - // Hash values must have the same type as they are inside a union in the native code - public ulong NameHash64 { get; } - public ulong NameHash32 { get; } - - public uint DataOffset { get; set; } - public uint DataSize { get; set; } - - public uint DebugDataOffset { get; set; } - public uint DebugDataSize { get; set; } - - public uint ConfigDataOffset { get; set; } - public uint ConfigDataSize { get; set; } - - public AssemblyStoreIndexEntry (string name, uint blobID, uint mappingIndex, uint localBlobIndex) - { - if (String.IsNullOrEmpty (name)) { - throw new ArgumentException ("must not be null or empty", nameof (name)); - } - - Name = name; - StoreID = blobID; - MappingIndex = mappingIndex; - LocalBlobIndex = localBlobIndex; - - byte[] nameBytes = Encoding.UTF8.GetBytes (name); - NameHash32 = XxHash32.HashToUInt32 (nameBytes); - NameHash64 = XxHash64.HashToUInt64 (nameBytes); - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyStore.cs b/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyStore.cs deleted file mode 100644 index 5059ac60a67..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyStore.cs +++ /dev/null @@ -1,36 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Text; - -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; - -namespace Xamarin.Android.Tasks -{ - class CommonAssemblyStore : AssemblyStore - { - readonly List assemblies; - - public CommonAssemblyStore (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log, uint id, AssemblyStoreGlobalIndex globalIndexCounter) - : base (apkName, archiveAssembliesPrefix, log, id, globalIndexCounter) - { - assemblies = new List (); - } - - public override void Add (AssemblyStoreAssemblyInfo blobAssembly) - { - if (!String.IsNullOrEmpty (blobAssembly.Abi)) { - throw new InvalidOperationException ($"Architecture-specific assembly cannot be added to an architecture-agnostic blob ({blobAssembly.FilesystemAssemblyPath})"); - } - - assemblies.Add (blobAssembly); - } - - public override void Generate (string outputDirectory, List globalIndex, List blobPaths) - { - // Always generate this blob, even if there are no assembly entries as this blob contains the global index - Generate (Path.Combine (outputDirectory, $"{ApkName}_{BlobPrefix}{BlobExtension}"), assemblies, globalIndex, blobPaths); - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index e9dd74d4fec..2487980c788 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -327,8 +327,6 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. <_AndroidAotStripLibraries Condition=" '$(_AndroidAotStripLibraries)' == '' And '$(AndroidIncludeDebugSymbols)' != 'true' ">True - false - true True False False @@ -1589,7 +1587,6 @@ because xbuild doesn't support framework reference assemblies. .so;$(AndroidStoreUncompressedFileExtensions) .dex;$(AndroidStoreUncompressedFileExtensions) - .blob;$(AndroidStoreUncompressedFileExtensions) @@ -1655,7 +1652,6 @@ because xbuild doesn't support framework reference assemblies. InstantRunEnabled="$(_InstantRunEnabled)" RuntimeConfigBinFilePath="$(_BinaryRuntimeConfigPath)" UsingAndroidNETSdk="$(UsingAndroidNETSdk)" - UseAssemblyStore="$(AndroidUseAssemblyStore)" EnableMarshalMethods="$(_AndroidUseMarshalMethods)" > @@ -1955,7 +1951,7 @@ because xbuild doesn't support framework reference assemblies. ExcludeFiles="@(AndroidPackagingOptionsExclude)" ZipFlushFilesLimit="$(_ZipFlushFilesLimit)" ZipFlushSizeLimit="$(_ZipFlushSizeLimit)" - UseAssemblyStore="$(AndroidUseAssemblyStore)"> + UseAssemblySharedLibraries="$(_AndroidUseAssemblySharedLibraries)"> + ZipFlushSizeLimit="$(_ZipFlushSizeLimit)"> c have_and_want_debug_symbols = register_debug_symbols && bundled_debug_data != nullptr; } -force_inline void -EmbeddedAssemblies::map_assembly_store (dynamic_local_string const& entry_name, ZipEntryLoadState &state) noexcept -{ - if (number_of_mapped_assembly_stores >= application_config.number_of_assembly_store_files) { - log_fatal (LOG_ASSEMBLY, "Too many assembly stores. Expected at most %u", application_config.number_of_assembly_store_files); - Helpers::abort_application (); - } - - md_mmap_info assembly_store_map = md_mmap_apk_file (state.apk_fd, state.data_offset, state.file_size, entry_name.get ()); - auto header = static_cast(assembly_store_map.area); - - if (header->magic != ASSEMBLY_STORE_MAGIC) { - log_fatal (LOG_ASSEMBLY, "Assembly store '%s' is not a valid Xamarin.Android assembly store file", entry_name.get ()); - Helpers::abort_application (); - } - - if (header->version > ASSEMBLY_STORE_FORMAT_VERSION) { - log_fatal (LOG_ASSEMBLY, "Assembly store '%s' uses format v%u which is not understood by this version of Xamarin.Android", entry_name.get (), header->version); - Helpers::abort_application (); - } - - if (header->store_id >= application_config.number_of_assembly_store_files) { - log_fatal ( - LOG_ASSEMBLY, - "Assembly store '%s' index %u exceeds the number of stores known at application build time, %u", - entry_name.get (), - header->store_id, - application_config.number_of_assembly_store_files - ); - Helpers::abort_application (); - } - - AssemblyStoreRuntimeData &rd = assembly_stores[header->store_id]; - if (rd.data_start != nullptr) { - log_fatal (LOG_ASSEMBLY, "Assembly store '%s' has a duplicate ID (%u)", entry_name.get (), header->store_id); - Helpers::abort_application (); - } - - constexpr size_t header_size = sizeof(AssemblyStoreHeader); - - rd.data_start = static_cast(assembly_store_map.area); - rd.assembly_count = header->local_entry_count; - rd.assemblies = reinterpret_cast(rd.data_start + header_size); - - number_of_found_assemblies += rd.assembly_count; - - if (header->store_id == 0) { - constexpr size_t bundled_assembly_size = sizeof(AssemblyStoreAssemblyDescriptor); - constexpr size_t hash_entry_size = sizeof(AssemblyStoreHashEntry); - - index_assembly_store_header = header; - - size_t bytes_before_hashes = header_size + (bundled_assembly_size * header->local_entry_count); - if constexpr (std::is_same_v) { - assembly_store_hashes = reinterpret_cast(rd.data_start + bytes_before_hashes + (hash_entry_size * header->global_entry_count)); - } else { - assembly_store_hashes = reinterpret_cast(rd.data_start + bytes_before_hashes); - } - } - - number_of_mapped_assembly_stores++; - have_and_want_debug_symbols = register_debug_symbols; -} - force_inline void EmbeddedAssemblies::zip_load_assembly_store_entries (std::vector const& buf, uint32_t num_entries, ZipEntryLoadState &state) noexcept { @@ -309,11 +245,7 @@ EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unus Helpers::abort_application (); } - if (application_config.have_assembly_store) { - zip_load_assembly_store_entries (buf, cd_entries, state); - } else { - zip_load_individual_assembly_entries (buf, cd_entries, should_register, state); - } + zip_load_individual_assembly_entries (buf, cd_entries, should_register, state); } template diff --git a/src/monodroid/jni/embedded-assemblies.cc b/src/monodroid/jni/embedded-assemblies.cc index ff53dc7ea48..4c217d3c297 100644 --- a/src/monodroid/jni/embedded-assemblies.cc +++ b/src/monodroid/jni/embedded-assemblies.cc @@ -98,12 +98,6 @@ EmbeddedAssemblies::get_assembly_data (XamarinAndroidBundledAssembly const& e, u get_assembly_data (e.data, e.data_size, e.name, assembly_data, assembly_data_size); } -force_inline void -EmbeddedAssemblies::get_assembly_data (AssemblyStoreSingleAssemblyRuntimeData const& e, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept -{ - get_assembly_data (e.image_data, e.descriptor->data_size, "", assembly_data, assembly_data_size); -} - template force_inline void EmbeddedAssemblies::map_runtime_file (XamarinAndroidBundledAssembly& file) noexcept @@ -267,144 +261,6 @@ EmbeddedAssemblies::individual_assemblies_open_from_bundles (dynamic_local_strin return nullptr; } -force_inline const AssemblyStoreHashEntry* -EmbeddedAssemblies::find_assembly_store_entry ([[maybe_unused]] hash_t hash, [[maybe_unused]] const AssemblyStoreHashEntry *entries, [[maybe_unused]] size_t entry_count) noexcept -{ -#if !defined (__MINGW32__) || (defined (__MINGW32__) && __GNUC__ >= 10) - hash_t entry_hash; - const AssemblyStoreHashEntry *ret = nullptr; - - while (entry_count > 0) { - ret = entries + (entry_count / 2); - if constexpr (std::is_same_v) { - entry_hash = ret->hash64; - } else { - entry_hash = ret->hash32; - } - auto result = hash <=> entry_hash; - - if (result < 0) { - entry_count /= 2; - } else if (result > 0) { - entries = ret + 1; - entry_count -= entry_count / 2 + 1; - } else { - return ret; - } - } -#endif // ndef __MINGW32__ || (def __MINGW32__ && __GNUC__ >= 10) - - return nullptr; -} - -template -force_inline MonoAssembly* -EmbeddedAssemblies::assembly_store_open_from_bundles (dynamic_local_string& name, TLoaderData loader_data, bool ref_only) noexcept -{ - size_t len = name.length (); - bool have_dll_ext = utils.ends_with (name, SharedConstants::DLL_EXTENSION); - - if (have_dll_ext) { - len -= sizeof(SharedConstants::DLL_EXTENSION) - 1; - } - - hash_t name_hash = xxhash::hash (name.get (), len); - log_debug (LOG_ASSEMBLY, "assembly_store_open_from_bundles: looking for bundled name: '%s' (hash 0x%zx)", name.get (), name_hash); - - const AssemblyStoreHashEntry *hash_entry = find_assembly_store_entry (name_hash, assembly_store_hashes, application_config.number_of_assemblies_in_apk); - if (hash_entry == nullptr) { - log_warn (LOG_ASSEMBLY, "Assembly '%s' (hash 0x%zx) not found", name.get (), name_hash); - return nullptr; - } - - if (hash_entry->mapping_index >= application_config.number_of_assemblies_in_apk) { - log_fatal (LOG_ASSEMBLY, "Invalid assembly index %u, exceeds the maximum index of %u", hash_entry->mapping_index, application_config.number_of_assemblies_in_apk - 1); - Helpers::abort_application (); - } - - AssemblyStoreSingleAssemblyRuntimeData &assembly_runtime_info = assembly_store_bundled_assemblies[hash_entry->mapping_index]; - if (assembly_runtime_info.image_data == nullptr) { - if (hash_entry->store_id >= application_config.number_of_assembly_store_files) { - log_fatal (LOG_ASSEMBLY, "Invalid assembly store ID %u, exceeds the maximum of %u", hash_entry->store_id, application_config.number_of_assembly_store_files - 1); - Helpers::abort_application (); - } - - AssemblyStoreRuntimeData &rd = assembly_stores[hash_entry->store_id]; - if (hash_entry->local_store_index >= rd.assembly_count) { - log_fatal (LOG_ASSEMBLY, "Invalid index %u into local store assembly descriptor array", hash_entry->local_store_index); - Helpers::abort_application (); - } - - AssemblyStoreAssemblyDescriptor *bba = &rd.assemblies[hash_entry->local_store_index]; - - // The assignments here don't need to be atomic, the value will always be the same, so even if two threads - // arrive here at the same time, nothing bad will happen. - assembly_runtime_info.image_data = rd.data_start + bba->data_offset; - assembly_runtime_info.descriptor = bba; - - if (bba->debug_data_offset != 0) { - assembly_runtime_info.debug_info_data = rd.data_start + bba->debug_data_offset; - } -#if !defined (NET) - if (bba->config_data_size != 0) { - assembly_runtime_info.config_data = rd.data_start + bba->config_data_offset; - - // Mono takes ownership of the pointers - mono_register_config_for_assembly ( - utils.string_concat (name.get (), ".dll"), - utils.strdup_new (reinterpret_cast(assembly_runtime_info.config_data)) - ); - } -#endif // NET - - log_debug ( - LOG_ASSEMBLY, - "Mapped: image_data == %p; debug_info_data == %p; config_data == %p; descriptor == %p; data size == %u; debug data size == %u; config data size == %u; name == '%s'", - assembly_runtime_info.image_data, - assembly_runtime_info.debug_info_data, - assembly_runtime_info.config_data, - assembly_runtime_info.descriptor, - assembly_runtime_info.descriptor->data_size, - assembly_runtime_info.descriptor->debug_data_size, - assembly_runtime_info.descriptor->config_data_size, - name.get () - ); - } - - uint8_t *assembly_data; - uint32_t assembly_data_size; - - if (!have_dll_ext) { - // AOT needs this since Mono will form the DSO name by appending the .so suffix to the assembly name passed to - // functions below and `monodroid_dlopen` uses the `.dll.so` extension to check whether we're being asked to load - // the AOTd code for an assembly. - name.append (SharedConstants::DLL_EXTENSION); - } - - get_assembly_data (assembly_runtime_info, assembly_data, assembly_data_size); - MonoImage *image = MonoImageLoader::load (name, loader_data, name_hash, assembly_data, assembly_data_size); - if (image == nullptr) { - log_warn (LOG_ASSEMBLY, "Failed to load MonoImage of '%s'", name.get ()); - return nullptr; - } - - if (have_and_want_debug_symbols && assembly_runtime_info.debug_info_data != nullptr) { - log_debug (LOG_ASSEMBLY, "Registering debug data for assembly '%s'", name.get ()); - mono_debug_open_image_from_memory (image, reinterpret_cast (assembly_runtime_info.debug_info_data), static_cast(assembly_runtime_info.descriptor->debug_data_size)); - } - - MonoImageOpenStatus status; - MonoAssembly *a = mono_assembly_load_from_full (image, name.get (), &status, ref_only); - if (a == nullptr || status != MonoImageOpenStatus::MONO_IMAGE_OK) { - log_warn (LOG_ASSEMBLY, "Failed to load managed assembly '%s'. %s", name.get (), mono_image_strerror (status)); - return nullptr; - } - -#if !defined (NET) - mono_config_for_assembly (image); -#endif - return a; -} // TODO: need to forbid loading assemblies into non-default ALC if they contain marshal method callbacks. // The best way is probably to store the information in the assembly `MonoImage*` cache. We should @@ -423,13 +279,7 @@ EmbeddedAssemblies::open_from_bundles (MonoAssemblyName* aname, TLoaderData load } name.append_c (asmname); - MonoAssembly *a; - if (application_config.have_assembly_store) { - a = assembly_store_open_from_bundles (name, loader_data, ref_only); - } else { - a = individual_assemblies_open_from_bundles (name, loader_data, ref_only); - } - + MonoAssembly *a = individual_assemblies_open_from_bundles (name, loader_data, ref_only); if (a == nullptr) { log_warn (LOG_ASSEMBLY, "open_from_bundles: failed to load assembly %s", name.get ()); } diff --git a/src/monodroid/jni/embedded-assemblies.hh b/src/monodroid/jni/embedded-assemblies.hh index 4b4c3324dae..23a34c37468 100644 --- a/src/monodroid/jni/embedded-assemblies.hh +++ b/src/monodroid/jni/embedded-assemblies.hh @@ -168,15 +168,6 @@ namespace xamarin::android::internal { return need_to_scan_more_apks; } - void ensure_valid_assembly_stores () const noexcept - { - if (!application_config.have_assembly_store) { - return; - } - - abort_unless (index_assembly_store_header != nullptr && assembly_store_hashes != nullptr, "Invalid or incomplete assembly store data"); - } - private: STATIC_IN_ANDROID_RELEASE const char* typemap_managed_to_java (MonoType *type, MonoClass *klass, const uint8_t *mvid) noexcept; STATIC_IN_ANDROID_RELEASE MonoReflectionType* typemap_java_to_managed (hash_t hash, const MonoString *java_type_name) noexcept; @@ -226,7 +217,6 @@ namespace xamarin::android::internal { void set_assembly_data_and_size (uint8_t* source_assembly_data, uint32_t source_assembly_data_size, uint8_t*& dest_assembly_data, uint32_t& dest_assembly_data_size) noexcept; void get_assembly_data (uint8_t *data, uint32_t data_size, const char *name, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept; void get_assembly_data (XamarinAndroidBundledAssembly const& e, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept; - void get_assembly_data (AssemblyStoreSingleAssemblyRuntimeData const& e, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept; void zip_load_entries (int fd, const char *apk_name, monodroid_should_register should_register); void zip_load_individual_assembly_entries (std::vector const& buf, uint32_t num_entries, monodroid_should_register should_register, ZipEntryLoadState &state) noexcept; @@ -307,7 +297,6 @@ namespace xamarin::android::internal { void set_assembly_entry_data (XamarinAndroidBundledAssembly &entry, int apk_fd, uint32_t data_offset, uint32_t data_size, uint32_t prefix_len, uint32_t max_name_size, dynamic_local_string const& entry_name) noexcept; void set_debug_entry_data (XamarinAndroidBundledAssembly &entry, int apk_fd, uint32_t data_offset, uint32_t data_size, uint32_t prefix_len, uint32_t max_name_size, dynamic_local_string const& entry_name) noexcept; void map_assembly_store (dynamic_local_string const& entry_name, ZipEntryLoadState &state) noexcept; - const AssemblyStoreHashEntry* find_assembly_store_entry (hash_t hash, const AssemblyStoreHashEntry *entries, size_t entry_count) noexcept; private: std::vector *bundled_debug_data = nullptr; @@ -331,9 +320,6 @@ namespace xamarin::android::internal { #endif // def NET uint32_t number_of_mapped_assembly_stores = 0; bool need_to_scan_more_apks = true; - - AssemblyStoreHeader *index_assembly_store_header = nullptr; - AssemblyStoreHashEntry *assembly_store_hashes; std::mutex assembly_decompress_mutex; }; } diff --git a/src/monodroid/jni/monodroid-glue.cc b/src/monodroid/jni/monodroid-glue.cc index ab041f4b1c9..dfddcb0bf34 100644 --- a/src/monodroid/jni/monodroid-glue.cc +++ b/src/monodroid/jni/monodroid-glue.cc @@ -395,8 +395,6 @@ MonodroidRuntime::gather_bundled_assemblies (jstring_array_wrapper &runtimeApks, break; } } - - embeddedAssemblies.ensure_valid_assembly_stores (); } #if defined (DEBUG) && !defined (WINDOWS) diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index 55d24e134db..866a566b49a 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -11,9 +11,6 @@ #include "xxhash.hh" static constexpr uint64_t FORMAT_TAG = 0x015E6972616D58; -static constexpr uint32_t ASSEMBLY_STORE_MAGIC = 0x41424158; // 'XABA', little-endian -static constexpr uint32_t ASSEMBLY_STORE_FORMAT_VERSION = 1; // Increase whenever an incompatible change is made to the - // assembly store format static constexpr uint32_t MODULE_MAGIC_NAMES = 0x53544158; // 'XATS', little-endian static constexpr uint32_t MODULE_INDEX_MAGIC = 0x49544158; // 'XATI', little-endian static constexpr uint8_t MODULE_FORMAT_VERSION = 2; // Keep in sync with the value in src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs @@ -147,86 +144,6 @@ MONO_API MONO_API_EXPORT const AssemblyIndexEntry xa_assembly_index[AssemblyCoun MONO_API MONO_API_EXPORT const char xa_assembly_names[AssemblyCount][AssemblyNameLength]; MONO_API MONO_API_EXPORT const char xa_assembly_dso_names[AssemblyCount][SharedLibraryNameLength]; -// -// Assembly store format -// -// The separate hash indices for 32 and 64-bit hashes are required because they will be sorted differently. -// The 'index' field of each of the hashes{32,64} entry points not only into the `assemblies` array in the -// store but also into the `uint8_t*` `assembly_store_bundled_assemblies*` arrays. -// -// This way the `assemblies` array in the store can remain read only, because we write the "mapped" assembly -// pointer somewhere else. Otherwise we'd have to copy the `assemblies` array to a writable area of memory. -// -// Each store has a unique ID assigned, which is an index into an array of pointers to arrays which store -// individual assembly addresses. Only store with ID 0 comes with the hashes32 and hashes64 arrays. This is -// done to make it possible to use a single sorted array to find assemblies insted of each store having its -// own sorted array of hashes, which would require several binary searches instead of just one. -// -// AssemblyStoreHeader header; -// AssemblyStoreAssemblyDescriptor assemblies[header.local_entry_count]; -// AssemblyStoreHashEntry hashes32[header.global_entry_count]; // only in assembly store with ID 0 -// AssemblyStoreHashEntry hashes64[header.global_entry_count]; // only in assembly store with ID 0 -// [DATA] -// - -// -// The structures which are found in the store files must be packed to avoid problems when calculating offsets (runtime -// size of a structure can be different than the real data size) -// -struct [[gnu::packed]] AssemblyStoreHeader final -{ - uint32_t magic; - uint32_t version; - uint32_t local_entry_count; - uint32_t global_entry_count; - uint32_t store_id; -}; - -struct [[gnu::packed]] AssemblyStoreHashEntry final -{ - union { - uint64_t hash64; - uint32_t hash32; - }; - - // Index into the array with pointers to assembly data. - // It **must** be unique across all the stores from all the apks - uint32_t mapping_index; - - // Index into the array with assembly descriptors inside a store - uint32_t local_store_index; - - // Index into the array with assembly store mmap addresses - uint32_t store_id; -}; - -struct [[gnu::packed]] AssemblyStoreAssemblyDescriptor final -{ - uint32_t data_offset; - uint32_t data_size; - - uint32_t debug_data_offset; - uint32_t debug_data_size; - - uint32_t config_data_offset; - uint32_t config_data_size; -}; - -struct AssemblyStoreRuntimeData final -{ - uint8_t *data_start; - uint32_t assembly_count; - AssemblyStoreAssemblyDescriptor *assemblies; -}; - -struct AssemblyStoreSingleAssemblyRuntimeData final -{ - uint8_t *image_data; - uint8_t *debug_info_data; - uint8_t *config_data; - AssemblyStoreAssemblyDescriptor *descriptor; -}; - enum class MonoComponent : uint32_t { None = 0x00, @@ -245,7 +162,6 @@ struct ApplicationConfig bool instant_run_enabled; bool jni_add_native_method_registration_attribute_present; bool have_runtime_config_blob; - bool have_assembly_store; bool marshal_methods_enabled; uint8_t bound_exception_type; uint32_t package_naming_policy; @@ -330,8 +246,6 @@ MONO_API MONO_API_EXPORT const char* const app_system_properties[]; MONO_API MONO_API_EXPORT const char* const mono_aot_mode_name; MONO_API MONO_API_EXPORT XamarinAndroidBundledAssembly bundled_assemblies[]; -MONO_API MONO_API_EXPORT AssemblyStoreSingleAssemblyRuntimeData assembly_store_bundled_assemblies[]; -MONO_API MONO_API_EXPORT AssemblyStoreRuntimeData assembly_stores[]; MONO_API MONO_API_EXPORT DSOCacheEntry dso_cache[]; From a298f00bdbbfc552e1b09ca0390b1d105cbdf5d1 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 6 Oct 2023 22:14:33 +0200 Subject: [PATCH 62/71] Some cleanup, DSO finding improvements, stuff --- ...soft.Android.Sdk.NativeCompilation.targets | 4 -- .../BuildAndLinkStandaloneAssemblyDSOs.cs | 34 ++++++-------- ...GenerateAppAssemblyDSONativeSourceFiles.cs | 18 ++++---- .../Tasks/GeneratePackageManagerJava.cs | 8 ++-- .../PrepareAssemblyStandaloneDSOAbiItems.cs | 11 +++-- .../Utilities/ApplicationConfig.cs | 2 +- ...pplicationConfigNativeAssemblyGenerator.cs | 5 +- .../Utilities/AssemblyDSOGenerator.Classes.cs | 18 +++++++- .../Utilities/AssemblyDSOGenerator.cs | 46 ++++++++++++++----- .../Utilities/DSOAssemblyInfo.cs | 13 +++--- .../Utilities/DSOMetadata.cs | 16 +++++++ .../Xamarin.Android.Common.targets | 2 +- src/monodroid/jni/application_dso_stub.cc | 9 ++-- src/monodroid/jni/embedded-assemblies-zip.cc | 34 +++++++------- src/monodroid/jni/embedded-assemblies.hh | 22 ++++----- src/monodroid/jni/shared-constants.hh | 5 ++ src/monodroid/jni/xamarin-app.hh | 18 +++++++- 17 files changed, 168 insertions(+), 97 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/DSOMetadata.cs diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets index b00c0052b47..9f1aafbfced 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets @@ -140,10 +140,6 @@ Condition=" '$(_AndroidUseAssemblySharedLibraries)' == 'true' " Inputs="@(_ResolvedUserAssemblies);@(_ResolvedFrameworkAssemblies);@(_AndroidResolvedSatellitePaths)" Outputs="@(_AssemblyDSOSourceApplication)"> - "BALSAD"; @@ -99,12 +94,13 @@ protected override void Generate () var dsoItem = new TaskItem (dso.DSOPath); DSOAssemblyInfo dsoInfo = AddAssembly (dso, dsoItem, assemblies); - dsoItem.SetMetadata ("Abi", dso.Abi); - dsoItem.SetMetadata (DSOMetadata.DataSize, MonoAndroidHelper.CultureInvariantToString (dsoInfo.CompressedDataSize == 0 ? dsoInfo.DataSize : dsoInfo.CompressedDataSize)); - dsoItem.SetMetadata (DSOMetadata.UncompressedDataSize, MonoAndroidHelper.CultureInvariantToString (dsoInfo.DataSize)); + dsoItem.SetMetadata (DSOMetadata.Abi, dso.Abi); + dsoItem.SetMetadata (DSOMetadata.AssemblyLoadInfoIndex, MonoAndroidHelper.CultureInvariantToString (dso.AssemblyLoadInfoIndex)); dsoItem.SetMetadata (DSOMetadata.Compressed, dsoInfo.CompressedDataSize == 0 ? "false" : "true"); - dsoItem.SetMetadata (DSOMetadata.OriginalAssemblyPath, dso.OriginalAssemblyPath); + dsoItem.SetMetadata (DSOMetadata.DataSize, MonoAndroidHelper.CultureInvariantToString (dsoInfo.CompressedDataSize == 0 ? dsoInfo.DataSize : dsoInfo.CompressedDataSize)); dsoItem.SetMetadata (DSOMetadata.InputAssemblyPath, dsoInfo.InputFile); + dsoItem.SetMetadata (DSOMetadata.OriginalAssemblyPath, dso.OriginalAssemblyPath); + dsoItem.SetMetadata (DSOMetadata.UncompressedDataSize, MonoAndroidHelper.CultureInvariantToString (dsoInfo.DataSize)); if (!String.IsNullOrEmpty (dso.Culture)) { dsoItem.SetMetadata (DSOMetadata.SatelliteAssemblyCulture, dso.Culture); diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAppAssemblyDSONativeSourceFiles.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAppAssemblyDSONativeSourceFiles.cs index 0fb0f962f98..c25ef5ca20e 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAppAssemblyDSONativeSourceFiles.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAppAssemblyDSONativeSourceFiles.cs @@ -108,30 +108,32 @@ void AddAssemblyToList (AndroidTargetArch arch, DSOAssemblyInfo info) DSOAssemblyInfo MakeStandaloneAssemblyInfo (ITaskItem dsoItem) { - string name = Path.GetFileName (GetRequiredMetadata (BuildAndLinkStandaloneAssemblyDSOs.DSOMetadata.OriginalAssemblyPath)); - string? cultureName = dsoItem.GetMetadata (BuildAndLinkStandaloneAssemblyDSOs.DSOMetadata.SatelliteAssemblyCulture); + string name = Path.GetFileName (GetRequiredMetadata (DSOMetadata.OriginalAssemblyPath)); + string? cultureName = dsoItem.GetMetadata (DSOMetadata.SatelliteAssemblyCulture); if (!String.IsNullOrEmpty (cultureName)) { name = $"{cultureName}/{name}"; } - string inputFile = GetRequiredMetadata (BuildAndLinkStandaloneAssemblyDSOs.DSOMetadata.InputAssemblyPath); - string compressed = GetRequiredMetadata (BuildAndLinkStandaloneAssemblyDSOs.DSOMetadata.Compressed); + string inputFile = GetRequiredMetadata (DSOMetadata.InputAssemblyPath); + string compressed = GetRequiredMetadata (DSOMetadata.Compressed); if (!Boolean.TryParse (compressed, out bool isCompressed)) { - throw new InvalidOperationException ($"Internal error: unable to parse '{compressed}' as a boolean value, from the '{BuildAndLinkStandaloneAssemblyDSOs.DSOMetadata.Compressed}' metadata of item {dsoItem}"); + throw new InvalidOperationException ($"Internal error: unable to parse '{compressed}' as a boolean value, from the '{DSOMetadata.Compressed}' metadata of item {dsoItem}"); } - uint dataSize = GetUintFromRequiredMetadata (BuildAndLinkStandaloneAssemblyDSOs.DSOMetadata.DataSize); + uint dataSize = GetUintFromRequiredMetadata (DSOMetadata.DataSize); uint compressedDataSize; if (!isCompressed) { compressedDataSize = 0; } else { compressedDataSize = dataSize; - dataSize = GetUintFromRequiredMetadata (BuildAndLinkStandaloneAssemblyDSOs.DSOMetadata.UncompressedDataSize); + dataSize = GetUintFromRequiredMetadata (DSOMetadata.UncompressedDataSize); } - return new DSOAssemblyInfo (name, inputFile, dataSize, compressedDataSize, isStandalone: true, Path.GetFileName (dsoItem.ItemSpec)); + return new DSOAssemblyInfo (name, inputFile, dataSize, compressedDataSize, isStandalone: true, Path.GetFileName (dsoItem.ItemSpec)) { + AssemblyLoadInfoIndex = GetUintFromRequiredMetadata (DSOMetadata.AssemblyLoadInfoIndex), + }; string GetRequiredMetadata (string name) { diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index e0a5deca367..4ada1c1d5a5 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -38,6 +38,9 @@ public class GeneratePackageManagerJava : AndroidTask public bool UseAssemblyStore { get; set; } + [Required] + public bool UseAssemblySharedLibraries { get; set; } + [Required] public string OutputDirectory { get; set; } @@ -364,12 +367,9 @@ void AddEnvironment () InstantRunEnabled = InstantRunEnabled, JniAddNativeMethodRegistrationAttributePresent = appConfState != null ? appConfState.JniAddNativeMethodRegistrationAttributePresent : false, HaveRuntimeConfigBlob = haveRuntimeConfigBlob, + HaveStandaloneAssemblyDSOs = UseAssemblySharedLibraries, NumberOfAssembliesInApk = assemblyCount, BundledAssemblyNameWidth = assemblyNameWidth, - NumberOfAssemblyStoresInApks = 2, // Until feature APKs are a thing, we're going to have just two stores in each app - one for arch-agnostic - // and up to 4 other for arch-specific assemblies. Only **one** arch-specific store is ever loaded on the app - // runtime, thus the number 2 here. All architecture specific stores contain assemblies with the same names - // and in the same order. MonoComponents = (MonoComponent)monoComponents, NativeLibraries = uniqueNativeLibraries, AndroidRuntimeJNIEnvToken = android_runtime_jnienv_class_token, diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAssemblyStandaloneDSOAbiItems.cs b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAssemblyStandaloneDSOAbiItems.cs index 395008783ba..ce2b99d58a2 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAssemblyStandaloneDSOAbiItems.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAssemblyStandaloneDSOAbiItems.cs @@ -71,7 +71,7 @@ List PrepareItems () foreach (string abi in seenAbis) { var newItem = MakeTaskItem (assembly, abi, baseName); - newItem.SetMetadata ("SatelliteAssemblyCulture", culture); + newItem.SetMetadata (DSOMetadata.SatelliteAssemblyCulture, culture); sharedLibraries.Add (newItem); } } @@ -90,12 +90,13 @@ ITaskItem MakeTaskItem (ITaskItem assembly, string abi, string baseName) string dsoName = $"libxa{baseName}.{index:X04}.so"; var item = new TaskItem (Path.Combine (SharedLibraryOutputDir, abi, dsoName)); - item.SetMetadata ("Abi", abi); - item.SetMetadata ("InputAssemblyPath", assembly.ItemSpec); - item.SetMetadata ("SourceFileBaseName", baseName); + item.SetMetadata (DSOMetadata.Abi, abi); + item.SetMetadata (DSOMetadata.InputAssemblyPath, assembly.ItemSpec); + item.SetMetadata (DSOMetadata.SourceFileBaseName, baseName); + item.SetMetadata (DSOMetadata.AssemblyLoadInfoIndex, MonoAndroidHelper.CultureInvariantToString (index)); string skipCompression = assembly.GetMetadata ("AndroidSkipCompression"); if (!String.IsNullOrEmpty (skipCompression)) { - item.SetMetadata ("AndroidSkipCompression", skipCompression); + item.SetMetadata (DSOMetadata.AndroidSkipCompression, skipCompression); } return item; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfig.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfig.cs index 3997baf2172..cacea6de2a3 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfig.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfig.cs @@ -32,6 +32,7 @@ sealed class ApplicationConfig public bool instant_run_enabled ; public bool jni_add_native_method_registration_attribute_present; public bool have_runtime_config_blob; + public bool have_standalone_assembly_dsos; public bool marshal_methods_enabled; public byte bound_stream_io_exception_type; public uint package_naming_policy; @@ -39,7 +40,6 @@ sealed class ApplicationConfig public uint system_property_count; public uint number_of_assemblies_in_apk; public uint bundled_assembly_name_width; - public uint number_of_assembly_store_files; public uint number_of_dso_cache_entries; [NativeAssembler (NumberFormat = LLVMIR.LlvmIrVariableNumberFormat.Hexadecimal)] diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs index ad1bae62f48..9c76a44a986 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs @@ -114,9 +114,8 @@ sealed class XamarinAndroidBundledAssembly public bool InstantRunEnabled { get; set; } public bool JniAddNativeMethodRegistrationAttributePresent { get; set; } public bool HaveRuntimeConfigBlob { get; set; } - public bool HaveAssemblyStore { get; set; } + public bool HaveStandaloneAssemblyDSOs { get; set; } public int NumberOfAssembliesInApk { get; set; } - public int NumberOfAssemblyStoresInApks { get; set; } public int BundledAssemblyNameWidth { get; set; } // including the trailing NUL public int AndroidRuntimeJNIEnvToken { get; set; } public int JNIEnvInitializeToken { get; set; } @@ -168,6 +167,7 @@ protected override void Construct (LlvmIrModule module) instant_run_enabled = InstantRunEnabled, jni_add_native_method_registration_attribute_present = JniAddNativeMethodRegistrationAttributePresent, have_runtime_config_blob = HaveRuntimeConfigBlob, + have_standalone_assembly_dsos = HaveStandaloneAssemblyDSOs, marshal_methods_enabled = MarshalMethodsEnabled, bound_stream_io_exception_type = (byte)BoundExceptionType, package_naming_policy = (uint)PackageNamingPolicy, @@ -175,7 +175,6 @@ protected override void Construct (LlvmIrModule module) system_property_count = (uint)(systemProperties == null ? 0 : systemProperties.Count * 2), number_of_assemblies_in_apk = (uint)NumberOfAssembliesInApk, bundled_assembly_name_width = (uint)BundledAssemblyNameWidth, - number_of_assembly_store_files = (uint)NumberOfAssemblyStoresInApks, number_of_dso_cache_entries = (uint)dsoCache.Count, android_runtime_jnienv_class_token = (uint)AndroidRuntimeJNIEnvToken, jnienv_initialize_method_token = (uint)JNIEnvInitializeToken, diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.Classes.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.Classes.cs index 51aa0a0b15b..4079dc41cf3 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.Classes.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.Classes.cs @@ -56,7 +56,12 @@ class AssemblyIndexEntryBase public T name_hash; // Index into the `xa_assemblies` descriptor array - public uint index; + public uint assemblies_index; + + // Index into the `xa_load_info` array. We can't reuse the `assemblies_index` above because the order + // of entries in `xa_load_info` is determined in a different task than that of `xa_assemblies` and it + // also depends on the number of assemblies placed in the standalone DSOs. + public uint load_info_index; // whether hashed name had extension public bool has_extension; @@ -71,6 +76,16 @@ sealed class AssemblyIndexEntry32 : AssemblyIndexEntryBase sealed class AssemblyIndexEntry64 : AssemblyIndexEntryBase {} + // Must be identical to AssemblyLoadInfo in src/monodroid/jni/xamarin-app.hh + sealed class AssemblyLoadInfo + { + // offset into the APK, or 0 if the assembly isn't in a standalone DSO or if the DSOs are extracted to disk at install time + public uint apk_offset; + + // Address at which the assembly data was mmapped + public IntPtr mmap_addr; + }; + // Must be identical to AssembliesConfig in src/monodroid/jni/xamarin-app.hh sealed class AssembliesConfig { @@ -78,6 +93,7 @@ sealed class AssembliesConfig public uint uncompressed_assembly_data_size; public uint assembly_name_length; public uint assembly_count; + public uint assembly_dso_count; public uint shared_library_name_length; }; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs index 8e9fe04c457..85f0a4a83e1 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs @@ -1,5 +1,4 @@ using System; -using System.Collections; using System.Collections.Generic; using System.IO; @@ -11,6 +10,7 @@ namespace Xamarin.Android.Tasks; partial class AssemblyDSOGenerator : LlvmIrComposer { const string XAAssembliesConfigVarName = "xa_assemblies_config"; + const string XAAssembliesLoadInfo = "xa_assemblies_load_info"; const string XAAssembliesVarName = "xa_assemblies"; const string XAAssemblyDSONamesVarName = "xa_assembly_dso_names"; const string XAAssemblyIndexVarName = "xa_assembly_index"; @@ -28,6 +28,7 @@ partial class AssemblyDSOGenerator : LlvmIrComposer StructureInfo? assemblyIndexEntry32StructureInfo; StructureInfo? assemblyIndexEntry64StructureInfo; StructureInfo? assembliesConfigStructureInfo; + StructureInfo? assemblyLoadInfoStructureInfo; public AssemblyDSOGenerator (Dictionary dsoAssemblies) { @@ -102,23 +103,23 @@ void ConstructFastPath (LlvmIrModule module) return; } - int expectedCount = -1; + int expectedAssemblyCount = -1; foreach (var kvp in allAssemblies) { AndroidTargetArch arch = kvp.Key; List infos = kvp.Value; - if (expectedCount < 0) { - expectedCount = infos.Count; + if (expectedAssemblyCount < 0) { + expectedAssemblyCount = infos.Count; } - if (infos.Count != expectedCount) { - throw new InvalidOperationException ($"Collection of assemblies for architecture {arch} has a different number of entries ({infos.Count}) than expected ({expectedCount})"); + if (infos.Count != expectedAssemblyCount) { + throw new InvalidOperationException ($"Collection of assemblies for architecture {arch} has a different number of entries ({infos.Count}) than expected ({expectedAssemblyCount})"); } AddAssemblyData (arch, infos); } - if (expectedCount <= 0) { + if (expectedAssemblyCount <= 0) { ConstructEmptyModule (); return; } @@ -155,6 +156,13 @@ void ConstructFastPath (LlvmIrModule module) }; module.Add (xa_assembly_dso_names); + var xa_assemblies_load_info = new LlvmIrGlobalVariable (typeof(StructureInstance[]), XAAssembliesLoadInfo) { + ArrayItemCount = (ulong)expectedAssemblyCount, + Options = LlvmIrVariableOptions.GlobalWritable, + ZeroInitializeArray = true, + }; + module.Add (xa_assemblies_load_info); + var xa_uncompressed_assembly_data = new LlvmIrGlobalVariable (typeof(byte[]), XAUncompressedAssemblyDataVarName) { Alignment = 4096, // align to page boundary, may make access slightly faster ArrayItemCount = uncompressedAssemblyDataSize, @@ -354,10 +362,19 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) ulong uncompressedOffset = 0; ulong assemblyNameLength = 0; ulong sharedLibraryNameLength = 0; + ulong dso_count = 0; + foreach (DSOAssemblyInfo info in infos) { bool isStandalone = info.IsStandalone; uint inputSize = GetInputSize (info); + if (isStandalone) { + if (!info.AssemblyLoadInfoIndex.HasValue) { + throw new InvalidOperationException ($"Internal error: item for assembly '{info.Name}' is missing the required assembly load index value"); + } + dso_count++; + } + // We need to read each file into a separate array, as it is (theoretically) possible that all the assemblies data will exceed 2GB, // which is the limit of we can allocate (or rent, below) in .NET, per single array. // @@ -417,12 +434,14 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) ulong nameWithoutExtensionHash = EnsureUniqueHash (GetXxHash (nameWithoutExtensionBytes, is64Bit), nameWithoutExtension); uint assemblyIndex = (uint)archState.xa_assemblies.Count - 1; + uint loadInfoIndex = isStandalone ? info.AssemblyLoadInfoIndex.Value : UInt32.MaxValue; if (is64Bit) { var indexEntry = new AssemblyIndexEntry64 { Name = info.Name, name_hash = nameHash, - index = assemblyIndex, + assemblies_index = assemblyIndex, + load_info_index = loadInfoIndex, has_extension = true, is_standalone = isStandalone, }; @@ -431,7 +450,8 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) indexEntry = new AssemblyIndexEntry64 { Name = nameWithoutExtension, name_hash = nameWithoutExtensionHash, - index = assemblyIndex, + assemblies_index = assemblyIndex, + load_info_index = loadInfoIndex, has_extension = false, is_standalone = isStandalone, }; @@ -440,7 +460,8 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) var indexEntry = new AssemblyIndexEntry32 { Name = info.Name, name_hash = (uint)nameHash, - index = assemblyIndex, + assemblies_index = assemblyIndex, + load_info_index = loadInfoIndex, is_standalone = isStandalone, }; archState.xa_assembly_index32.Add (new StructureInstance (assemblyIndexEntry32StructureInfo, indexEntry)); @@ -448,7 +469,8 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) indexEntry = new AssemblyIndexEntry32 { Name = nameWithoutExtension, name_hash = (uint)nameWithoutExtensionHash, - index = assemblyIndex, + assemblies_index = assemblyIndex, + load_info_index = loadInfoIndex, has_extension = false, is_standalone = isStandalone, }; @@ -467,6 +489,7 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) } archState.xa_assemblies_config.assembly_count = (uint)archState.xa_assemblies.Count; + archState.xa_assemblies_config.assembly_dso_count = (uint)dso_count; archState.xa_assemblies_config.input_assembly_data_size = (uint)inputOffset; archState.xa_assemblies_config.uncompressed_assembly_data_size = (uint)uncompressedOffset; @@ -506,5 +529,6 @@ void MapStructures (LlvmIrModule module) assemblyIndexEntry32StructureInfo = module.MapStructure (); assemblyIndexEntry64StructureInfo = module.MapStructure (); assembliesConfigStructureInfo = module.MapStructure (); + assemblyLoadInfoStructureInfo = module.MapStructure (); } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/DSOAssemblyInfo.cs b/src/Xamarin.Android.Build.Tasks/Utilities/DSOAssemblyInfo.cs index eb94b9a9d47..a0a074d4eb0 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/DSOAssemblyInfo.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/DSOAssemblyInfo.cs @@ -7,29 +7,30 @@ class DSOAssemblyInfo /// /// Size of the loadable assembly data (after decompression, if compression is enabled). /// - public uint DataSize { get; } + public uint DataSize { get; } /// /// Size of the compressed assembly data or `0` if assembly is uncompressed. /// - public uint CompressedDataSize { get; } + public uint CompressedDataSize { get; } /// /// The file data comes from, either the original assembly or its compressed copy /// - public string InputFile { get; } + public string InputFile { get; } /// /// Name of the assembly, including culture prefix if it's a satellite assembly. Must include the extension. /// - public string Name { get; } + public string Name { get; } /// /// Indicates whether assembly data is stored in a standalone shared library. /// - public bool IsStandalone { get; } + public bool IsStandalone { get; } - public string? StandaloneDSOName { get; } + public string? StandaloneDSOName { get; } + public uint? AssemblyLoadInfoIndex { get; set; } /// /// is the original assembly name, including culture prefix (e.g. `en_US/`) if it is a diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/DSOMetadata.cs b/src/Xamarin.Android.Build.Tasks/Utilities/DSOMetadata.cs new file mode 100644 index 00000000000..d9f11fb1e4a --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/DSOMetadata.cs @@ -0,0 +1,16 @@ +namespace Xamarin.Android.Tasks; + +public static class DSOMetadata +{ + public const string Abi = "Abi"; + public const string AndroidSkipCompression = "AndroidSkipCompression"; + public const string Compressed = "Compressed"; + public const string DataSize = "DataSize"; + public const string DataSymbolOffset = "DataSymbolOffset"; + public const string InputAssemblyPath = "InputAssemblyPath"; + public const string OriginalAssemblyPath = "OriginalAssemblyPath"; + public const string AssemblyLoadInfoIndex = "AssemblyLoadInfoIndex"; + public const string SatelliteAssemblyCulture = "SatelliteAssemblyCulture"; + public const string SourceFileBaseName = "SourceFileBaseName"; + public const string UncompressedDataSize = "UncompressedDataSize"; +} diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 2487980c788..ed119660f0a 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -1653,7 +1653,7 @@ because xbuild doesn't support framework reference assemblies. RuntimeConfigBinFilePath="$(_BinaryRuntimeConfigPath)" UsingAndroidNETSdk="$(UsingAndroidNETSdk)" EnableMarshalMethods="$(_AndroidUseMarshalMethods)" - > + UseAssemblySharedLibraries="$(_AndroidUseAssemblySharedLibraries)"> diff --git a/src/monodroid/jni/application_dso_stub.cc b/src/monodroid/jni/application_dso_stub.cc index bbc5188cf2e..e301eb790a0 100644 --- a/src/monodroid/jni/application_dso_stub.cc +++ b/src/monodroid/jni/application_dso_stub.cc @@ -45,6 +45,7 @@ const ApplicationConfig application_config = { .instant_run_enabled = false, .jni_add_native_method_registration_attribute_present = false, .have_runtime_config_blob = false, + .have_standalone_assembly_dsos = false, .marshal_methods_enabled = false, .bound_exception_type = 0, // System .package_naming_policy = 0, @@ -52,7 +53,6 @@ const ApplicationConfig application_config = { .system_property_count = 0, .number_of_assemblies_in_apk = 2, .bundled_assembly_name_width = 0, - .number_of_assembly_store_files = 2, .number_of_dso_cache_entries = 2, .android_runtime_jnienv_class_token = 1, .jnienv_initialize_method_token = 2, @@ -194,14 +194,14 @@ const AssemblyEntry xa_assemblies[AssemblyCount] = { const AssemblyIndexEntry xa_assembly_index[AssemblyCount] = { { .name_hash = 11111u, - .index = 0, + .assemblies_index = 0, .has_extension = true, .is_standalone = false, }, { .name_hash = 22222u, - .index = 1, + .assemblies_index = 1, .has_extension = true, .is_standalone = true, }, @@ -222,9 +222,12 @@ const AssembliesConfig xa_assemblies_config = { .uncompressed_assembly_data_size = UncompressedAssemblyDataSize, .assembly_name_length = AssemblyNameLength, .assembly_count = AssemblyCount, + .assembly_dso_count = 2, .shared_library_name_length = SharedLibraryNameLength, }; +AssemblyLoadInfo xa_assemblies_load_info[AssemblyCount]; + void xamarin_app_init ([[maybe_unused]] JNIEnv *env, [[maybe_unused]] get_function_pointer_fn fn) noexcept { // Dummy diff --git a/src/monodroid/jni/embedded-assemblies-zip.cc b/src/monodroid/jni/embedded-assemblies-zip.cc index ce979059056..4d465a2d53b 100644 --- a/src/monodroid/jni/embedded-assemblies-zip.cc +++ b/src/monodroid/jni/embedded-assemblies-zip.cc @@ -1,14 +1,11 @@ #include #include -#include #include -#include #include #include #include "embedded-assemblies.hh" -#include "cpp-util.hh" #include "globals.hh" #include "xamarin-app.hh" @@ -169,18 +166,17 @@ EmbeddedAssemblies::zip_load_individual_assembly_entries (std::vector c have_and_want_debug_symbols = register_debug_symbols && bundled_debug_data != nullptr; } +#if defined(RELEASE) force_inline void -EmbeddedAssemblies::zip_load_assembly_store_entries (std::vector const& buf, uint32_t num_entries, ZipEntryLoadState &state) noexcept +EmbeddedAssemblies::zip_load_standalone_dso_entries (std::vector const& buf, uint32_t num_entries, ZipEntryLoadState &state) noexcept { if (all_required_zip_entries_found ()) { return; } dynamic_local_string entry_name; - bool common_assembly_store_found = false; - bool arch_assembly_store_found = false; - log_debug (LOG_ASSEMBLY, "Looking for assembly stores in APK (common: '%s'; arch-specific: '%s')", assembly_store_common_file_name.data (), assembly_store_arch_file_name.data ()); + log_debug (LOG_ASSEMBLY, "Looking for assembly DSOs in APK, at prefix %s", assembly_dso_prefix); for (size_t i = 0; i < num_entries; i++) { if (all_required_zip_entries_found ()) { need_to_scan_more_apks = false; @@ -192,17 +188,16 @@ EmbeddedAssemblies::zip_load_assembly_store_entries (std::vector const& continue; } - if (!common_assembly_store_found && utils.ends_with (entry_name, assembly_store_common_file_name)) { - common_assembly_store_found = true; - map_assembly_store (entry_name, state); - } + // if (!common_assembly_store_found && utils.ends_with (entry_name, assembly_store_common_file_name)) { + // common_assembly_store_found = true; + // } - if (!arch_assembly_store_found && utils.ends_with (entry_name, assembly_store_arch_file_name)) { - arch_assembly_store_found = true; - map_assembly_store (entry_name, state); - } + // if (!arch_assembly_store_found && utils.ends_with (entry_name, assembly_store_arch_file_name)) { + // arch_assembly_store_found = true; + // } } } +#endif void EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unused]] monodroid_should_register should_register) @@ -245,7 +240,14 @@ EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unus Helpers::abort_application (); } - zip_load_individual_assembly_entries (buf, cd_entries, should_register, state); +#if defined (RELEASE) + if (application_config.have_standalone_assembly_dsos) { + zip_load_standalone_dso_entries (buf, cd_entries, state); + } else +#endif // def RELEASE + { + zip_load_individual_assembly_entries (buf, cd_entries, should_register, state); + } } template diff --git a/src/monodroid/jni/embedded-assemblies.hh b/src/monodroid/jni/embedded-assemblies.hh index 23a34c37468..67e66b51932 100644 --- a/src/monodroid/jni/embedded-assemblies.hh +++ b/src/monodroid/jni/embedded-assemblies.hh @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -96,12 +97,7 @@ namespace xamarin::android::internal { static constexpr off_t ZIP_LOCAL_LEN = 30; static constexpr char assemblies_prefix[] = "assemblies/"; static constexpr char zip_path_separator[] = "/"; - - static constexpr char assembly_store_prefix[] = "assemblies"; - static constexpr char assembly_store_extension[] = ".blob"; - static constexpr auto assembly_store_common_file_name = concat_const ("/", assembly_store_prefix, assembly_store_extension); - static constexpr auto assembly_store_arch_file_name = concat_const ("/", assembly_store_prefix, ".", SharedConstants::android_abi, assembly_store_extension); - + static constexpr auto assembly_dso_prefix = concat_const ("lib/", SharedConstants::android_apk_abi, "/libxa"); #if defined (DEBUG) || !defined (ANDROID) static constexpr char override_typemap_entry_name[] = ".__override__"; @@ -220,11 +216,14 @@ namespace xamarin::android::internal { void zip_load_entries (int fd, const char *apk_name, monodroid_should_register should_register); void zip_load_individual_assembly_entries (std::vector const& buf, uint32_t num_entries, monodroid_should_register should_register, ZipEntryLoadState &state) noexcept; - void zip_load_assembly_store_entries (std::vector const& buf, uint32_t num_entries, ZipEntryLoadState &state) noexcept; bool zip_load_entry_common (size_t entry_index, std::vector const& buf, dynamic_local_string &entry_name, ZipEntryLoadState &state) noexcept; bool zip_read_cd_info (int fd, uint32_t& cd_offset, uint32_t& cd_size, uint16_t& cd_entries); bool zip_adjust_data_offset (int fd, ZipEntryLoadState &state); +#if defined (RELEASE) + void zip_load_standalone_dso_entries (std::vector const& buf, uint32_t num_entries, ZipEntryLoadState &state) noexcept; +#endif // def RELEASE + template bool zip_extract_cd_info (std::array const& buf, uint32_t& cd_offset, uint32_t& cd_size, uint16_t& cd_entries); @@ -269,11 +268,8 @@ namespace xamarin::android::internal { bool all_required_zip_entries_found () const noexcept { return - number_of_mapped_assembly_stores == application_config.number_of_assembly_store_files -#if defined (NET) - && ((application_config.have_runtime_config_blob && runtime_config_blob_found) || !application_config.have_runtime_config_blob) -#endif // NET - ; + number_of_standalone_dsos == xa_assemblies_config.assembly_dso_count && + ((application_config.have_runtime_config_blob && runtime_config_blob_found) || !application_config.have_runtime_config_blob); } static force_inline c_unique_ptr to_utf8 (const MonoString *s) noexcept @@ -318,7 +314,7 @@ namespace xamarin::android::internal { md_mmap_info runtime_config_blob_mmap{}; bool runtime_config_blob_found = false; #endif // def NET - uint32_t number_of_mapped_assembly_stores = 0; + uint32_t number_of_standalone_dsos = 0; bool need_to_scan_more_apks = true; std::mutex assembly_decompress_mutex; }; diff --git a/src/monodroid/jni/shared-constants.hh b/src/monodroid/jni/shared-constants.hh index a88dbcdf569..43890c98621 100644 --- a/src/monodroid/jni/shared-constants.hh +++ b/src/monodroid/jni/shared-constants.hh @@ -1,6 +1,7 @@ #ifndef __SHARED_CONSTANTS_HH #define __SHARED_CONSTANTS_HH +#include #include "cpp-util.hh" namespace xamarin::android::internal @@ -51,15 +52,19 @@ namespace xamarin::android::internal #if __arm__ static constexpr char android_abi[] = "armeabi_v7a"; + static constexpr char android_apk_abi[] = "armeabi_v7a"; static constexpr char runtime_identifier[] = "android-arm"; #elif __aarch64__ static constexpr char android_abi[] = "arm64_v8a"; + static constexpr char android_apk_abi[] { "arm64-v8a" }; static constexpr char runtime_identifier[] = "android-arm64"; #elif __x86_64__ static constexpr char android_abi[] = "x86_64"; + static constexpr char android_apk_abi[] = "x86_64"; static constexpr char runtime_identifier[] = "android-x64"; #elif __i386__ static constexpr char android_abi[] = "x86"; + static constexpr char android_apk_abi[] = "x86"; static constexpr char runtime_identifier[] = "android-x86"; #endif diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index 866a566b49a..60f1c8422ce 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -108,7 +108,12 @@ struct AssemblyIndexEntry xamarin::android::hash_t name_hash; // Index into the `xa_assemblies` descriptor array - uint32_t index; + uint32_t assemblies_index; + + // Index into the `xa_load_info` array. We can't reuse the `assemblies_index` above because the order + // of entries in `xa_load_info` is determined in a different task than that of `xa_assemblies` and it + // also depends on the number of assemblies placed in the standalone DSOs. + uint32_t load_info_index; // whether hashed name had extension bool has_extension; @@ -117,6 +122,13 @@ struct AssemblyIndexEntry bool is_standalone; }; +struct AssemblyLoadInfo +{ + uint32_t apk_offset; // offset into the APK, or 0 if the assembly isn't in a standalone DSO or if the DSOs are + // extracted to disk at install time + void *mmap_addr; // Address at which the assembly data was mmapped +}; + constexpr uint32_t InputAssemblyDataSize = 1024; constexpr uint32_t UncompressedAssemblyDataSize = 2048; constexpr uint32_t AssemblyCount = 2; @@ -129,6 +141,7 @@ struct AssembliesConfig uint32_t uncompressed_assembly_data_size; uint32_t assembly_name_length; uint32_t assembly_count; + uint32_t assembly_dso_count; uint32_t shared_library_name_length; }; @@ -143,6 +156,7 @@ MONO_API MONO_API_EXPORT const AssemblyEntry xa_assemblies[AssemblyCount]; MONO_API MONO_API_EXPORT const AssemblyIndexEntry xa_assembly_index[AssemblyCount]; MONO_API MONO_API_EXPORT const char xa_assembly_names[AssemblyCount][AssemblyNameLength]; MONO_API MONO_API_EXPORT const char xa_assembly_dso_names[AssemblyCount][SharedLibraryNameLength]; +MONO_API MONO_API_EXPORT AssemblyLoadInfo xa_assemblies_load_info[AssemblyCount]; enum class MonoComponent : uint32_t { @@ -162,6 +176,7 @@ struct ApplicationConfig bool instant_run_enabled; bool jni_add_native_method_registration_attribute_present; bool have_runtime_config_blob; + bool have_standalone_assembly_dsos; bool marshal_methods_enabled; uint8_t bound_exception_type; uint32_t package_naming_policy; @@ -169,7 +184,6 @@ struct ApplicationConfig uint32_t system_property_count; uint32_t number_of_assemblies_in_apk; uint32_t bundled_assembly_name_width; - uint32_t number_of_assembly_store_files; uint32_t number_of_dso_cache_entries; uint32_t android_runtime_jnienv_class_token; uint32_t jnienv_initialize_method_token; From 78cb6ae4262ca5bdf863da6909427773e6858950 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 10 Oct 2023 22:09:10 +0200 Subject: [PATCH 63/71] More native work --- .../PrepareAssemblyStandaloneDSOAbiItems.cs | 6 +- src/monodroid/jni/embedded-assemblies-zip.cc | 79 +++++++++++++------ src/monodroid/jni/embedded-assemblies.hh | 17 +++- src/monodroid/jni/monodroid-glue.cc | 10 ++- 4 files changed, 82 insertions(+), 30 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAssemblyStandaloneDSOAbiItems.cs b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAssemblyStandaloneDSOAbiItems.cs index ce2b99d58a2..fddf7585e50 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAssemblyStandaloneDSOAbiItems.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAssemblyStandaloneDSOAbiItems.cs @@ -86,8 +86,10 @@ ITaskItem MakeTaskItem (ITaskItem assembly, string abi, string baseName) assemblyIndexes.Add (baseName, index); } - // the 'xa' infix is to make it harder to produce library names that clash with 3rd party libraries - string dsoName = $"libxa{baseName}.{index:X04}.so"; + // the 'XA' infix is to make it harder to produce library names that clash with 3rd party libraries + // If the infix changes, the `assembly_dso_prefix` constant in src/monodroid/jni/embedded-assemblies.hh must + // be changed as well. + string dsoName = $"libXA{baseName}.{index:X04}.so"; var item = new TaskItem (Path.Combine (SharedLibraryOutputDir, abi, dsoName)); item.SetMetadata (DSOMetadata.Abi, abi); diff --git a/src/monodroid/jni/embedded-assemblies-zip.cc b/src/monodroid/jni/embedded-assemblies-zip.cc index 4d465a2d53b..e3317c89e99 100644 --- a/src/monodroid/jni/embedded-assemblies-zip.cc +++ b/src/monodroid/jni/embedded-assemblies-zip.cc @@ -58,12 +58,18 @@ EmbeddedAssemblies::zip_load_entry_common (size_t entry_index, std::vector 1, "assembly_dso_prefix must be longer than 1 byte"); + + if ( // library location is not overridable, so we can test for it beginning with a hardcoded 'l' + (entry_name.get ()[0] != 'l' || entry_name.length () < assembly_dso_prefix.size () || memcmp (assembly_dso_prefix.data (), entry_name.get (), assembly_dso_prefix.size () - 1) != 0) && + (entry_name.get ()[0] != state.prefix[0] || entry_name.length () < state.prefix_len || memcmp (state.prefix, entry_name.get (), state.prefix_len) != 0) + ) { return false; } + state.location = entry_name.get ()[0] == 'l' ? EntryLocation::Libs : EntryLocation::Assemblies; #if defined (NET) - if (application_config.have_runtime_config_blob && !runtime_config_blob_found) { + if (application_config.have_runtime_config_blob && !runtime_config_blob_found && state.location == EntryLocation::Assemblies) { if (utils.ends_with (entry_name, SharedConstants::RUNTIME_CONFIG_BLOB_NAME)) { runtime_config_blob_found = true; runtime_config_blob_mmap = md_mmap_apk_file (state.apk_fd, state.data_offset, state.file_size, entry_name.get ()); @@ -72,9 +78,9 @@ EmbeddedAssemblies::zip_load_entry_common (size_t entry_index, std::vector c continue; } -#if !defined(NET) - if (utils.ends_with (entry_name, ".config")) { - char *assembly_name = strdup (basename (entry_name.get ())); - // Remove '.config' suffix - *strrchr (assembly_name, '.') = '\0'; - - md_mmap_info map_info = md_mmap_apk_file (state.apk_fd, state.data_offset, state.file_size, entry_name.get ()); - mono_register_config_for_assembly (assembly_name, (const char*)map_info.area); - + if (state.location != EntryLocation::Assemblies || !utils.ends_with (entry_name, SharedConstants::DLL_EXTENSION)) { continue; } -#endif // ndef NET - - if (!utils.ends_with (entry_name, SharedConstants::DLL_EXTENSION)) - continue; #if defined (DEBUG) if (entry_is_overridden) @@ -184,17 +178,54 @@ EmbeddedAssemblies::zip_load_standalone_dso_entries (std::vector const& } bool interesting_entry = zip_load_entry_common (i, buf, entry_name, state); - if (!interesting_entry) { + if (!interesting_entry || state.location != EntryLocation::Libs) { continue; } - // if (!common_assembly_store_found && utils.ends_with (entry_name, assembly_store_common_file_name)) { - // common_assembly_store_found = true; - // } + if (entry_name.length () < assembly_dso_min_length) { + log_warn (LOG_ASSEMBLY, "APK entry '%s' looks like an assembly DSO, but its name is not long enough. Expected at least %zu characters", entry_name.get (), assembly_dso_min_length); + continue; + } + + number_of_found_assemblies++; + + // We have an assembly DSO + log_info (LOG_ASSEMBLY, "Found an assembly DSO: %s; index: %s", entry_name.get (), entry_name.get () + (entry_name.length () - 7)); + + bool valid_hex = true; + auto integer_from_hex_char = [] (dynamic_local_string const& s, size_t pos, bool &is_valid, size_t shift) -> uint16_t + { + uint8_t ch = s[pos]; + if (ch >= '0' && ch <= '9') { + return static_cast((ch - 48) << shift); // 48 is ASCII '0' + } + + if (ch >= 'A' && ch <= 'Z') { + return static_cast((ch - 55) << shift); // ASCII 'A' is 65, and it represents decimal 10 + } + + is_valid = false; + return static_cast(0); + }; + + const size_t index_pos = entry_name.length () - assembly_index_start_offset; + uint16_t index = + integer_from_hex_char (entry_name, index_pos, valid_hex, 12u) | + integer_from_hex_char (entry_name, index_pos + 1, valid_hex, 8u) | + integer_from_hex_char (entry_name, index_pos + 2, valid_hex, 4u) | + integer_from_hex_char (entry_name, index_pos + 3, valid_hex, 0u); + + if (!valid_hex) [[unlikely]] { + log_fatal (LOG_ASSEMBLY, "Unable to determine DSO storage index from '%s'", entry_name.get ()); + Helpers::abort_application (); + } + + if (index >= xa_assemblies_config.assembly_dso_count) [[unlikely]] { + log_fatal (LOG_ASSEMBLY, "Index retrieved from '%s' exceeds the maximum allowed value of %u", entry_name.get (), xa_assemblies_config.assembly_dso_count - 1); + Helpers::abort_application (); + } - // if (!arch_assembly_store_found && utils.ends_with (entry_name, assembly_store_arch_file_name)) { - // arch_assembly_store_found = true; - // } + xa_assemblies_load_info[index].apk_offset = state.data_offset; } } #endif diff --git a/src/monodroid/jni/embedded-assemblies.hh b/src/monodroid/jni/embedded-assemblies.hh index 67e66b51932..44203359f8a 100644 --- a/src/monodroid/jni/embedded-assemblies.hh +++ b/src/monodroid/jni/embedded-assemblies.hh @@ -68,6 +68,12 @@ namespace xamarin::android::internal { #define LoaderData typename #endif + enum class EntryLocation + { + Assemblies, + Libs, + }; + class EmbeddedAssemblies final { struct md_mmap_info { @@ -86,6 +92,7 @@ namespace xamarin::android::internal { uint32_t local_header_offset; uint32_t data_offset; uint32_t file_size; + EntryLocation location; }; private: @@ -97,7 +104,15 @@ namespace xamarin::android::internal { static constexpr off_t ZIP_LOCAL_LEN = 30; static constexpr char assemblies_prefix[] = "assemblies/"; static constexpr char zip_path_separator[] = "/"; - static constexpr auto assembly_dso_prefix = concat_const ("lib/", SharedConstants::android_apk_abi, "/libxa"); + + // if the `libXA` embedded assembly prefix changes here, it must be updated in + // src/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAssemblyStandaloneDSOAbiItems.cs as well + static constexpr auto assembly_dso_prefix = concat_const ("lib/", SharedConstants::android_apk_abi, "/libXA"); + + // This includes the `.0000.so` suffix each assembly DSO must have and at least one name character (in addition + // to the `libXA` prefix) + static constexpr size_t assembly_dso_min_length = assembly_dso_prefix.size () - 1 + 9; + static constexpr size_t assembly_index_start_offset = 7; #if defined (DEBUG) || !defined (ANDROID) static constexpr char override_typemap_entry_name[] = ".__override__"; diff --git a/src/monodroid/jni/monodroid-glue.cc b/src/monodroid/jni/monodroid-glue.cc index dfddcb0bf34..b962dc3f302 100644 --- a/src/monodroid/jni/monodroid-glue.cc +++ b/src/monodroid/jni/monodroid-glue.cc @@ -851,15 +851,19 @@ MonodroidRuntime::create_domain (JNIEnv *env, jstring_array_wrapper &runtimeApks } #endif // def NET - if (user_assemblies_count == 0 && androidSystem.count_override_assemblies () == 0 && !is_running_on_desktop) { + if (user_assemblies_count == 0 && androidSystem.count_override_assemblies () == 0) { #if defined (DEBUG) log_fatal (LOG_DEFAULT, "No assemblies found in '%s' or '%s'. Assuming this is part of Fast Deployment. Exiting...", androidSystem.get_override_dir (0), (AndroidSystem::MAX_OVERRIDES > 1 && androidSystem.get_override_dir (1) != nullptr) ? androidSystem.get_override_dir (1) : ""); #else - log_fatal (LOG_DEFAULT, "No assemblies (or assembly blobs) were found in the application APK file(s)"); + log_fatal (LOG_DEFAULT, "No assemblies (or assembly shared libraries) were found in the application APK file(s)"); +#endif + log_fatal (LOG_DEFAULT, "Make sure that all entries (if any) in the APK directory named `assemblies/` are STORED (not compressed)"); +#if defined (RELEASE) + log_fatal (LOG_DEFAULT, "Make sure that all the shared libraries (if any) found in the APK directory named lib/%s are STORED, or that they are extracted to the filesystem", SharedConstants::android_apk_abi); + log_fatal (LOG_DEFAULT, "Check the 'android:extractNativeLibs=' attribute of the 'application' element in your AndroidManifest.xml file to see whether shared libraries are extracted or not"); #endif - log_fatal (LOG_DEFAULT, "Make sure that all entries in the APK directory named `assemblies/` are STORED (not compressed)"); log_fatal (LOG_DEFAULT, "If Android Gradle Plugin's minification feature is enabled, it is likely all the entries in `assemblies/` are compressed"); Helpers::abort_application (); From db70fa3068373b27a88568a3df40a75fd1978398 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 11 Oct 2023 23:22:41 +0200 Subject: [PATCH 64/71] Fixlets and some progress --- .../PrepareAssemblyStandaloneDSOAbiItems.cs | 3 +- src/monodroid/jni/embedded-assemblies-zip.cc | 30 +++++------ src/monodroid/jni/embedded-assemblies.cc | 54 +++++++++++++++++-- src/monodroid/jni/embedded-assemblies.hh | 5 +- src/monodroid/jni/search.hh | 18 +++++-- 5 files changed, 86 insertions(+), 24 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAssemblyStandaloneDSOAbiItems.cs b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAssemblyStandaloneDSOAbiItems.cs index fddf7585e50..43b16bc8daf 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAssemblyStandaloneDSOAbiItems.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAssemblyStandaloneDSOAbiItems.cs @@ -88,7 +88,8 @@ ITaskItem MakeTaskItem (ITaskItem assembly, string abi, string baseName) // the 'XA' infix is to make it harder to produce library names that clash with 3rd party libraries // If the infix changes, the `assembly_dso_prefix` constant in src/monodroid/jni/embedded-assemblies.hh must - // be changed as well. + // be changed as well. Index must be encoded as a hexadecimal number, without the 0x prefix and using capital + // letters. If this changes then code in src/monodroid/jni/embedded-assemblies-zip.cc must be adjusted accordingly. string dsoName = $"libXA{baseName}.{index:X04}.so"; var item = new TaskItem (Path.Combine (SharedLibraryOutputDir, abi, dsoName)); diff --git a/src/monodroid/jni/embedded-assemblies-zip.cc b/src/monodroid/jni/embedded-assemblies-zip.cc index e3317c89e99..942a1a2d516 100644 --- a/src/monodroid/jni/embedded-assemblies-zip.cc +++ b/src/monodroid/jni/embedded-assemblies-zip.cc @@ -154,7 +154,7 @@ EmbeddedAssemblies::zip_load_individual_assembly_entries (std::vector c set_assembly_entry_data (bundled_assemblies [bundled_assembly_index], state.apk_fd, state.data_offset, state.file_size, state.prefix_len, max_assembly_name_size, entry_name); bundled_assembly_index++; - number_of_found_assemblies = bundled_assembly_index; + number_of_found_assembly_dsos = bundled_assembly_index; } have_and_want_debug_symbols = register_debug_symbols && bundled_debug_data != nullptr; @@ -187,26 +187,26 @@ EmbeddedAssemblies::zip_load_standalone_dso_entries (std::vector const& continue; } - number_of_found_assemblies++; + number_of_found_assembly_dsos++; // We have an assembly DSO log_info (LOG_ASSEMBLY, "Found an assembly DSO: %s; index: %s", entry_name.get (), entry_name.get () + (entry_name.length () - 7)); bool valid_hex = true; auto integer_from_hex_char = [] (dynamic_local_string const& s, size_t pos, bool &is_valid, size_t shift) -> uint16_t - { - uint8_t ch = s[pos]; - if (ch >= '0' && ch <= '9') { - return static_cast((ch - 48) << shift); // 48 is ASCII '0' - } - - if (ch >= 'A' && ch <= 'Z') { - return static_cast((ch - 55) << shift); // ASCII 'A' is 65, and it represents decimal 10 - } - - is_valid = false; - return static_cast(0); - }; + { + uint8_t ch = s[pos]; + if (ch >= '0' && ch <= '9') { + return static_cast((ch - 48) << shift); // 48 is ASCII '0' + } + + if (ch >= 'A' && ch <= 'F') { + return static_cast((ch - 55) << shift); // ASCII 'A' is 65, and it represents decimal 10 + } + + is_valid = false; + return static_cast(0); + }; const size_t index_pos = entry_name.length () - assembly_index_start_offset; uint16_t index = diff --git a/src/monodroid/jni/embedded-assemblies.cc b/src/monodroid/jni/embedded-assemblies.cc index 4c217d3c297..372f9651757 100644 --- a/src/monodroid/jni/embedded-assemblies.cc +++ b/src/monodroid/jni/embedded-assemblies.cc @@ -224,6 +224,43 @@ EmbeddedAssemblies::load_bundled_assembly ( return a; } +#if defined(RELEASE) +template +force_inline MonoAssembly* +EmbeddedAssemblies::standalone_dso_open_from_bundles (dynamic_local_string& name, TLoaderData loader_data) noexcept +{ + hash_t hash = xxhash::hash (name.get (), name.length ()); + log_debug (LOG_ASSEMBLY, "standalone_dso_open_from_bundles: looking for bundled name: '%s', hash 0x%zx", name.get (), hash); + + auto equal = [](AssemblyIndexEntry const& entry, hash_t key) -> bool { return entry.name_hash == key; }; + auto less_than = [](AssemblyIndexEntry const& entry, hash_t key) -> bool { return entry.name_hash < key; }; + ssize_t index = Search::binary_search (hash, xa_assembly_index, xa_assemblies_config.assembly_count); + + if (index == -1) { + log_debug (LOG_ASSEMBLY, "assembly '%s' not found in the DSO assembly index array", name.get ()); + return null; + } + + AssemblyIndexEntry const& entry = xa_assembly_index[index]; + log_debug ( + LOG_ASSEMBLY, + "assembly '%s' found at index %zd; standalone? %s; name recorded at compilation time: '%s'; DSO name: '%s'", + name.get (), + index, + entry.is_standalone ? "yes" : "no", + + // Pointer arithmetics **MUST** be used, because the **runtime** dimensions of the two arrays will always be + // different to the **compile** time ones. We compile against the dummy `libxamarin-app.so` and the + // compiler optimizes code and does pointer arithmetics for the array dimensions in that dummy (via the + // xamarin-app.hh header) + reinterpret_cast(xa_assembly_names) + (entry.assemblies_index * xa_assemblies_config.assembly_name_length), + reinterpret_cast(xa_assembly_dso_names) + (entry.assemblies_index * xa_assemblies_config.shared_library_name_length) + ); + + return nullptr; +} +#endif // def RELEASE + template force_inline MonoAssembly* EmbeddedAssemblies::individual_assemblies_open_from_bundles (dynamic_local_string& name, TLoaderData loader_data, bool ref_only) noexcept @@ -279,7 +316,16 @@ EmbeddedAssemblies::open_from_bundles (MonoAssemblyName* aname, TLoaderData load } name.append_c (asmname); - MonoAssembly *a = individual_assemblies_open_from_bundles (name, loader_data, ref_only); + MonoAssembly *a; +#if defined (RELEASE) + if (application_config.have_standalone_assembly_dsos) { + a = standalone_dso_open_from_bundles (name, loader_data); + } else +#endif // def RELEASE + { + a = individual_assemblies_open_from_bundles (name, loader_data, ref_only); + } + if (a == nullptr) { log_warn (LOG_ASSEMBLY, "open_from_bundles: failed to load assembly %s", name.get ()); } @@ -1018,11 +1064,11 @@ EmbeddedAssemblies::try_load_typemaps_from_directory (const char *path) size_t EmbeddedAssemblies::register_from (const char *apk_file, monodroid_should_register should_register) { - size_t prev = number_of_found_assemblies; + size_t prev = number_of_found_assembly_dsos; gather_bundled_assemblies_from_apk (apk_file, should_register); - log_info (LOG_ASSEMBLY, "Package '%s' contains %i assemblies", apk_file, number_of_found_assemblies - prev); + log_info (LOG_ASSEMBLY, "Package '%s' contains %i assemblies", apk_file, number_of_found_assembly_dsos - prev); - return number_of_found_assemblies; + return number_of_found_assembly_dsos; } diff --git a/src/monodroid/jni/embedded-assemblies.hh b/src/monodroid/jni/embedded-assemblies.hh index 44203359f8a..61bdbe9291a 100644 --- a/src/monodroid/jni/embedded-assemblies.hh +++ b/src/monodroid/jni/embedded-assemblies.hh @@ -185,6 +185,9 @@ namespace xamarin::android::internal { size_t register_from (const char *apk_file, monodroid_should_register should_register); void gather_bundled_assemblies_from_apk (const char* apk, monodroid_should_register should_register); + template + MonoAssembly* standalone_dso_open_from_bundles (dynamic_local_string& name, TLoaderData loader_data) noexcept; + template MonoAssembly* individual_assemblies_open_from_bundles (dynamic_local_string& name, TLoaderData loader_data, bool ref_only) noexcept; @@ -316,7 +319,7 @@ namespace xamarin::android::internal { bool register_debug_symbols; bool have_and_want_debug_symbols; size_t bundled_assembly_index = 0; - size_t number_of_found_assemblies = 0; + size_t number_of_found_assembly_dsos = 0; #if defined (DEBUG) || !defined (ANDROID) TypeMappingInfo *java_to_managed_maps; diff --git a/src/monodroid/jni/search.hh b/src/monodroid/jni/search.hh index 554126d2bee..472330fcfa8 100644 --- a/src/monodroid/jni/search.hh +++ b/src/monodroid/jni/search.hh @@ -11,21 +11,33 @@ namespace xamarin::android::internal { class Search final { public: - force_inline static ssize_t binary_search (hash_t key, const hash_t *arr, size_t n) noexcept + template + force_inline static ssize_t binary_search (hash_t key, const T *arr, size_t n) noexcept { + static_assert (equal != nullptr, "equal is a required templae parameter"); + static_assert (less_than != nullptr, "less_than is a required templae parameter"); + ssize_t left = -1; ssize_t right = static_cast(n); while (right - left > 1) { ssize_t middle = (left + right) >> 1; - if (arr[middle] < key) { + if (less_than (arr[middle], key)) { left = middle; } else { right = middle; } } - return arr[right] == key ? right : -1; + return equal (arr[right], key) ? right : -1; + } + + force_inline static ssize_t binary_search (hash_t key, const hash_t *arr, size_t n) noexcept + { + auto equal = [](hash_t const& entry, hash_t key) -> bool { return entry == key; }; + auto less_than = [](hash_t const& entry, hash_t key) -> bool { return entry < key; }; + + return binary_search (key, arr, n); } force_inline static ptrdiff_t binary_search_branchless (hash_t x, const hash_t *arr, uint32_t len) noexcept From 073c50799a5db7ed026f804f87751d0b3167a487 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 12 Oct 2023 00:10:51 +0200 Subject: [PATCH 65/71] Oopsie --- src/monodroid/jni/embedded-assemblies.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/monodroid/jni/embedded-assemblies.cc b/src/monodroid/jni/embedded-assemblies.cc index 372f9651757..74b4f87cfa7 100644 --- a/src/monodroid/jni/embedded-assemblies.cc +++ b/src/monodroid/jni/embedded-assemblies.cc @@ -238,7 +238,7 @@ EmbeddedAssemblies::standalone_dso_open_from_bundles (dynamic_local_string Date: Thu, 12 Oct 2023 22:55:09 +0200 Subject: [PATCH 66/71] Some experiments Will most likely end up using function pointers instead of virtual dispatch. A bit more cumbersome but more efficient. The goal is to get rid of `if` statements which check, for each assembly load request, whether we should load from the APK or from the filesystem - this is simply wasted time, since we know at the start which one of those choices is correct and that it doesn't change until the app is reinstalled. --- src/monodroid/jni/assembly-data-provider.hh | 79 +++++++++++++++++++++ src/monodroid/jni/embedded-assemblies.cc | 29 ++++++-- src/monodroid/jni/embedded-assemblies.hh | 7 +- src/monodroid/jni/globals.cc | 3 - src/monodroid/jni/globals.hh | 4 -- src/monodroid/jni/monodroid-glue.cc | 1 + src/monodroid/jni/shared-constants.hh | 12 ++++ 7 files changed, 121 insertions(+), 14 deletions(-) create mode 100644 src/monodroid/jni/assembly-data-provider.hh diff --git a/src/monodroid/jni/assembly-data-provider.hh b/src/monodroid/jni/assembly-data-provider.hh new file mode 100644 index 00000000000..c6de7f0d83a --- /dev/null +++ b/src/monodroid/jni/assembly-data-provider.hh @@ -0,0 +1,79 @@ +#if !defined (ASSEMBLY_DATA_PROVIDER_HH) +#define ASSEMBLY_DATA_PROVIDER_HH + +#include "helpers.hh" +#include "logger.hh" +#include "xamarin-app.hh" + +namespace xamarin::android::internal +{ + class AssemblyDataProvider + { + public: + virtual auto get_data (AssemblyEntry const& entry, bool fast_path) noexcept -> uint8_t* = 0; + }; + +#if defined (RELEASE) + namespace detail + { + class SO_AssemblyDataProvider + { + public: + SO_AssemblyDataProvider () = delete; + ~SO_AssemblyDataProvider () = delete; + + static auto get_data_fastpath (AssemblyEntry const& entry) noexcept -> uint8_t* + { + log_debug (LOG_ASSEMBLY, "Loading assembly from libxamarin-app.so, fast path"); + if (entry.uncompressed_data_size == 0) { + log_debug (LOG_ASSEMBLY, "Assembly is not compressed; input offset: %u; data size: %u", entry.input_data_offset, entry.input_data_size); + } else { + log_debug (LOG_ASSEMBLY, "Assembly is compressed; input offset: %u; compressed data size: %u; uncompressed data size: %u; input offset: %u; output offset: %u", + entry.input_data_offset, entry.input_data_size, entry.uncompressed_data_size, entry.input_data_offset, entry.uncompressed_data_offset); + } + + return nullptr; + } + }; + } + + class SO_APK_AssemblyDataProvider final : public AssemblyDataProvider + { + public: + auto get_data (AssemblyEntry const& entry, bool fast_path) noexcept -> uint8_t* override final + { + log_debug (LOG_ASSEMBLY, __PRETTY_FUNCTION__); + if (fast_path) { + return detail::SO_AssemblyDataProvider::get_data_fastpath (entry); + } + log_debug (LOG_ASSEMBLY, "Looking for assembly in the APK"); + + return nullptr; + } + }; + + class SO_FILESYSTEM_AssemblyDataProvider : public AssemblyDataProvider + { + public: + auto get_data (AssemblyEntry const& entry, bool fast_path) noexcept -> uint8_t* override final + { + log_debug (LOG_ASSEMBLY, __PRETTY_FUNCTION__); + log_debug (LOG_ASSEMBLY, "Loading assembly from a standalone DSO"); + log_debug (LOG_ASSEMBLY, "Looking for assembly on the filesystem"); + + return nullptr; + } + }; +#else // def RELEASE + class DLL_APK_AssemblyDataProvider : public AssemblyDataProvider + { + public: + auto get_data (AssemblyEntry const& entry, bool fast_path) noexcept -> uint8_t* override final + { + log_fatal (LOG_ASSEMBLY, "DSO assembly loading not supported in Debug mode"); + Helpers::abort_application (); + } + }; +#endif // ndef RELEASE +} +#endif // ndef ASSEMBLY_DATA_PROVIDER_HH diff --git a/src/monodroid/jni/embedded-assemblies.cc b/src/monodroid/jni/embedded-assemblies.cc index 74b4f87cfa7..22d91d856d7 100644 --- a/src/monodroid/jni/embedded-assemblies.cc +++ b/src/monodroid/jni/embedded-assemblies.cc @@ -39,6 +39,7 @@ #include "startup-aware-lock.hh" #include "timing-internal.hh" #include "search.hh" +#include "assembly-data-provider.hh" using namespace xamarin::android; using namespace xamarin::android::internal; @@ -67,6 +68,20 @@ class MonoGuidString final char *guid = nullptr; }; +void EmbeddedAssemblies::init () noexcept +{ + log_debug (LOG_ASSEMBLY, __PRETTY_FUNCTION__); +#if defined (RELEASE) + if (androidSystem.is_embedded_dso_mode_enabled ()) { + assembly_data_provider = new SO_APK_AssemblyDataProvider; + } else { + assembly_data_provider = new SO_FILESYSTEM_AssemblyDataProvider; + } +#else + assembly_data_provider = new DLL_APK_AssemblyDataProvider; +#endif +} + void EmbeddedAssemblies::set_assemblies_prefix (const char *prefix) { if (assemblies_prefix_override != nullptr) @@ -237,26 +252,30 @@ EmbeddedAssemblies::standalone_dso_open_from_bundles (dynamic_local_string (hash, xa_assembly_index, xa_assemblies_config.assembly_count); if (index == -1) { - log_debug (LOG_ASSEMBLY, "assembly '%s' not found in the DSO assembly index array", name.get ()); + log_warn (LOG_ASSEMBLY, "assembly '%s' not found in the DSO assembly index", name.get ()); return nullptr; } - AssemblyIndexEntry const& entry = xa_assembly_index[index]; + AssemblyIndexEntry const& index_entry = xa_assembly_index[index]; log_debug ( LOG_ASSEMBLY, "assembly '%s' found at index %zd; standalone? %s; name recorded at compilation time: '%s'; DSO name: '%s'", name.get (), index, - entry.is_standalone ? "yes" : "no", + index_entry.is_standalone ? "yes" : "no", // Pointer arithmetics **MUST** be used, because the **runtime** dimensions of the two arrays will always be // different to the **compile** time ones. We compile against the dummy `libxamarin-app.so` and the // compiler optimizes code and does pointer arithmetics for the array dimensions in that dummy (via the // xamarin-app.hh header) - reinterpret_cast(xa_assembly_names) + (entry.assemblies_index * xa_assemblies_config.assembly_name_length), - reinterpret_cast(xa_assembly_dso_names) + (entry.assemblies_index * xa_assemblies_config.shared_library_name_length) + reinterpret_cast(xa_assembly_names) + (index_entry.assemblies_index * xa_assemblies_config.assembly_name_length), + reinterpret_cast(xa_assembly_dso_names) + (index_entry.assemblies_index * xa_assemblies_config.shared_library_name_length) ); + AssemblyEntry const& entry = xa_assemblies[index_entry.assemblies_index]; + const uint8_t *assembly_data = assembly_data_provider->get_data (entry, index_entry.is_standalone); + log_debug (LOG_ASSEMBLY, "assembly_data == %p", assembly_data); + return nullptr; } #endif // def RELEASE diff --git a/src/monodroid/jni/embedded-assemblies.hh b/src/monodroid/jni/embedded-assemblies.hh index 61bdbe9291a..2e0b7a971d8 100644 --- a/src/monodroid/jni/embedded-assemblies.hh +++ b/src/monodroid/jni/embedded-assemblies.hh @@ -19,6 +19,8 @@ #include #endif +#include "android-system.hh" +#include "assembly-data-provider.hh" #include "strings.hh" #include "xamarin-app.hh" #include "cpp-util.hh" @@ -123,10 +125,10 @@ namespace xamarin::android::internal { using monodroid_should_register = bool (*)(const char *filename); public: -#if defined (RELEASE) && defined (ANDROID) EmbeddedAssemblies () noexcept {} -#endif // def RELEASE && def ANDROID + + void init () noexcept; #if defined (DEBUG) || !defined (ANDROID) void try_load_typemaps_from_directory (const char *path); @@ -335,6 +337,7 @@ namespace xamarin::android::internal { uint32_t number_of_standalone_dsos = 0; bool need_to_scan_more_apks = true; std::mutex assembly_decompress_mutex; + AssemblyDataProvider* assembly_data_provider; }; } diff --git a/src/monodroid/jni/globals.cc b/src/monodroid/jni/globals.cc index 697ef2d037c..a390e650c0f 100644 --- a/src/monodroid/jni/globals.cc +++ b/src/monodroid/jni/globals.cc @@ -9,7 +9,4 @@ OSBridge osBridge; EmbeddedAssemblies embeddedAssemblies; MonodroidRuntime monodroidRuntime; Timing *timing = nullptr; -#ifndef ANDROID -DesignerAssemblies designerAssemblies; -#endif Debug debug; diff --git a/src/monodroid/jni/globals.hh b/src/monodroid/jni/globals.hh index e163ffc2e00..17c9034a562 100644 --- a/src/monodroid/jni/globals.hh +++ b/src/monodroid/jni/globals.hh @@ -19,8 +19,4 @@ extern xamarin::android::internal::EmbeddedAssemblies embeddedAssemblies; extern xamarin::android::internal::MonodroidRuntime monodroidRuntime; extern xamarin::android::Timing *timing; -#ifndef ANDROID -extern xamarin::android::internal::DesignerAssemblies designerAssemblies; -#endif - #endif // !__GLOBALS_H diff --git a/src/monodroid/jni/monodroid-glue.cc b/src/monodroid/jni/monodroid-glue.cc index b9df6550b2a..69a1e1a9553 100644 --- a/src/monodroid/jni/monodroid-glue.cc +++ b/src/monodroid/jni/monodroid-glue.cc @@ -2158,6 +2158,7 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl android_api_level = apiLevel; androidSystem.detect_embedded_dso_mode (applicationDirs); + embeddedAssemblies.init (); // **MUST** be called after androidSystem.detect_embedded_dso_mode! androidSystem.set_running_in_emulator (isEmulator); java_TimeZone = utils.get_class_from_runtime_field (env, klass, "java_util_TimeZone", true); diff --git a/src/monodroid/jni/shared-constants.hh b/src/monodroid/jni/shared-constants.hh index 43890c98621..b1fdc5189f3 100644 --- a/src/monodroid/jni/shared-constants.hh +++ b/src/monodroid/jni/shared-constants.hh @@ -18,6 +18,18 @@ namespace xamarin::android::internal class SharedConstants { public: +#if defined (RELEASE) + static constexpr bool is_release = true; +#else + static constexpr bool is_release = false; +#endif + +#if defined (RELEASE) + static constexpr bool is_debug = false; +#else + static constexpr bool is_debug = true; +#endif + #if defined (NET) static constexpr char MONO_ANDROID_RUNTIME_ASSEMBLY_NAME[] = "Mono.Android.Runtime"; #endif From 69dc249abb6c0ee0b7fa5d8fdc3411f6dafa3ebf Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 13 Oct 2023 21:54:45 +0200 Subject: [PATCH 67/71] Use function pointers instead of classes, faster Some other experiments turned out not to fit the bill, good old pointers win hands down. Tiny performance improvements and code rearranging --- src/monodroid/jni/assembly-data-provider.hh | 38 ++++++++++--------- src/monodroid/jni/basic-android-system.cc | 18 +-------- src/monodroid/jni/basic-android-system.hh | 39 +++++++++++++------- src/monodroid/jni/basic-utilities.cc | 4 +- src/monodroid/jni/basic-utilities.hh | 4 +- src/monodroid/jni/debug-app-helper.cc | 5 ++- src/monodroid/jni/embedded-assemblies.cc | 12 +++--- src/monodroid/jni/embedded-assemblies.hh | 25 ++++++++++++- src/monodroid/jni/logger.cc | 10 ++--- src/monodroid/jni/logger.hh | 11 +++++- src/monodroid/jni/monodroid-glue-internal.hh | 2 +- src/monodroid/jni/monodroid-glue.cc | 18 ++++----- src/monodroid/jni/strings.hh | 10 +++++ 13 files changed, 116 insertions(+), 80 deletions(-) diff --git a/src/monodroid/jni/assembly-data-provider.hh b/src/monodroid/jni/assembly-data-provider.hh index c6de7f0d83a..45c18162b5f 100644 --- a/src/monodroid/jni/assembly-data-provider.hh +++ b/src/monodroid/jni/assembly-data-provider.hh @@ -7,10 +7,10 @@ namespace xamarin::android::internal { - class AssemblyDataProvider + struct AssemblyData { - public: - virtual auto get_data (AssemblyEntry const& entry, bool fast_path) noexcept -> uint8_t* = 0; + const uint8_t *data; + const uint32_t size; }; #if defined (RELEASE) @@ -22,7 +22,8 @@ namespace xamarin::android::internal SO_AssemblyDataProvider () = delete; ~SO_AssemblyDataProvider () = delete; - static auto get_data_fastpath (AssemblyEntry const& entry) noexcept -> uint8_t* + force_inline + static auto get_data_fastpath (AssemblyEntry const& entry) noexcept -> const AssemblyData { log_debug (LOG_ASSEMBLY, "Loading assembly from libxamarin-app.so, fast path"); if (entry.uncompressed_data_size == 0) { @@ -32,47 +33,48 @@ namespace xamarin::android::internal entry.input_data_offset, entry.input_data_size, entry.uncompressed_data_size, entry.input_data_offset, entry.uncompressed_data_offset); } - return nullptr; + return {nullptr, 0}; } }; } - class SO_APK_AssemblyDataProvider final : public AssemblyDataProvider + class SO_APK_AssemblyDataProvider { public: - auto get_data (AssemblyEntry const& entry, bool fast_path) noexcept -> uint8_t* override final + force_inline + static auto get_data (AssemblyEntry const& entry, bool standalone) noexcept -> const AssemblyData { log_debug (LOG_ASSEMBLY, __PRETTY_FUNCTION__); - if (fast_path) { + if (!standalone) { return detail::SO_AssemblyDataProvider::get_data_fastpath (entry); } log_debug (LOG_ASSEMBLY, "Looking for assembly in the APK"); - return nullptr; + return {nullptr, 0}; } }; - class SO_FILESYSTEM_AssemblyDataProvider : public AssemblyDataProvider + class SO_FILESYSTEM_AssemblyDataProvider { public: - auto get_data (AssemblyEntry const& entry, bool fast_path) noexcept -> uint8_t* override final + force_inline + static auto get_data (AssemblyEntry const& entry, bool standalone) noexcept -> const AssemblyData { log_debug (LOG_ASSEMBLY, __PRETTY_FUNCTION__); + if (!standalone) { + return detail::SO_AssemblyDataProvider::get_data_fastpath (entry); + } + log_debug (LOG_ASSEMBLY, "Loading assembly from a standalone DSO"); log_debug (LOG_ASSEMBLY, "Looking for assembly on the filesystem"); - return nullptr; + return {nullptr, 0}; } }; #else // def RELEASE - class DLL_APK_AssemblyDataProvider : public AssemblyDataProvider + class DLL_APK_AssemblyDataProvider { public: - auto get_data (AssemblyEntry const& entry, bool fast_path) noexcept -> uint8_t* override final - { - log_fatal (LOG_ASSEMBLY, "DSO assembly loading not supported in Debug mode"); - Helpers::abort_application (); - } }; #endif // ndef RELEASE } diff --git a/src/monodroid/jni/basic-android-system.cc b/src/monodroid/jni/basic-android-system.cc index db163b72206..982bcc17565 100644 --- a/src/monodroid/jni/basic-android-system.cc +++ b/src/monodroid/jni/basic-android-system.cc @@ -3,6 +3,7 @@ #include "basic-android-system.hh" #include "cpp-util.hh" #include "globals.hh" +#include "monodroid-glue-internal.hh" using namespace xamarin::android; using namespace xamarin::android::internal; @@ -12,25 +13,10 @@ const char **BasicAndroidSystem::app_lib_directories; size_t BasicAndroidSystem::app_lib_directories_size = 0; const char* BasicAndroidSystem::built_for_abi_name = nullptr; -void -BasicAndroidSystem::detect_embedded_dso_mode (jstring_array_wrapper& appDirs) noexcept -{ - // appDirs[SharedConstants::APP_DIRS_DATA_DIR_INDEX] points to the native library directory - std::unique_ptr libmonodroid_path {utils.path_combine (appDirs[SharedConstants::APP_DIRS_DATA_DIR_INDEX].get_cstr (), "libmonodroid.so")}; - log_debug (LOG_ASSEMBLY, "Checking if libmonodroid was unpacked to %s", libmonodroid_path.get ()); - if (!utils.file_exists (libmonodroid_path.get ())) { - log_debug (LOG_ASSEMBLY, "%s not found, assuming application/android:extractNativeLibs == false", libmonodroid_path.get ()); - set_embedded_dso_mode_enabled (true); - } else { - log_debug (LOG_ASSEMBLY, "Native libs extracted to %s, assuming application/android:extractNativeLibs == true", appDirs[SharedConstants::APP_DIRS_DATA_DIR_INDEX].get_cstr ()); - set_embedded_dso_mode_enabled (false); - } -} - void BasicAndroidSystem::setup_app_library_directories (jstring_array_wrapper& runtimeApks, jstring_array_wrapper& appDirs, bool have_split_apks) { - if (!is_embedded_dso_mode_enabled ()) { + if (!BasicAndroidSystem::is_embedded_dso_mode_enabled ()) { log_debug (LOG_DEFAULT, "Setting up for DSO lookup in app data directories"); BasicAndroidSystem::app_lib_directories_size = 1; BasicAndroidSystem::app_lib_directories = new const char*[app_lib_directories_size](); diff --git a/src/monodroid/jni/basic-android-system.hh b/src/monodroid/jni/basic-android-system.hh index a12ba47fa1d..c04233c9649 100644 --- a/src/monodroid/jni/basic-android-system.hh +++ b/src/monodroid/jni/basic-android-system.hh @@ -6,6 +6,8 @@ #include "cpu-arch.hh" #include "jni-wrappers.hh" +#include "strings.hh" +#include "basic-utilities.hh" namespace xamarin::android::internal { @@ -60,6 +62,29 @@ namespace xamarin::android::internal public: void setup_app_library_directories (jstring_array_wrapper& runtimeApks, jstring_array_wrapper& appDirs, bool have_split_apks); + force_inline + static void detect_embedded_dso_mode (jstring_array_wrapper& appDirs) noexcept + { + // appDirs[SharedConstants::APP_DIRS_DATA_DIR_INDEX] points to the native library directory + dynamic_local_string libmonodroid_path; + jstring_wrapper &libdir = appDirs[SharedConstants::APP_DIRS_DATA_DIR_INDEX]; + libmonodroid_path.append (libdir.get_cstr ()).append (MONODROID_PATH_SEPARATOR).append ("libmonodroid.so"); + + log_debug (LOG_ASSEMBLY, "Checking if libmonodroid was unpacked to %s", libmonodroid_path.get ()); + if (!BasicUtilities::file_exists (libmonodroid_path.get ())) { + log_debug (LOG_ASSEMBLY, "%s not found, assuming application/android:extractNativeLibs == false", libmonodroid_path.get ()); + embedded_dso_mode_enabled = true; + } else { + log_debug (LOG_ASSEMBLY, "Native libs extracted to %s, assuming application/android:extractNativeLibs == true", libdir.get_cstr ()); + embedded_dso_mode_enabled = false; + } + } + + static bool is_embedded_dso_mode_enabled () noexcept + { + return embedded_dso_mode_enabled; + } + const char* get_override_dir (size_t index) const { if (index >= MAX_OVERRIDES) @@ -76,13 +101,6 @@ namespace xamarin::android::internal override_dirs [index] = const_cast (dir); } - bool is_embedded_dso_mode_enabled () const - { - return embedded_dso_mode_enabled; - } - - void detect_embedded_dso_mode (jstring_array_wrapper& appDirs) noexcept; - char *get_runtime_libdir () const { return runtime_libdir; @@ -111,15 +129,10 @@ namespace xamarin::android::internal void setup_apk_directories (unsigned short running_on_cpu, jstring_array_wrapper &runtimeApks, bool have_split_apks) noexcept; char* determine_primary_override_dir (jstring_wrapper &home); - void set_embedded_dso_mode_enabled (bool yesno) noexcept - { - embedded_dso_mode_enabled = yesno; - } - private: - bool embedded_dso_mode_enabled = false; char *runtime_libdir = nullptr; char *primary_override_dir = nullptr; + inline static bool embedded_dso_mode_enabled = false; }; } #endif // !__BASIC_ANDROID_SYSTEM_HH diff --git a/src/monodroid/jni/basic-utilities.cc b/src/monodroid/jni/basic-utilities.cc index adf67f7a9c1..11aef82956a 100644 --- a/src/monodroid/jni/basic-utilities.cc +++ b/src/monodroid/jni/basic-utilities.cc @@ -117,7 +117,7 @@ BasicUtilities::set_user_executable ([[maybe_unused]] const char *path) } bool -BasicUtilities::file_exists (const char *file) +BasicUtilities::file_exists (const char *file) noexcept { monodroid_stat_t s; if (monodroid_stat (file, &s) == 0 && (s.st_mode & S_IFMT) == S_IFREG) @@ -221,7 +221,7 @@ BasicUtilities::monodroid_fopen (const char *filename, const char *mode) } int -BasicUtilities::monodroid_stat (const char *path, monodroid_stat_t *s) +BasicUtilities::monodroid_stat (const char *path, monodroid_stat_t *s) noexcept { int result; diff --git a/src/monodroid/jni/basic-utilities.hh b/src/monodroid/jni/basic-utilities.hh index d7b67d83014..5094aa9780f 100644 --- a/src/monodroid/jni/basic-utilities.hh +++ b/src/monodroid/jni/basic-utilities.hh @@ -27,7 +27,7 @@ namespace xamarin::android { public: FILE *monodroid_fopen (const char* filename, const char* mode); - int monodroid_stat (const char *path, monodroid_stat_t *s); + static int monodroid_stat (const char *path, monodroid_stat_t *s) noexcept; monodroid_dir_t *monodroid_opendir (const char *filename); int monodroid_closedir (monodroid_dir_t *dirp); int monodroid_dirent_hasextension (monodroid_dirent_t *e, const char *extension); @@ -40,7 +40,7 @@ namespace xamarin::android int create_directory (const char *pathname, mode_t mode); void set_world_accessable (const char *path); void set_user_executable (const char *path); - bool file_exists (const char *file); + static bool file_exists (const char *file) noexcept; bool directory_exists (const char *directory); bool file_copy (const char *to, const char *from); diff --git a/src/monodroid/jni/debug-app-helper.cc b/src/monodroid/jni/debug-app-helper.cc index e79c62b238d..17613725dd1 100644 --- a/src/monodroid/jni/debug-app-helper.cc +++ b/src/monodroid/jni/debug-app-helper.cc @@ -15,6 +15,7 @@ #include "debug-app-helper.hh" #include "shared-constants.hh" #include "jni-wrappers.hh" +#include "monodroid-glue-internal.hh" using namespace xamarin::android; using namespace xamarin::android::internal; @@ -74,7 +75,7 @@ Java_mono_android_DebugRuntime_init (JNIEnv *env, [[maybe_unused]] jclass klass, jstring_array_wrapper applicationDirs (env, appDirs); jstring_array_wrapper runtimeApks (env, runtimeApksJava); - androidSystem.detect_embedded_dso_mode (applicationDirs); + BasicAndroidSystem::detect_embedded_dso_mode (applicationDirs); androidSystem.set_primary_override_dir (applicationDirs [0]); androidSystem.set_override_dir (0, androidSystem.get_primary_override_dir ()); androidSystem.setup_app_library_directories (runtimeApks, applicationDirs, haveSplitApks); @@ -206,7 +207,7 @@ get_libmonosgen_path () // storage location before loading it. copy_native_libraries_to_internal_location (); - if (androidSystem.is_embedded_dso_mode_enabled ()) { + if (BasicAndroidSystem::is_embedded_dso_mode_enabled ()) { return SharedConstants::MONO_SGEN_SO; } diff --git a/src/monodroid/jni/embedded-assemblies.cc b/src/monodroid/jni/embedded-assemblies.cc index 22d91d856d7..ef5994829fc 100644 --- a/src/monodroid/jni/embedded-assemblies.cc +++ b/src/monodroid/jni/embedded-assemblies.cc @@ -72,13 +72,11 @@ void EmbeddedAssemblies::init () noexcept { log_debug (LOG_ASSEMBLY, __PRETTY_FUNCTION__); #if defined (RELEASE) - if (androidSystem.is_embedded_dso_mode_enabled ()) { - assembly_data_provider = new SO_APK_AssemblyDataProvider; + if (BasicAndroidSystem::is_embedded_dso_mode_enabled ()) { + get_assembly_data_dso_impl = get_assembly_data_dso_apk; } else { - assembly_data_provider = new SO_FILESYSTEM_AssemblyDataProvider; + get_assembly_data_dso_impl = get_assembly_data_dso_fs; } -#else - assembly_data_provider = new DLL_APK_AssemblyDataProvider; #endif } @@ -273,8 +271,8 @@ EmbeddedAssemblies::standalone_dso_open_from_bundles (dynamic_local_stringget_data (entry, index_entry.is_standalone); - log_debug (LOG_ASSEMBLY, "assembly_data == %p", assembly_data); + const AssemblyData assembly_data = get_assembly_data_dso (entry, index_entry.is_standalone); + log_debug (LOG_ASSEMBLY, "assembly_data == %p; size == %u", assembly_data.data, assembly_data.size); return nullptr; } diff --git a/src/monodroid/jni/embedded-assemblies.hh b/src/monodroid/jni/embedded-assemblies.hh index 2e0b7a971d8..c720bda379e 100644 --- a/src/monodroid/jni/embedded-assemblies.hh +++ b/src/monodroid/jni/embedded-assemblies.hh @@ -96,7 +96,9 @@ namespace xamarin::android::internal { uint32_t file_size; EntryLocation location; }; - +#if defined (RELEASE) + using get_assembly_data_dso_fn = const AssemblyData (*)(AssemblyEntry const& entry, bool fast_path); +#endif private: static constexpr char ZIP_CENTRAL_MAGIC[] = "PK\1\2"; static constexpr char ZIP_LOCAL_MAGIC[] = "PK\3\4"; @@ -234,6 +236,23 @@ namespace xamarin::android::internal { void get_assembly_data (uint8_t *data, uint32_t data_size, const char *name, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept; void get_assembly_data (XamarinAndroidBundledAssembly const& e, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept; +#if defined (RELEASE) + static const AssemblyData get_assembly_data_dso_apk (AssemblyEntry const& entry, bool standalone) noexcept + { + return SO_APK_AssemblyDataProvider::get_data (entry, standalone); + } + + static const AssemblyData get_assembly_data_dso_fs (AssemblyEntry const& entry, bool standalone) noexcept + { + return SO_FILESYSTEM_AssemblyDataProvider::get_data (entry, standalone); + } + + force_inline + const AssemblyData get_assembly_data_dso (AssemblyEntry const& entry, bool standalone) const noexcept + { + return (*get_assembly_data_dso_impl) (entry, standalone); + } +#endif void zip_load_entries (int fd, const char *apk_name, monodroid_should_register should_register); void zip_load_individual_assembly_entries (std::vector const& buf, uint32_t num_entries, monodroid_should_register should_register, ZipEntryLoadState &state) noexcept; bool zip_load_entry_common (size_t entry_index, std::vector const& buf, dynamic_local_string &entry_name, ZipEntryLoadState &state) noexcept; @@ -337,7 +356,9 @@ namespace xamarin::android::internal { uint32_t number_of_standalone_dsos = 0; bool need_to_scan_more_apks = true; std::mutex assembly_decompress_mutex; - AssemblyDataProvider* assembly_data_provider; +#if defined (RELEASE) + get_assembly_data_dso_fn get_assembly_data_dso_impl = nullptr; +#endif }; } diff --git a/src/monodroid/jni/logger.cc b/src/monodroid/jni/logger.cc index 57f1ea79957..24146e25779 100644 --- a/src/monodroid/jni/logger.cc +++ b/src/monodroid/jni/logger.cc @@ -16,6 +16,7 @@ #include "debug.hh" #include "util.hh" #include "globals.hh" +#include "strings.hh" #undef DO_LOG #define DO_LOG(_level_,_category_,_format_,_args_) \ @@ -145,11 +146,8 @@ set_category (const char (&name)[NameSize], string_segment& arg, unsigned int en } void -init_logging_categories (char*& mono_log_mask, char*& mono_log_level) +init_logging_categories (xamarin::android::internal::dynamic_local_string& mono_log_mask, xamarin::android::internal::dynamic_local_string& mono_log_level) noexcept { - mono_log_mask = nullptr; - mono_log_level = nullptr; - #if !ANDROID log_categories = LOG_DEFAULT; #endif @@ -254,14 +252,14 @@ init_logging_categories (char*& mono_log_mask, char*& mono_log_level) constexpr char MONO_LOG_MASK_ARG[] = "mono_log_mask="; constexpr size_t MONO_LOG_MASK_ARG_LEN = sizeof(MONO_LOG_MASK_ARG) - 1; if (param.starts_with (MONO_LOG_MASK_ARG)) { - mono_log_mask = utils.strdup_new (param, MONO_LOG_MASK_ARG_LEN); + mono_log_mask.append (param, MONO_LOG_MASK_ARG_LEN); continue; } constexpr char MONO_LOG_LEVEL_ARG[] = "mono_log_level="; constexpr size_t MONO_LOG_LEVEL_ARG_LEN = sizeof(MONO_LOG_LEVEL_ARG) - 1; if (param.starts_with (MONO_LOG_LEVEL_ARG)) { - mono_log_level = utils.strdup_new (param, MONO_LOG_LEVEL_ARG_LEN); + mono_log_level.append (param, MONO_LOG_LEVEL_ARG_LEN); continue; } diff --git a/src/monodroid/jni/logger.hh b/src/monodroid/jni/logger.hh index 2f65333751f..88fc0d26a4c 100644 --- a/src/monodroid/jni/logger.hh +++ b/src/monodroid/jni/logger.hh @@ -1,6 +1,9 @@ #ifndef __MONODROID_LOGGER_H__ #define __MONODROID_LOGGER_H__ +#include +#include + #include "java-interop-logger.h" #ifndef ANDROID @@ -17,7 +20,13 @@ typedef enum android_LogPriority { } android_LogPriority; #endif -void init_logging_categories (char*& mono_log_mask, char*& mono_log_level); +namespace xamarin::android::internal +{ + template + class dynamic_local_string; +} + +void init_logging_categories (xamarin::android::internal::dynamic_local_string& mono_log_mask, xamarin::android::internal::dynamic_local_string& mono_log_level) noexcept; void init_reference_logging (const char *override_dir); diff --git a/src/monodroid/jni/monodroid-glue-internal.hh b/src/monodroid/jni/monodroid-glue-internal.hh index 84ac5f5188a..219be96f349 100644 --- a/src/monodroid/jni/monodroid-glue-internal.hh +++ b/src/monodroid/jni/monodroid-glue-internal.hh @@ -254,7 +254,7 @@ namespace xamarin::android::internal static void mono_log_standard_streams_handler (const char *str, mono_bool is_stdout); // A reference to unique_ptr is not the best practice ever, but it's faster this way - void setup_mono_tracing (std::unique_ptr const& mono_log_mask, bool have_log_assembly, bool have_log_gc); + void setup_mono_tracing (dynamic_local_string const& mono_log_mask, bool have_log_assembly, bool have_log_gc); void install_logging_handlers (); #endif // def ANDROID diff --git a/src/monodroid/jni/monodroid-glue.cc b/src/monodroid/jni/monodroid-glue.cc index 69a1e1a9553..da55050624d 100644 --- a/src/monodroid/jni/monodroid-glue.cc +++ b/src/monodroid/jni/monodroid-glue.cc @@ -2025,7 +2025,7 @@ MonodroidRuntime::get_my_location (bool remove_file_name) #if defined (ANDROID) force_inline void -MonodroidRuntime::setup_mono_tracing (std::unique_ptr const& mono_log_mask, bool have_log_assembly, bool have_log_gc) +MonodroidRuntime::setup_mono_tracing (dynamic_local_string const& mono_log_mask, bool have_log_assembly, bool have_log_gc) { constexpr char MASK_ASM[] = "asm"; constexpr size_t MASK_ASM_LEN = sizeof(MASK_ASM) - 1; @@ -2036,7 +2036,7 @@ MonodroidRuntime::setup_mono_tracing (std::unique_ptr const& mono_log_ma constexpr char COMMA[] = ","; dynamic_local_string log_mask; - if (mono_log_mask == nullptr || *mono_log_mask.get () == '\0') { + if (mono_log_mask.empty () || *mono_log_mask.get () == '\0') { if (have_log_assembly) { log_mask.append (MASK_ASM); log_mask.append (COMMA); @@ -2116,13 +2116,10 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl jobject loader, jobjectArray assembliesJava, jint apiLevel, jboolean isEmulator, jboolean haveSplitApks) { - char *mono_log_mask_raw = nullptr; - char *mono_log_level_raw = nullptr; + dynamic_local_string mono_log_mask; + dynamic_local_string mono_log_level; - init_logging_categories (mono_log_mask_raw, mono_log_level_raw); - - std::unique_ptr mono_log_mask (mono_log_mask_raw); - std::unique_ptr mono_log_level (mono_log_level_raw); + init_logging_categories (mono_log_mask, mono_log_level); // If fast logging is disabled, log messages immediately FastTiming::initialize ((log_timing_categories & LOG_TIMING_FAST_BARE) == 0); @@ -2136,6 +2133,8 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl jstring_array_wrapper applicationDirs (env, appDirs); jstring_wrapper &home = applicationDirs[SharedConstants::APP_DIRS_FILES_DIR_INDEX]; + BasicAndroidSystem::detect_embedded_dso_mode (applicationDirs); + #if defined (NET) mono_opt_aot_lazy_assembly_load = application_config.aot_lazy_load ? TRUE : FALSE; @@ -2157,7 +2156,6 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl #endif // def NET android_api_level = apiLevel; - androidSystem.detect_embedded_dso_mode (applicationDirs); embeddedAssemblies.init (); // **MUST** be called after androidSystem.detect_embedded_dso_mode! androidSystem.set_running_in_emulator (isEmulator); @@ -2192,7 +2190,7 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl bool have_log_assembly = (log_categories & LOG_ASSEMBLY) != 0; bool have_log_gc = (log_categories & LOG_GC) != 0; - if (mono_log_level == nullptr || *mono_log_level.get () == '\0') { + if (mono_log_level.empty () || *mono_log_level.get () == '\0') { mono_trace_set_level_string ((have_log_assembly || have_log_gc) ? "info" : "error"); } else { mono_trace_set_level_string (mono_log_level.get ()); diff --git a/src/monodroid/jni/strings.hh b/src/monodroid/jni/strings.hh index bf41b519b44..13a64467b5d 100644 --- a/src/monodroid/jni/strings.hh +++ b/src/monodroid/jni/strings.hh @@ -444,6 +444,11 @@ namespace xamarin::android::internal return *this; } + force_inline string_base& append (const TChar *s) noexcept + { + return append (s, strlen (s)); + } + template force_inline string_base& append (internal::string_base const& str) noexcept { @@ -456,6 +461,11 @@ namespace xamarin::android::internal return append (s, Size - 1); } + force_inline string_base& append (string_segment const& s, size_t start_pos = 0) noexcept + { + return append (s.start () + start_pos); + } + force_inline string_base& append_c (const char *s) noexcept { if (s == nullptr) From d65392aeb11d2654178993f5ca8dd7bd7fc0fd68 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 16 Oct 2023 22:28:39 +0200 Subject: [PATCH 68/71] Make compression optional on per-assembly basis Experimented with various ideas to improve performance, none of which really panned out as the results were very unstable. --- ...soft.Android.Sdk.NativeCompilation.targets | 5 +++- .../AssemblyNativeSourceGenerationTask.cs | 23 +++++++++++++++++++ .../BuildAndLinkStandaloneAssemblyDSOs.cs | 22 +++++++++++++++--- ...GenerateAppAssemblyDSONativeSourceFiles.cs | 2 +- .../PrepareAssemblyStandaloneDSOAbiItems.cs | 17 +++++++------- 5 files changed, 56 insertions(+), 13 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets index 9f1aafbfced..ea456840df0 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets @@ -67,6 +67,9 @@ <_DSOFastPathAssembly Include="System.Private.CoreLib.dll" /> <_DSOFastPathAssembly Include="System.Runtime.dll" /> <_DSOFastPathAssembly Include="$(AssemblyName).dll" /> + <_DSOFastPathAssembly Include="$(_DesignerAssemblyName).dll"> + true + diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/AssemblyNativeSourceGenerationTask.cs b/src/Xamarin.Android.Build.Tasks/Tasks/AssemblyNativeSourceGenerationTask.cs index 0b115e960eb..942c257bee4 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/AssemblyNativeSourceGenerationTask.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/AssemblyNativeSourceGenerationTask.cs @@ -56,8 +56,31 @@ public override bool RunTask () return !Log.HasLoggedErrors; } + protected static bool ShouldSkipCompression (ITaskItem item) + { + string val = item.GetMetadata (DSOMetadata.AndroidSkipCompression); + if (String.IsNullOrEmpty (val)) { + return false; + } + + if (!Boolean.TryParse (val, out bool skipCompression)) { + throw new InvalidOperationException ($"Internal error: unable to parse '{val}' as a boolean value, in item '{item.ItemSpec}', from the '{DSOMetadata.AndroidSkipCompression}' metadata"); + } + + return skipCompression; + } + protected CompressionResult Compress (ITaskItem assembly) { + if (ShouldSkipCompression (assembly)) { + return new CompressionResult { + Compressed = false, + CompressedSize = 0, + OutputFile = assembly.ItemSpec, + InputFileInfo = new FileInfo (assembly.ItemSpec) + }; + } + return Compress (assembly.ItemSpec, assembly.GetMetadata ("DestinationSubDirectory")); } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildAndLinkStandaloneAssemblyDSOs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildAndLinkStandaloneAssemblyDSOs.cs index 66ab973027c..221b533ff78 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildAndLinkStandaloneAssemblyDSOs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildAndLinkStandaloneAssemblyDSOs.cs @@ -20,6 +20,7 @@ sealed class TargetDSO public readonly string SourceFileBaseName; public readonly string DSOPath; public readonly string? Culture; + public readonly bool SkipCompression; public readonly ITaskItem TaskItem; public readonly uint AssemblyLoadInfoIndex; @@ -37,6 +38,8 @@ public TargetDSO (ITaskItem dso) throw new InvalidOperationException ($"Internal error: unable to parse string '{index}' as an unsigned 32-bit integer"); } + SkipCompression = ShouldSkipCompression (dso); + string EnsureValidMetadata (string what) { string v = dso.GetMetadata (what); @@ -304,11 +307,24 @@ DSOAssemblyInfo AddAssembly (TargetDSO dso, ITaskItem dsoItem, Dictionary PrepareItems () { var sharedLibraries = new List (); - - if (FastPathAssemblyNames.Length == 0) { - return sharedLibraries; - } - - var fastPathItems = new HashSet (FastPathAssemblyNames, StringComparer.OrdinalIgnoreCase); var seenAbis = new HashSet (StringComparer.Ordinal); var satelliteAssemblies = new List (); ushort dsoIndexCounter = 0; var assemblyIndexes = new Dictionary (StringComparer.OrdinalIgnoreCase); + var fastPathItems = new HashSet (StringComparer.OrdinalIgnoreCase); + + if (FastPathAssemblies.Length != 0) { + foreach (ITaskItem item in FastPathAssemblies) { + fastPathItems.Add (item.ItemSpec); + } + } foreach (ITaskItem assembly in Assemblies) { if (fastPathItems.Contains (Path.GetFileName (assembly.ItemSpec))) { @@ -97,6 +97,7 @@ ITaskItem MakeTaskItem (ITaskItem assembly, string abi, string baseName) item.SetMetadata (DSOMetadata.InputAssemblyPath, assembly.ItemSpec); item.SetMetadata (DSOMetadata.SourceFileBaseName, baseName); item.SetMetadata (DSOMetadata.AssemblyLoadInfoIndex, MonoAndroidHelper.CultureInvariantToString (index)); + string skipCompression = assembly.GetMetadata ("AndroidSkipCompression"); if (!String.IsNullOrEmpty (skipCompression)) { item.SetMetadata (DSOMetadata.AndroidSkipCompression, skipCompression); From 4efb7049868e9fd0d86a69d0a940b9c00b0bdca1 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 17 Oct 2023 22:24:57 +0200 Subject: [PATCH 69/71] Selective compression + actual loading and decompressing of assembly data --- ...soft.Android.Sdk.NativeCompilation.targets | 2 +- ...GenerateAppAssemblyDSONativeSourceFiles.cs | 25 ++- .../PrepareAssemblyStandaloneDSOAbiItems.cs | 8 +- .../Utilities/AssemblyCompression.cs | 1 + .../Utilities/AssemblyDSOGenerator.Classes.cs | 7 +- .../Utilities/AssemblyDSOGenerator.cs | 1 + .../Utilities/MonoAndroidHelper.cs | 15 ++ src/monodroid/jni/assembly-data-provider.hh | 65 -------- src/monodroid/jni/embedded-assemblies-zip.cc | 2 +- src/monodroid/jni/embedded-assemblies.cc | 143 +++++++++++++++++- src/monodroid/jni/embedded-assemblies.hh | 20 +-- src/monodroid/jni/xamarin-app.hh | 1 + 12 files changed, 190 insertions(+), 100 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets index ea456840df0..d763db9c342 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.NativeCompilation.targets @@ -148,7 +148,7 @@ CompressedAssembliesOutputDirectory="$(_CompressedAssembliesOutputDir)" SupportedAbis="@(_BuildTargetAbis)" Assemblies="@(_ResolvedUserAssemblies);@(_ResolvedFrameworkAssemblies);@(_AndroidResolvedSatellitePaths)" - FastPathAssemblyNames="@(_DSOFastPathAssembly)" + FastPathAssemblies="@(_DSOFastPathAssembly)" StandaloneAssemblyDSOs="@(_StandaloneAssemblyDSO)" EnableCompression="$(AndroidEnableAssemblyCompression)"/> diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAppAssemblyDSONativeSourceFiles.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAppAssemblyDSONativeSourceFiles.cs index 9807aea1dba..436a00431bb 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAppAssemblyDSONativeSourceFiles.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateAppAssemblyDSONativeSourceFiles.cs @@ -20,7 +20,7 @@ public class GenerateAppAssemblyDSONativeSourceFiles : AssemblyNativeSourceGener public ITaskItem[] Assemblies { get; set; } [Required] - public string[] FastPathAssemblyNames { get; set; } + public ITaskItem[] FastPathAssemblies { get; set; } [Required] public ITaskItem[] StandaloneAssemblyDSOs { get; set; } @@ -28,7 +28,6 @@ public class GenerateAppAssemblyDSONativeSourceFiles : AssemblyNativeSourceGener protected override void Generate () { Dictionary> dsoAssembliesInfo = new (); - var satelliteAssemblies = new List (); // The figure here includes only the "fast path" assemblies, that is those which end up in libxamarin-app.so @@ -38,14 +37,30 @@ protected override void Generate () // their respective .so shared libraries. ulong uncompressedAssemblyDataSize = 0; AndroidTargetArch arch; - var fastPathAssemblies = new HashSet (FastPathAssemblyNames, StringComparer.OrdinalIgnoreCase); + var fastPathAssemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); + + if (FastPathAssemblies.Length > 0) { + foreach (ITaskItem item in FastPathAssemblies) { + if (fastPathAssemblies.ContainsKey (item.ItemSpec)) { + continue; + } + fastPathAssemblies.Add (item.ItemSpec, item); + } + }; Log.LogDebugMessage ("Processing input assemblies"); foreach (ITaskItem assembly in Assemblies) { - if (!fastPathAssemblies.Contains (Path.GetFileName (assembly.ItemSpec))) { + if (!fastPathAssemblies.TryGetValue (Path.GetFileName (assembly.ItemSpec), out ITaskItem fastPathItem)) { continue; } + string? skipCompression = fastPathItem.GetMetadata (DSOMetadata.AndroidSkipCompression); + if (!String.IsNullOrEmpty (skipCompression)) { + // It can potentially override the same metadata item in `assembly`, but that's fine, since the item's just a copy and we want the fast path items + // to dictate whether or not the assembly is compressed. + assembly.SetMetadata (DSOMetadata.AndroidSkipCompression, skipCompression); + } + CompressionResult cres = Compress (assembly); string inputFile = cres.OutputFile; @@ -87,7 +102,7 @@ protected override void Generate () Log.LogDebugMessage ($" {kvp.Key}: {kvp.Value.Count} assemblies"); } - var generator = new AssemblyDSOGenerator (FastPathAssemblyNames, dsoAssembliesInfo, inputAssemblyDataSize, uncompressedAssemblyDataSize); + var generator = new AssemblyDSOGenerator (fastPathAssemblies.Keys, dsoAssembliesInfo, inputAssemblyDataSize, uncompressedAssemblyDataSize); GenerateSources (SupportedAbis, generator, generator.Construct (), PrepareAbiItems.AssemblyDSOBase); void StoreAssembly (AndroidTargetArch arch, ITaskItem assembly, string inputFile, long fileLength, uint compressedSize, DSOAssemblyInfo? info = null) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAssemblyStandaloneDSOAbiItems.cs b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAssemblyStandaloneDSOAbiItems.cs index 32ff4b7428c..9200f96b38b 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAssemblyStandaloneDSOAbiItems.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAssemblyStandaloneDSOAbiItems.cs @@ -37,13 +37,7 @@ List PrepareItems () var satelliteAssemblies = new List (); ushort dsoIndexCounter = 0; var assemblyIndexes = new Dictionary (StringComparer.OrdinalIgnoreCase); - var fastPathItems = new HashSet (StringComparer.OrdinalIgnoreCase); - - if (FastPathAssemblies.Length != 0) { - foreach (ITaskItem item in FastPathAssemblies) { - fastPathItems.Add (item.ItemSpec); - } - } + HashSet fastPathItems = MonoAndroidHelper.MakeHashSet (FastPathAssemblies); foreach (ITaskItem assembly in Assemblies) { if (fastPathItems.Contains (Path.GetFileName (assembly.ItemSpec))) { diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs index d0c89eaab8c..feea6f7ccb3 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs @@ -61,6 +61,7 @@ public AssemblyCompression (TaskLoggingHelper log, string compressedOutputDir) this.compressedOutputDir = compressedOutputDir; } + // TODO: consider using https://github.com/emmanuel-marty/lz4ultra CompressionResult Compress (AssemblyData data, string outputDirectory) { if (data == null) { diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.Classes.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.Classes.cs index 4079dc41cf3..25a65049d98 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.Classes.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.Classes.cs @@ -93,6 +93,7 @@ sealed class AssembliesConfig public uint uncompressed_assembly_data_size; public uint assembly_name_length; public uint assembly_count; + public uint assembly_index_count; public uint assembly_dso_count; public uint shared_library_name_length; }; @@ -100,6 +101,9 @@ sealed class AssembliesConfig // Members with underscores correspond to the native fields we output. sealed class ArchState { + // Currently we hash assembly name with and without the extension + const int AssemblyNameVariationsCount = 2; + public readonly List> xa_assemblies; public readonly List>? xa_assembly_index32; public readonly List>? xa_assembly_index64; @@ -116,7 +120,7 @@ public ArchState (int assemblyCount, AndroidTargetArch arch, StandaloneAssemblyE StandaloneAssembly = standaloneAssembly; xa_assemblies = new List> (assemblyCount); - xa_assembly_names = new List (assemblyCount); + xa_assembly_names = new List (assemblyCount * AssemblyNameVariationsCount); xa_assembly_dso_names = new List (assemblyCount); switch (arch) { @@ -139,6 +143,7 @@ public ArchState (int assemblyCount, AndroidTargetArch arch, StandaloneAssemblyE uncompressed_assembly_data_size = 0, assembly_name_length = 0, assembly_count = (uint)assemblyCount, + assembly_index_count = (uint)assemblyCount * AssemblyNameVariationsCount, }; } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs index 85f0a4a83e1..a18497c293b 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.cs @@ -436,6 +436,7 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) uint assemblyIndex = (uint)archState.xa_assemblies.Count - 1; uint loadInfoIndex = isStandalone ? info.AssemblyLoadInfoIndex.Value : UInt32.MaxValue; + // If the number of assembly name variations in the index changes, ArchState.AssemblyNameVariationsCount **MUST** be updated accordingly if (is64Bit) { var indexEntry = new AssemblyIndexEntry64 { Name = info.Name, diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs index 04de7aaffb0..d4e6a454111 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs @@ -666,5 +666,20 @@ bool IsNotEmptyMetadataAndMaybeMatches (ITaskItem item, string metadataName, str return String.Compare (data, expectedValue, StringComparison.Ordinal) == 0; } } + + public static HashSet MakeHashSet (ICollection items) + { + var ret = new HashSet (StringComparer.OrdinalIgnoreCase); + + if (items.Count == 0) { + return ret; + } + + foreach (ITaskItem item in items) { + ret.Add (item.ItemSpec); + } + + return ret; + } } } diff --git a/src/monodroid/jni/assembly-data-provider.hh b/src/monodroid/jni/assembly-data-provider.hh index 45c18162b5f..dd1837ce36d 100644 --- a/src/monodroid/jni/assembly-data-provider.hh +++ b/src/monodroid/jni/assembly-data-provider.hh @@ -12,70 +12,5 @@ namespace xamarin::android::internal const uint8_t *data; const uint32_t size; }; - -#if defined (RELEASE) - namespace detail - { - class SO_AssemblyDataProvider - { - public: - SO_AssemblyDataProvider () = delete; - ~SO_AssemblyDataProvider () = delete; - - force_inline - static auto get_data_fastpath (AssemblyEntry const& entry) noexcept -> const AssemblyData - { - log_debug (LOG_ASSEMBLY, "Loading assembly from libxamarin-app.so, fast path"); - if (entry.uncompressed_data_size == 0) { - log_debug (LOG_ASSEMBLY, "Assembly is not compressed; input offset: %u; data size: %u", entry.input_data_offset, entry.input_data_size); - } else { - log_debug (LOG_ASSEMBLY, "Assembly is compressed; input offset: %u; compressed data size: %u; uncompressed data size: %u; input offset: %u; output offset: %u", - entry.input_data_offset, entry.input_data_size, entry.uncompressed_data_size, entry.input_data_offset, entry.uncompressed_data_offset); - } - - return {nullptr, 0}; - } - }; - } - - class SO_APK_AssemblyDataProvider - { - public: - force_inline - static auto get_data (AssemblyEntry const& entry, bool standalone) noexcept -> const AssemblyData - { - log_debug (LOG_ASSEMBLY, __PRETTY_FUNCTION__); - if (!standalone) { - return detail::SO_AssemblyDataProvider::get_data_fastpath (entry); - } - log_debug (LOG_ASSEMBLY, "Looking for assembly in the APK"); - - return {nullptr, 0}; - } - }; - - class SO_FILESYSTEM_AssemblyDataProvider - { - public: - force_inline - static auto get_data (AssemblyEntry const& entry, bool standalone) noexcept -> const AssemblyData - { - log_debug (LOG_ASSEMBLY, __PRETTY_FUNCTION__); - if (!standalone) { - return detail::SO_AssemblyDataProvider::get_data_fastpath (entry); - } - - log_debug (LOG_ASSEMBLY, "Loading assembly from a standalone DSO"); - log_debug (LOG_ASSEMBLY, "Looking for assembly on the filesystem"); - - return {nullptr, 0}; - } - }; -#else // def RELEASE - class DLL_APK_AssemblyDataProvider - { - public: - }; -#endif // ndef RELEASE } #endif // ndef ASSEMBLY_DATA_PROVIDER_HH diff --git a/src/monodroid/jni/embedded-assemblies-zip.cc b/src/monodroid/jni/embedded-assemblies-zip.cc index 942a1a2d516..b7ac4fc63b9 100644 --- a/src/monodroid/jni/embedded-assemblies-zip.cc +++ b/src/monodroid/jni/embedded-assemblies-zip.cc @@ -190,7 +190,7 @@ EmbeddedAssemblies::zip_load_standalone_dso_entries (std::vector const& number_of_found_assembly_dsos++; // We have an assembly DSO - log_info (LOG_ASSEMBLY, "Found an assembly DSO: %s; index: %s", entry_name.get (), entry_name.get () + (entry_name.length () - 7)); + log_info (LOG_ASSEMBLY, "Found an assembly DSO: %s; index: %s; data offset: %u", entry_name.get (), entry_name.get () + (entry_name.length () - 7), state.data_offset); bool valid_hex = true; auto integer_from_hex_char = [] (dynamic_local_string const& s, size_t pos, bool &is_valid, size_t shift) -> uint16_t diff --git a/src/monodroid/jni/embedded-assemblies.cc b/src/monodroid/jni/embedded-assemblies.cc index ef5994829fc..461d6e1c0a2 100644 --- a/src/monodroid/jni/embedded-assemblies.cc +++ b/src/monodroid/jni/embedded-assemblies.cc @@ -68,6 +68,110 @@ class MonoGuidString final char *guid = nullptr; }; +namespace { +#if defined (RELEASE) + class SO_AssemblyDataProvider + { + public: + SO_AssemblyDataProvider () = delete; + ~SO_AssemblyDataProvider () = delete; + + private: + force_inline + static auto uncompress (const uint8_t *const compressed_start, size_t compressed_size, uint8_t *dest, size_t expected_uncompressed_size, dynamic_local_string const& assembly_name) noexcept -> void + { + StartupAwareLock decompress_lock (assembly_decompress_mutex); + const char *data_start = reinterpret_cast(compressed_start); + int ret = LZ4_decompress_safe (data_start, reinterpret_cast(dest), static_cast(compressed_size), static_cast(expected_uncompressed_size)); + + if (ret < 0) { + log_fatal (LOG_ASSEMBLY, "Decompression of assembly %s failed with code %d", assembly_name.get (), ret); + Helpers::abort_application (); + } + + if (static_cast(ret) != expected_uncompressed_size) { + log_debug (LOG_ASSEMBLY, "Decompression of assembly %s yielded a different size (expected %lu, got %zu)", assembly_name.get (), expected_uncompressed_size, static_cast(ret)); + Helpers::abort_application (); + } + } + + force_inline + static auto get_compressed_data_fastpath (AssemblyEntry const& entry, dynamic_local_string const& assembly_name) noexcept -> const AssemblyData + { + log_debug (LOG_ASSEMBLY, "Assembly '%s' is compressed; input offset: %u; compressed data size: %u; uncompressed data size: %u; input offset: %u; output offset: %u", + assembly_name.get (), entry.input_data_offset, entry.input_data_size, entry.uncompressed_data_size, entry.input_data_offset, entry.uncompressed_data_offset); + const uint8_t *const compressed_start = xa_input_assembly_data + entry.input_data_offset; + uint8_t *dest = xa_uncompressed_assembly_data + entry.uncompressed_data_offset; + uncompress (compressed_start, entry.input_data_size, dest, entry.uncompressed_data_size, assembly_name); + return {dest, entry.uncompressed_data_size}; + } + + force_inline + static auto get_uncompressed_data_fastpath (AssemblyEntry const& entry, dynamic_local_string const& assembly_name) noexcept -> const AssemblyData + { + log_debug (LOG_ASSEMBLY, "Assembly %s' is not compressed; input offset: %u; data size: %u", assembly_name.get (), entry.input_data_offset, entry.input_data_size); + return {nullptr, 0}; + } + + public: + force_inline + static auto get_data_fastpath (AssemblyEntry const& entry, dynamic_local_string const& assembly_name) noexcept -> const AssemblyData + { + log_debug (LOG_ASSEMBLY, "Loading assembly from libxamarin-app.so, fast path"); + if (entry.uncompressed_data_size > 0) { + return get_compressed_data_fastpath (entry, assembly_name); + } + + return get_uncompressed_data_fastpath (entry, assembly_name); + } + + private: + static inline std::mutex assembly_decompress_mutex; + }; + + class SO_APK_AssemblyDataProvider + { + public: + force_inline + static auto get_data (AssemblyEntry const& entry, AssemblyIndexEntry const& index_entry, dynamic_local_string const& assembly_name) noexcept -> const AssemblyData + { + log_debug (LOG_ASSEMBLY, __PRETTY_FUNCTION__); + if (!index_entry.is_standalone) { + return SO_AssemblyDataProvider::get_data_fastpath (entry, assembly_name); + } + + AssemblyLoadInfo const& load_info = xa_assemblies_load_info[index_entry.load_info_index]; + log_debug (LOG_ASSEMBLY, "Looking for assembly in the APK; Data offset in the archive: %u; mmap address: %p", load_info.apk_offset, load_info.mmap_addr); + + return {nullptr, 0}; + } + }; + + class SO_FILESYSTEM_AssemblyDataProvider + { + public: + force_inline + static auto get_data (AssemblyEntry const& entry, AssemblyIndexEntry const& index_entry, dynamic_local_string const& assembly_name) noexcept -> const AssemblyData + { + log_debug (LOG_ASSEMBLY, __PRETTY_FUNCTION__); + if (!index_entry.is_standalone) { + return SO_AssemblyDataProvider::get_data_fastpath (entry, assembly_name); + } + + log_debug (LOG_ASSEMBLY, "Loading assembly from a standalone DSO"); + log_debug (LOG_ASSEMBLY, "Looking for assembly on the filesystem"); + + return {nullptr, 0}; + } + }; +#else // def RELEASE + class DLL_APK_AssemblyDataProvider + { + public: + }; +#endif // ndef RELEASE +} + void EmbeddedAssemblies::init () noexcept { log_debug (LOG_ASSEMBLY, __PRETTY_FUNCTION__); @@ -94,6 +198,20 @@ EmbeddedAssemblies::set_assembly_data_and_size (uint8_t* source_assembly_data, u dest_assembly_data_size = source_assembly_data_size; } +#if defined (RELEASE) +const AssemblyData +EmbeddedAssemblies::get_assembly_data_dso_apk (AssemblyEntry const& entry, AssemblyIndexEntry const& index_entry, dynamic_local_string const& assembly_name) noexcept +{ + return SO_APK_AssemblyDataProvider::get_data (entry, index_entry, assembly_name); +} + +const AssemblyData +EmbeddedAssemblies::get_assembly_data_dso_fs (AssemblyEntry const& entry, AssemblyIndexEntry const& index_entry, dynamic_local_string const& assembly_name) noexcept +{ + return SO_FILESYSTEM_AssemblyDataProvider::get_data (entry, index_entry, assembly_name); +} +#endif // def RELEASE + force_inline void EmbeddedAssemblies::get_assembly_data (uint8_t *data, uint32_t data_size, [[maybe_unused]] const char *name, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept { @@ -240,14 +358,14 @@ EmbeddedAssemblies::load_bundled_assembly ( #if defined(RELEASE) template force_inline MonoAssembly* -EmbeddedAssemblies::standalone_dso_open_from_bundles (dynamic_local_string& name, TLoaderData loader_data) noexcept +EmbeddedAssemblies::standalone_dso_open_from_bundles (dynamic_local_string const& name, TLoaderData loader_data) noexcept { - hash_t hash = xxhash::hash (name.get (), name.length ()); - log_debug (LOG_ASSEMBLY, "standalone_dso_open_from_bundles: looking for bundled name: '%s', hash 0x%zx", name.get (), hash); + hash_t name_hash = xxhash::hash (name.get (), name.length ()); + log_debug (LOG_ASSEMBLY, "standalone_dso_open_from_bundles: looking for bundled name: '%s', hash 0x%zx", name.get (), name_hash); auto equal = [](AssemblyIndexEntry const& entry, hash_t key) -> bool { return entry.name_hash == key; }; auto less_than = [](AssemblyIndexEntry const& entry, hash_t key) -> bool { return entry.name_hash < key; }; - ssize_t index = Search::binary_search (hash, xa_assembly_index, xa_assemblies_config.assembly_count); + ssize_t index = Search::binary_search (name_hash, xa_assembly_index, xa_assemblies_config.assembly_index_count); if (index == -1) { log_warn (LOG_ASSEMBLY, "assembly '%s' not found in the DSO assembly index", name.get ()); @@ -271,10 +389,23 @@ EmbeddedAssemblies::standalone_dso_open_from_bundles (dynamic_local_string(assembly_data.data), assembly_data.size); + if (image == nullptr) { + log_warn (LOG_ASSEMBLY, "Failed to load MonoImage of '%s'", name.get ()); + return nullptr; + } + + MonoImageOpenStatus status; + MonoAssembly *a = mono_assembly_load_from_full (image, name.get (), &status, false /* ref_only */); + if (a == nullptr || status != MonoImageOpenStatus::MONO_IMAGE_OK) { + log_warn (LOG_ASSEMBLY, "Failed to load managed assembly '%s'. %s", name.get (), mono_image_strerror (status)); + return nullptr; + } + + return a; } #endif // def RELEASE diff --git a/src/monodroid/jni/embedded-assemblies.hh b/src/monodroid/jni/embedded-assemblies.hh index c720bda379e..2fc65b6fe0c 100644 --- a/src/monodroid/jni/embedded-assemblies.hh +++ b/src/monodroid/jni/embedded-assemblies.hh @@ -97,7 +97,7 @@ namespace xamarin::android::internal { EntryLocation location; }; #if defined (RELEASE) - using get_assembly_data_dso_fn = const AssemblyData (*)(AssemblyEntry const& entry, bool fast_path); + using get_assembly_data_dso_fn = const AssemblyData (*)(AssemblyEntry const& entry, AssemblyIndexEntry const& index_entry, dynamic_local_string const& assembly_name); #endif private: static constexpr char ZIP_CENTRAL_MAGIC[] = "PK\1\2"; @@ -190,7 +190,7 @@ namespace xamarin::android::internal { void gather_bundled_assemblies_from_apk (const char* apk, monodroid_should_register should_register); template - MonoAssembly* standalone_dso_open_from_bundles (dynamic_local_string& name, TLoaderData loader_data) noexcept; + MonoAssembly* standalone_dso_open_from_bundles (dynamic_local_string const& name, TLoaderData loader_data) noexcept; template MonoAssembly* individual_assemblies_open_from_bundles (dynamic_local_string& name, TLoaderData loader_data, bool ref_only) noexcept; @@ -237,20 +237,13 @@ namespace xamarin::android::internal { void get_assembly_data (XamarinAndroidBundledAssembly const& e, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept; #if defined (RELEASE) - static const AssemblyData get_assembly_data_dso_apk (AssemblyEntry const& entry, bool standalone) noexcept - { - return SO_APK_AssemblyDataProvider::get_data (entry, standalone); - } - - static const AssemblyData get_assembly_data_dso_fs (AssemblyEntry const& entry, bool standalone) noexcept - { - return SO_FILESYSTEM_AssemblyDataProvider::get_data (entry, standalone); - } + static const AssemblyData get_assembly_data_dso_apk (AssemblyEntry const& entry, AssemblyIndexEntry const& index_entry, dynamic_local_string const& assembly_name) noexcept; + static const AssemblyData get_assembly_data_dso_fs (AssemblyEntry const& entry, AssemblyIndexEntry const& index_entry, dynamic_local_string const& assembly_name) noexcept; force_inline - const AssemblyData get_assembly_data_dso (AssemblyEntry const& entry, bool standalone) const noexcept + const AssemblyData get_assembly_data_dso (AssemblyEntry const& entry, AssemblyIndexEntry const& index_entry, dynamic_local_string const& assembly_name) const noexcept { - return (*get_assembly_data_dso_impl) (entry, standalone); + return (*get_assembly_data_dso_impl) (entry, index_entry, assembly_name); } #endif void zip_load_entries (int fd, const char *apk_name, monodroid_should_register should_register); @@ -355,7 +348,6 @@ namespace xamarin::android::internal { #endif // def NET uint32_t number_of_standalone_dsos = 0; bool need_to_scan_more_apks = true; - std::mutex assembly_decompress_mutex; #if defined (RELEASE) get_assembly_data_dso_fn get_assembly_data_dso_impl = nullptr; #endif diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index 60f1c8422ce..d0b1cf99672 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -141,6 +141,7 @@ struct AssembliesConfig uint32_t uncompressed_assembly_data_size; uint32_t assembly_name_length; uint32_t assembly_count; + uint32_t assembly_index_count; uint32_t assembly_dso_count; uint32_t shared_library_name_length; }; From 13a8f6d204366231ffd76509d0261fabc8f66eec Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 18 Oct 2023 22:25:06 +0200 Subject: [PATCH 70/71] Playing with concurrency Ditched mutex, switched to simple spinlocks and atomic reads instead. Generates much better code on ARM64 (and that's our main target atm) --- .../Utilities/AssemblyDSOGenerator.Classes.cs | 5 + src/monodroid/jni/embedded-assemblies.cc | 212 +++++++++++------- src/monodroid/jni/embedded-assemblies.hh | 14 +- src/monodroid/jni/monodroid-glue.cc | 1 - src/monodroid/jni/xamarin-app.hh | 3 + 5 files changed, 146 insertions(+), 89 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.Classes.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.Classes.cs index 25a65049d98..a66a69d6dd9 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.Classes.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyDSOGenerator.Classes.cs @@ -84,6 +84,11 @@ sealed class AssemblyLoadInfo // Address at which the assembly data was mmapped public IntPtr mmap_addr; + + // Address at which the assembly data is available. It may be the same as `mmap_addr` if the + // data wasn't compressed, different otherwise. + public IntPtr data_addr; + public uint data_size; }; // Must be identical to AssembliesConfig in src/monodroid/jni/xamarin-app.hh diff --git a/src/monodroid/jni/embedded-assemblies.cc b/src/monodroid/jni/embedded-assemblies.cc index 461d6e1c0a2..1b5d2917e67 100644 --- a/src/monodroid/jni/embedded-assemblies.cc +++ b/src/monodroid/jni/embedded-assemblies.cc @@ -70,78 +70,151 @@ class MonoGuidString final namespace { #if defined (RELEASE) - class SO_AssemblyDataProvider - { - public: - SO_AssemblyDataProvider () = delete; - ~SO_AssemblyDataProvider () = delete; - - private: - force_inline - static auto uncompress (const uint8_t *const compressed_start, size_t compressed_size, uint8_t *dest, size_t expected_uncompressed_size, dynamic_local_string const& assembly_name) noexcept -> void + namespace detail { + class SO_AssemblyDataProvider { - StartupAwareLock decompress_lock (assembly_decompress_mutex); - const char *data_start = reinterpret_cast(compressed_start); - int ret = LZ4_decompress_safe (data_start, reinterpret_cast(dest), static_cast(compressed_size), static_cast(expected_uncompressed_size)); + public: + SO_AssemblyDataProvider () = delete; + ~SO_AssemblyDataProvider () = delete; + + private: + force_inline + static auto acquire_lock (std::atomic_flag &lock) noexcept -> void + { + if (monodroidRuntime.is_startup_in_progress ()) { + return; // no need to acquire before we start the managed code + } - if (ret < 0) { - log_fatal (LOG_ASSEMBLY, "Decompression of assembly %s failed with code %d", assembly_name.get (), ret); - Helpers::abort_application (); + while (lock.test_and_set (std::memory_order_acquire)) { + while (lock.test (std::memory_order_relaxed)) { + ; + } + } } - if (static_cast(ret) != expected_uncompressed_size) { - log_debug (LOG_ASSEMBLY, "Decompression of assembly %s yielded a different size (expected %lu, got %zu)", assembly_name.get (), expected_uncompressed_size, static_cast(ret)); - Helpers::abort_application (); + force_inline + static auto release_lock (std::atomic_flag &lock) noexcept -> void + { + if (monodroidRuntime.is_startup_in_progress ()) { + return; // no need to release before we start the managed code + } + + lock.clear (std::memory_order_release); } - } - force_inline - static auto get_compressed_data_fastpath (AssemblyEntry const& entry, dynamic_local_string const& assembly_name) noexcept -> const AssemblyData - { - log_debug (LOG_ASSEMBLY, "Assembly '%s' is compressed; input offset: %u; compressed data size: %u; uncompressed data size: %u; input offset: %u; output offset: %u", - assembly_name.get (), entry.input_data_offset, entry.input_data_size, entry.uncompressed_data_size, entry.input_data_offset, entry.uncompressed_data_offset); - const uint8_t *const compressed_start = xa_input_assembly_data + entry.input_data_offset; - uint8_t *dest = xa_uncompressed_assembly_data + entry.uncompressed_data_offset; - uncompress (compressed_start, entry.input_data_size, dest, entry.uncompressed_data_size, assembly_name); - return {dest, entry.uncompressed_data_size}; - } + force_inline + static auto decompress (const uint8_t *const compressed_start, size_t compressed_size, uint8_t *dest, size_t expected_uncompressed_size, dynamic_local_string const& assembly_name) noexcept -> void + { + acquire_lock (decompression_flag); - force_inline - static auto get_uncompressed_data_fastpath (AssemblyEntry const& entry, dynamic_local_string const& assembly_name) noexcept -> const AssemblyData - { - log_debug (LOG_ASSEMBLY, "Assembly %s' is not compressed; input offset: %u; data size: %u", assembly_name.get (), entry.input_data_offset, entry.input_data_size); - return {nullptr, 0}; - } + const char *data_start = reinterpret_cast(compressed_start); + int ret = LZ4_decompress_safe (data_start, reinterpret_cast(dest), static_cast(compressed_size), static_cast(expected_uncompressed_size)); - public: - force_inline - static auto get_data_fastpath (AssemblyEntry const& entry, dynamic_local_string const& assembly_name) noexcept -> const AssemblyData - { - log_debug (LOG_ASSEMBLY, "Loading assembly from libxamarin-app.so, fast path"); - if (entry.uncompressed_data_size > 0) { - return get_compressed_data_fastpath (entry, assembly_name); + if (ret < 0) { + log_fatal (LOG_ASSEMBLY, "Decompression of assembly %s failed with code %d", assembly_name.get (), ret); + Helpers::abort_application (); + } + + if (static_cast(ret) != expected_uncompressed_size) { + log_debug (LOG_ASSEMBLY, "Decompression of assembly %s yielded a different size (expected %lu, got %zu)", assembly_name.get (), expected_uncompressed_size, static_cast(ret)); + Helpers::abort_application (); + } + + release_lock (decompression_flag); } - return get_uncompressed_data_fastpath (entry, assembly_name); - } + force_inline + static auto get_compressed_data_fastpath (AssemblyEntry const& entry, dynamic_local_string const& assembly_name) noexcept -> const AssemblyData + { + log_debug (LOG_ASSEMBLY, "Assembly '%s' is compressed; input offset: %u; compressed data size: %u; uncompressed data size: %u; input offset: %u; output offset: %u", + assembly_name.get (), entry.input_data_offset, entry.input_data_size, entry.uncompressed_data_size, entry.input_data_offset, entry.uncompressed_data_offset); + const uint8_t *const compressed_start = xa_input_assembly_data + entry.input_data_offset; + uint8_t *dest = xa_uncompressed_assembly_data + entry.uncompressed_data_offset; + decompress (compressed_start, entry.input_data_size, dest, entry.uncompressed_data_size, assembly_name); + return {dest, entry.uncompressed_data_size}; + } - private: - static inline std::mutex assembly_decompress_mutex; - }; + force_inline + static auto get_uncompressed_data_fastpath (AssemblyEntry const& entry, dynamic_local_string const& assembly_name) noexcept -> const AssemblyData + { + log_debug (LOG_ASSEMBLY, "Assembly %s' is not compressed; input offset: %u; data size: %u", assembly_name.get (), entry.input_data_offset, entry.input_data_size); + return {nullptr, 0}; + } + + public: + force_inline + static auto get_data_fastpath (AssemblyEntry const& entry, dynamic_local_string const& assembly_name) noexcept -> const AssemblyData + { + log_debug (LOG_ASSEMBLY, "Loading assembly from libxamarin-app.so, fast path"); + if (entry.uncompressed_data_size > 0) { + return get_compressed_data_fastpath (entry, assembly_name); + } + + return get_uncompressed_data_fastpath (entry, assembly_name); + } + + force_inline + static auto acquire_data_load_lock () noexcept -> void + { + acquire_lock (data_load_flag); + } + + force_inline + static auto release_data_load_lock () noexcept -> void + { + release_lock (data_load_flag); + } + + // Returns `true` if data load lock has been acquired, `false` otherwise. Caller is responsible for releasing + // the lock + force_inline + static auto mmap_dso (AssemblyLoadInfo &load_info, int fd, off_t data_offset, size_t data_length) noexcept -> bool + { + if (__atomic_load_n (&load_info.mmap_addr, __ATOMIC_CONSUME) != nullptr) { + return false; // lock not acquired + } + + acquire_data_load_lock (); + log_debug (LOG_ASSEMBLY, "Will mmap from fd %d, at offset %zd with length %zu", fd, data_offset, data_length); + // TODO: map here + + return true; // lock acquired + } + + private: + static inline std::atomic_flag decompression_flag = ATOMIC_FLAG_INIT; + static inline std::atomic_flag data_load_flag = ATOMIC_FLAG_INIT; + }; + } class SO_APK_AssemblyDataProvider { public: force_inline - static auto get_data (AssemblyEntry const& entry, AssemblyIndexEntry const& index_entry, dynamic_local_string const& assembly_name) noexcept -> const AssemblyData + static auto get_data (int apk_fd, AssemblyEntry const& entry, AssemblyIndexEntry const& index_entry, dynamic_local_string const& assembly_name) noexcept -> const AssemblyData { log_debug (LOG_ASSEMBLY, __PRETTY_FUNCTION__); if (!index_entry.is_standalone) { - return SO_AssemblyDataProvider::get_data_fastpath (entry, assembly_name); + return detail::SO_AssemblyDataProvider::get_data_fastpath (entry, assembly_name); + } + + AssemblyLoadInfo &load_info = xa_assemblies_load_info[index_entry.load_info_index]; + log_debug (LOG_ASSEMBLY, "Looking for assembly in the APK; Data offset in the archive: %u; mmap address: %p; data address: %p; data size: %u", load_info.apk_offset, load_info.mmap_addr, load_info.data_addr, load_info.data_size); + + if (__atomic_load_n (&load_info.data_addr, __ATOMIC_CONSUME) != nullptr) { + return {load_info.data_addr, load_info.data_size}; } - AssemblyLoadInfo const& load_info = xa_assemblies_load_info[index_entry.load_info_index]; - log_debug (LOG_ASSEMBLY, "Looking for assembly in the APK; Data offset in the archive: %u; mmap address: %p", load_info.apk_offset, load_info.mmap_addr); + bool lock_acquired = detail::SO_AssemblyDataProvider::mmap_dso (load_info, apk_fd, static_cast(load_info.apk_offset), entry.input_data_size); + if (!lock_acquired) { + detail::SO_AssemblyDataProvider::acquire_data_load_lock (); + lock_acquired = true; + // TODO: actually load data here + } + + if (lock_acquired) { + detail::SO_AssemblyDataProvider::release_data_load_lock (); + } return {nullptr, 0}; } @@ -151,11 +224,11 @@ namespace { { public: force_inline - static auto get_data (AssemblyEntry const& entry, AssemblyIndexEntry const& index_entry, dynamic_local_string const& assembly_name) noexcept -> const AssemblyData + static auto get_data ([[maybe_unused]] int apk_fd, AssemblyEntry const& entry, AssemblyIndexEntry const& index_entry, dynamic_local_string const& assembly_name) noexcept -> const AssemblyData { log_debug (LOG_ASSEMBLY, __PRETTY_FUNCTION__); if (!index_entry.is_standalone) { - return SO_AssemblyDataProvider::get_data_fastpath (entry, assembly_name); + return detail::SO_AssemblyDataProvider::get_data_fastpath (entry, assembly_name); } log_debug (LOG_ASSEMBLY, "Loading assembly from a standalone DSO"); @@ -172,18 +245,6 @@ namespace { #endif // ndef RELEASE } -void EmbeddedAssemblies::init () noexcept -{ - log_debug (LOG_ASSEMBLY, __PRETTY_FUNCTION__); -#if defined (RELEASE) - if (BasicAndroidSystem::is_embedded_dso_mode_enabled ()) { - get_assembly_data_dso_impl = get_assembly_data_dso_apk; - } else { - get_assembly_data_dso_impl = get_assembly_data_dso_fs; - } -#endif -} - void EmbeddedAssemblies::set_assemblies_prefix (const char *prefix) { if (assemblies_prefix_override != nullptr) @@ -199,16 +260,15 @@ EmbeddedAssemblies::set_assembly_data_and_size (uint8_t* source_assembly_data, u } #if defined (RELEASE) +force_inline const AssemblyData -EmbeddedAssemblies::get_assembly_data_dso_apk (AssemblyEntry const& entry, AssemblyIndexEntry const& index_entry, dynamic_local_string const& assembly_name) noexcept +EmbeddedAssemblies::get_assembly_data_dso (AssemblyEntry const& entry, AssemblyIndexEntry const& index_entry, dynamic_local_string const& assembly_name) const noexcept { - return SO_APK_AssemblyDataProvider::get_data (entry, index_entry, assembly_name); -} + if (BasicAndroidSystem::is_embedded_dso_mode_enabled ()) { + return SO_APK_AssemblyDataProvider::get_data (apk_fd, entry, index_entry, assembly_name); + } -const AssemblyData -EmbeddedAssemblies::get_assembly_data_dso_fs (AssemblyEntry const& entry, AssemblyIndexEntry const& index_entry, dynamic_local_string const& assembly_name) noexcept -{ - return SO_FILESYSTEM_AssemblyDataProvider::get_data (entry, index_entry, assembly_name); + return SO_FILESYSTEM_AssemblyDataProvider::get_data (apk_fd, entry, index_entry, assembly_name); } #endif // def RELEASE @@ -922,15 +982,13 @@ EmbeddedAssemblies::md_mmap_apk_file (int fd, uint32_t offset, size_t size, cons void EmbeddedAssemblies::gather_bundled_assemblies_from_apk (const char* apk, monodroid_should_register should_register) { - int fd; - - if ((fd = open (apk, O_RDONLY)) < 0) { + if ((apk_fd = open (apk, O_RDONLY)) < 0) { log_error (LOG_DEFAULT, "ERROR: Unable to load application package %s.", apk); Helpers::abort_application (); } - log_info (LOG_ASSEMBLY, "APK %s FD: %d", apk, fd); + log_info (LOG_ASSEMBLY, "APK %s FD: %d", apk, apk_fd); - zip_load_entries (fd, apk, should_register); + zip_load_entries (apk_fd, apk, should_register); } #if defined (DEBUG) || !defined (ANDROID) diff --git a/src/monodroid/jni/embedded-assemblies.hh b/src/monodroid/jni/embedded-assemblies.hh index 2fc65b6fe0c..9c37139521a 100644 --- a/src/monodroid/jni/embedded-assemblies.hh +++ b/src/monodroid/jni/embedded-assemblies.hh @@ -97,7 +97,7 @@ namespace xamarin::android::internal { EntryLocation location; }; #if defined (RELEASE) - using get_assembly_data_dso_fn = const AssemblyData (*)(AssemblyEntry const& entry, AssemblyIndexEntry const& index_entry, dynamic_local_string const& assembly_name); + using get_assembly_data_dso_fn = const AssemblyData (*)(int apk_fd, AssemblyEntry const& entry, AssemblyIndexEntry const& index_entry, dynamic_local_string const& assembly_name); #endif private: static constexpr char ZIP_CENTRAL_MAGIC[] = "PK\1\2"; @@ -130,8 +130,6 @@ namespace xamarin::android::internal { EmbeddedAssemblies () noexcept {} - void init () noexcept; - #if defined (DEBUG) || !defined (ANDROID) void try_load_typemaps_from_directory (const char *path); #endif @@ -237,14 +235,7 @@ namespace xamarin::android::internal { void get_assembly_data (XamarinAndroidBundledAssembly const& e, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept; #if defined (RELEASE) - static const AssemblyData get_assembly_data_dso_apk (AssemblyEntry const& entry, AssemblyIndexEntry const& index_entry, dynamic_local_string const& assembly_name) noexcept; - static const AssemblyData get_assembly_data_dso_fs (AssemblyEntry const& entry, AssemblyIndexEntry const& index_entry, dynamic_local_string const& assembly_name) noexcept; - - force_inline - const AssemblyData get_assembly_data_dso (AssemblyEntry const& entry, AssemblyIndexEntry const& index_entry, dynamic_local_string const& assembly_name) const noexcept - { - return (*get_assembly_data_dso_impl) (entry, index_entry, assembly_name); - } + const AssemblyData get_assembly_data_dso (AssemblyEntry const& entry, AssemblyIndexEntry const& index_entry, dynamic_local_string const& assembly_name) const noexcept; #endif void zip_load_entries (int fd, const char *apk_name, monodroid_should_register should_register); void zip_load_individual_assembly_entries (std::vector const& buf, uint32_t num_entries, monodroid_should_register should_register, ZipEntryLoadState &state) noexcept; @@ -348,6 +339,7 @@ namespace xamarin::android::internal { #endif // def NET uint32_t number_of_standalone_dsos = 0; bool need_to_scan_more_apks = true; + int apk_fd = -1; #if defined (RELEASE) get_assembly_data_dso_fn get_assembly_data_dso_impl = nullptr; #endif diff --git a/src/monodroid/jni/monodroid-glue.cc b/src/monodroid/jni/monodroid-glue.cc index da55050624d..707438250b4 100644 --- a/src/monodroid/jni/monodroid-glue.cc +++ b/src/monodroid/jni/monodroid-glue.cc @@ -2156,7 +2156,6 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl #endif // def NET android_api_level = apiLevel; - embeddedAssemblies.init (); // **MUST** be called after androidSystem.detect_embedded_dso_mode! androidSystem.set_running_in_emulator (isEmulator); java_TimeZone = utils.get_class_from_runtime_field (env, klass, "java_util_TimeZone", true); diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index d0b1cf99672..bbdcd715d4e 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -127,6 +127,9 @@ struct AssemblyLoadInfo uint32_t apk_offset; // offset into the APK, or 0 if the assembly isn't in a standalone DSO or if the DSOs are // extracted to disk at install time void *mmap_addr; // Address at which the assembly data was mmapped + uint8_t *data_addr; // Address at which the assembly data is available. It may be the same as `mmap_addr` if the + // data wasn't compressed, different otherwise. + uint32_t data_size; }; constexpr uint32_t InputAssemblyDataSize = 1024; From c0b1348a6285afded0b9565916f3a3291c2db002 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 19 Oct 2023 23:29:02 +0200 Subject: [PATCH 71/71] XA hello world works, but much slower... 31% performance loss atm :( --- .../BuildAndLinkStandaloneAssemblyDSOs.cs | 12 +-- ...GenerateAppAssemblyDSONativeSourceFiles.cs | 11 +++ .../Utilities/AssemblyDSOGenerator.Classes.cs | 4 +- .../Utilities/AssemblyDSOGenerator.cs | 9 ++- .../Utilities/DSOAssemblyInfo.cs | 1 + .../Utilities/DSOMetadata.cs | 2 +- src/monodroid/jni/embedded-assemblies-zip.cc | 4 +- src/monodroid/jni/embedded-assemblies.cc | 77 +++++++++++++++---- src/monodroid/jni/embedded-assemblies.hh | 10 ++- src/monodroid/jni/mono-image-loader.hh | 2 +- src/monodroid/jni/xamarin-app.hh | 3 +- 11 files changed, 107 insertions(+), 28 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildAndLinkStandaloneAssemblyDSOs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildAndLinkStandaloneAssemblyDSOs.cs index 221b533ff78..e35f651bff4 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildAndLinkStandaloneAssemblyDSOs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildAndLinkStandaloneAssemblyDSOs.cs @@ -33,9 +33,9 @@ public TargetDSO (ITaskItem dso) SourceFileBaseName = EnsureValidMetadata (DSOMetadata.SourceFileBaseName); Culture = dso.GetMetadata (DSOMetadata.SatelliteAssemblyCulture); - string index = EnsureValidMetadata (DSOMetadata.AssemblyLoadInfoIndex); - if (!UInt32.TryParse (index, out AssemblyLoadInfoIndex)) { - throw new InvalidOperationException ($"Internal error: unable to parse string '{index}' as an unsigned 32-bit integer"); + string metadata = EnsureValidMetadata (DSOMetadata.AssemblyLoadInfoIndex); + if (!UInt32.TryParse (metadata, out AssemblyLoadInfoIndex)) { + throw new InvalidOperationException ($"Internal error: unable to parse string '{metadata}' as an unsigned 32-bit integer"); } SkipCompression = ShouldSkipCompression (dso); @@ -95,7 +95,7 @@ protected override void Generate () supportedAbis.Add (dso.Abi); var dsoItem = new TaskItem (dso.DSOPath); - DSOAssemblyInfo dsoInfo = AddAssembly (dso, dsoItem, assemblies); + LocalDSOAssemblyInfo dsoInfo = AddAssembly (dso, dsoItem, assemblies); dsoItem.SetMetadata (DSOMetadata.Abi, dso.Abi); dsoItem.SetMetadata (DSOMetadata.AssemblyLoadInfoIndex, MonoAndroidHelper.CultureInvariantToString (dso.AssemblyLoadInfoIndex)); @@ -290,7 +290,7 @@ void DoCompileAndLink (NativeCompilationHelper.AssemblerConfig config, Dictionar return configs; } - DSOAssemblyInfo AddAssembly (TargetDSO dso, ITaskItem dsoItem, Dictionary> assemblies) + LocalDSOAssemblyInfo AddAssembly (TargetDSO dso, ITaskItem dsoItem, Dictionary> assemblies) { string asmName = Path.GetFileNameWithoutExtension (dso.OriginalAssemblyPath); if (!String.IsNullOrEmpty (dso.Culture)) { @@ -324,7 +324,7 @@ DSOAssemblyInfo AddAssembly (TargetDSO dso, ITaskItem dsoItem, Dictionary[]), XAAssembliesLoadInfo) { - ArrayItemCount = (ulong)expectedAssemblyCount, + ArrayItemCount = (ulong)expectedAssemblyCount, // TODO: this should be equal to DSO count Options = LlvmIrVariableOptions.GlobalWritable, ZeroInitializeArray = true, }; @@ -372,6 +372,11 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) if (!info.AssemblyLoadInfoIndex.HasValue) { throw new InvalidOperationException ($"Internal error: item for assembly '{info.Name}' is missing the required assembly load index value"); } + + if (!info.AssemblyDataSymbolOffset.HasValue) { + throw new InvalidOperationException ($"Internal error: item for assembly '{info.Name}' is missing the required assembly data offset value"); + } + dso_count++; } @@ -388,7 +393,7 @@ void AddAssemblyData (AndroidTargetArch arch, List infos) IsStandalone = isStandalone, Name = info.Name, InputFilePath = info.InputFile, - input_data_offset = (uint)inputOffset, + input_data_offset = isStandalone ? (uint)info.AssemblyDataSymbolOffset : (uint)inputOffset, input_data_size = inputSize, uncompressed_data_size = info.CompressedDataSize == 0 ? 0 : (uint)info.DataSize, uncompressed_data_offset = (uint)uncompressedOffset, diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/DSOAssemblyInfo.cs b/src/Xamarin.Android.Build.Tasks/Utilities/DSOAssemblyInfo.cs index a0a074d4eb0..8563bc81d6c 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/DSOAssemblyInfo.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/DSOAssemblyInfo.cs @@ -31,6 +31,7 @@ class DSOAssemblyInfo public string? StandaloneDSOName { get; } public uint? AssemblyLoadInfoIndex { get; set; } + public ulong? AssemblyDataSymbolOffset { get; set; } /// /// is the original assembly name, including culture prefix (e.g. `en_US/`) if it is a diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/DSOMetadata.cs b/src/Xamarin.Android.Build.Tasks/Utilities/DSOMetadata.cs index d9f11fb1e4a..129b7ebc60f 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/DSOMetadata.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/DSOMetadata.cs @@ -4,12 +4,12 @@ public static class DSOMetadata { public const string Abi = "Abi"; public const string AndroidSkipCompression = "AndroidSkipCompression"; + public const string AssemblyLoadInfoIndex = "AssemblyLoadInfoIndex"; public const string Compressed = "Compressed"; public const string DataSize = "DataSize"; public const string DataSymbolOffset = "DataSymbolOffset"; public const string InputAssemblyPath = "InputAssemblyPath"; public const string OriginalAssemblyPath = "OriginalAssemblyPath"; - public const string AssemblyLoadInfoIndex = "AssemblyLoadInfoIndex"; public const string SatelliteAssemblyCulture = "SatelliteAssemblyCulture"; public const string SourceFileBaseName = "SourceFileBaseName"; public const string UncompressedDataSize = "UncompressedDataSize"; diff --git a/src/monodroid/jni/embedded-assemblies-zip.cc b/src/monodroid/jni/embedded-assemblies-zip.cc index b7ac4fc63b9..1c14906def9 100644 --- a/src/monodroid/jni/embedded-assemblies-zip.cc +++ b/src/monodroid/jni/embedded-assemblies-zip.cc @@ -225,7 +225,9 @@ EmbeddedAssemblies::zip_load_standalone_dso_entries (std::vector const& Helpers::abort_application (); } - xa_assemblies_load_info[index].apk_offset = state.data_offset; + AssemblyLoadInfo &load_info = xa_assemblies_load_info[index]; + load_info.apk_offset = state.data_offset; + load_info.apk_data_size = state.file_size; } } #endif diff --git a/src/monodroid/jni/embedded-assemblies.cc b/src/monodroid/jni/embedded-assemblies.cc index 1b5d2917e67..0f2841cda0f 100644 --- a/src/monodroid/jni/embedded-assemblies.cc +++ b/src/monodroid/jni/embedded-assemblies.cc @@ -126,8 +126,9 @@ namespace { force_inline static auto get_compressed_data_fastpath (AssemblyEntry const& entry, dynamic_local_string const& assembly_name) noexcept -> const AssemblyData { - log_debug (LOG_ASSEMBLY, "Assembly '%s' is compressed; input offset: %u; compressed data size: %u; uncompressed data size: %u; input offset: %u; output offset: %u", - assembly_name.get (), entry.input_data_offset, entry.input_data_size, entry.uncompressed_data_size, entry.input_data_offset, entry.uncompressed_data_offset); + log_debug (LOG_ASSEMBLY, "[fast path] assembly '%s' is compressed; input offset: %u; compressed data size: %u; uncompressed data size: %u; output offset: %u", + assembly_name.get (), entry.input_data_offset, entry.input_data_size, entry.uncompressed_data_size, entry.uncompressed_data_offset); + const uint8_t *const compressed_start = xa_input_assembly_data + entry.input_data_offset; uint8_t *dest = xa_uncompressed_assembly_data + entry.uncompressed_data_offset; decompress (compressed_start, entry.input_data_size, dest, entry.uncompressed_data_size, assembly_name); @@ -137,7 +138,28 @@ namespace { force_inline static auto get_uncompressed_data_fastpath (AssemblyEntry const& entry, dynamic_local_string const& assembly_name) noexcept -> const AssemblyData { - log_debug (LOG_ASSEMBLY, "Assembly %s' is not compressed; input offset: %u; data size: %u", assembly_name.get (), entry.input_data_offset, entry.input_data_size); + log_debug (LOG_ASSEMBLY, "[fast path] assembly %s' is not compressed; input offset: %u; data size: %u", assembly_name.get (), entry.input_data_offset, entry.input_data_size); + return {nullptr, 0}; + } + + force_inline + static auto get_compressed_data_dso (AssemblyEntry const& entry, const void *const data_start, dynamic_local_string const& assembly_name) noexcept -> const AssemblyData + { + log_debug (LOG_ASSEMBLY, "[slow path] assembly '%s' is compressed; input offset: %u; compressed data size: %u; uncompressed data size: %u; output offset: %u", + assembly_name.get (), entry.input_data_offset, entry.input_data_size, entry.uncompressed_data_size, entry.uncompressed_data_offset); + + // `data_start` points to the beginning of the .so file, `entry.input_data_offset` contains offset into + // that file where the actual assembly data begins + const uint8_t * const compressed_start = static_cast(data_start) + entry.input_data_offset; + uint8_t *dest = xa_uncompressed_assembly_data + entry.uncompressed_data_offset; + decompress (compressed_start, entry.input_data_size, dest, entry.uncompressed_data_size, assembly_name); + return {dest, entry.uncompressed_data_size}; + } + + force_inline + static auto get_uncompressed_data_dso (AssemblyEntry const& entry, void const * const data_start, dynamic_local_string const& assembly_name) noexcept -> const AssemblyData + { + log_debug (LOG_ASSEMBLY, "[slow path] assembly %s' is not compressed; input offset: %u; data size: %u", assembly_name.get (), entry.input_data_offset, entry.input_data_size); return {nullptr, 0}; } @@ -146,11 +168,27 @@ namespace { static auto get_data_fastpath (AssemblyEntry const& entry, dynamic_local_string const& assembly_name) noexcept -> const AssemblyData { log_debug (LOG_ASSEMBLY, "Loading assembly from libxamarin-app.so, fast path"); - if (entry.uncompressed_data_size > 0) { - return get_compressed_data_fastpath (entry, assembly_name); + if (entry.uncompressed_data_size == 0) { + return get_uncompressed_data_fastpath (entry, assembly_name); } - return get_uncompressed_data_fastpath (entry, assembly_name); + return get_compressed_data_fastpath (entry, assembly_name); + } + + force_inline + static auto get_data_dso (AssemblyEntry const& entry, AssemblyLoadInfo &load_info, dynamic_local_string const& assembly_name) noexcept -> const AssemblyData + { + log_debug (LOG_ASSEMBLY, "Loading assembly from a DSO, slow path"); + if (entry.uncompressed_data_size == 0) { + return get_uncompressed_data_dso (entry, load_info.mmap_addr, assembly_name); + } + + AssemblyData ret = get_compressed_data_dso (entry, load_info.mmap_addr, assembly_name); + munmap (load_info.mmap_addr, load_info.data_size); + load_info.data_addr = ret.data; + load_info.mmap_addr = nullptr; + + return ret; } force_inline @@ -168,15 +206,23 @@ namespace { // Returns `true` if data load lock has been acquired, `false` otherwise. Caller is responsible for releasing // the lock force_inline - static auto mmap_dso (AssemblyLoadInfo &load_info, int fd, off_t data_offset, size_t data_length) noexcept -> bool + static auto mmap_dso (AssemblyLoadInfo &load_info, int fd, uint32_t data_offset, uint32_t data_length, dynamic_local_string const& assembly_name) noexcept -> bool { if (__atomic_load_n (&load_info.mmap_addr, __ATOMIC_CONSUME) != nullptr) { return false; // lock not acquired } acquire_data_load_lock (); + if (load_info.mmap_addr != nullptr) { + log_debug (LOG_ASSEMBLY, "'%s' already mmapped from fd %d by some other thread while we were waiting for the lock. Map address: %p; size: %u", load_info.mmap_addr, load_info.data_size); + return true; // lock acquired + } + log_debug (LOG_ASSEMBLY, "Will mmap from fd %d, at offset %zd with length %zu", fd, data_offset, data_length); - // TODO: map here + + EmbeddedAssemblies::md_mmap_info map_info = EmbeddedAssemblies::md_mmap_apk_file (fd, data_offset, data_length, assembly_name.get ()); + load_info.mmap_addr = map_info.area; + load_info.data_size = data_length; return true; // lock acquired } @@ -201,22 +247,25 @@ namespace { AssemblyLoadInfo &load_info = xa_assemblies_load_info[index_entry.load_info_index]; log_debug (LOG_ASSEMBLY, "Looking for assembly in the APK; Data offset in the archive: %u; mmap address: %p; data address: %p; data size: %u", load_info.apk_offset, load_info.mmap_addr, load_info.data_addr, load_info.data_size); - if (__atomic_load_n (&load_info.data_addr, __ATOMIC_CONSUME) != nullptr) { - return {load_info.data_addr, load_info.data_size}; + const uint8_t *data = __atomic_load_n (&load_info.data_addr, __ATOMIC_CONSUME); + if (data != nullptr) { + log_debug (LOG_ASSEMBLY, "Assembly data already loaded, at %p; size: %u", load_info.data_addr, load_info.data_size); + return {data, load_info.data_size}; } - bool lock_acquired = detail::SO_AssemblyDataProvider::mmap_dso (load_info, apk_fd, static_cast(load_info.apk_offset), entry.input_data_size); + bool lock_acquired = detail::SO_AssemblyDataProvider::mmap_dso (load_info, apk_fd, load_info.apk_offset, load_info.apk_data_size, assembly_name); if (!lock_acquired) { detail::SO_AssemblyDataProvider::acquire_data_load_lock (); lock_acquired = true; - // TODO: actually load data here } + AssemblyData ret = detail::SO_AssemblyDataProvider::get_data_dso (entry, load_info, assembly_name); + if (lock_acquired) { detail::SO_AssemblyDataProvider::release_data_load_lock (); } - return {nullptr, 0}; + return ret; } }; @@ -951,7 +1000,7 @@ EmbeddedAssemblies::typemap_managed_to_java (MonoReflectionType *reflection_type } EmbeddedAssemblies::md_mmap_info -EmbeddedAssemblies::md_mmap_apk_file (int fd, uint32_t offset, size_t size, const char* filename) +EmbeddedAssemblies::md_mmap_apk_file (int fd, uint32_t offset, size_t size, const char* filename) noexcept { md_mmap_info file_info; md_mmap_info mmap_info; diff --git a/src/monodroid/jni/embedded-assemblies.hh b/src/monodroid/jni/embedded-assemblies.hh index 9c37139521a..3b012e70518 100644 --- a/src/monodroid/jni/embedded-assemblies.hh +++ b/src/monodroid/jni/embedded-assemblies.hh @@ -38,6 +38,12 @@ struct TypeMapHeader; +namespace { + namespace detail { + class SO_AssemblyDataProvider; + } +} + namespace xamarin::android::internal { #if defined (DEBUG) || !defined (ANDROID) struct TypeMappingInfo; @@ -78,6 +84,8 @@ namespace xamarin::android::internal { class EmbeddedAssemblies final { + friend class detail::SO_AssemblyDataProvider; + struct md_mmap_info { void *area; size_t size; @@ -223,7 +231,7 @@ namespace xamarin::android::internal { const TypeMapEntry *typemap_managed_to_java (const char *managed_type_name) noexcept; #endif // DEBUG || !ANDROID - static md_mmap_info md_mmap_apk_file (int fd, uint32_t offset, size_t size, const char* filename); + static md_mmap_info md_mmap_apk_file (int fd, uint32_t offset, size_t size, const char* filename) noexcept; static MonoAssembly* open_from_bundles_full (MonoAssemblyName *aname, char **assemblies_path, void *user_data); #if defined (NET) static MonoAssembly* open_from_bundles (MonoAssemblyLoadContextGCHandle alc_gchandle, MonoAssemblyName *aname, char **assemblies_path, void *user_data, MonoError *error); diff --git a/src/monodroid/jni/mono-image-loader.hh b/src/monodroid/jni/mono-image-loader.hh index 87bcbc2ed3a..fcd1b434e52 100644 --- a/src/monodroid/jni/mono-image-loader.hh +++ b/src/monodroid/jni/mono-image-loader.hh @@ -120,7 +120,7 @@ namespace xamarin::android::internal { #if defined (USE_CACHE) ssize_t index = find_index (hash); if (index < 0) { - log_warn (LOG_ASSEMBLY, "Failed to look up image index for hash 0x%zx", hash); + //log_warn (LOG_ASSEMBLY, "Failed to look up image index for hash 0x%zx", hash); return image; } diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index bbdcd715d4e..6254a262ef8 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -126,8 +126,9 @@ struct AssemblyLoadInfo { uint32_t apk_offset; // offset into the APK, or 0 if the assembly isn't in a standalone DSO or if the DSOs are // extracted to disk at install time + uint32_t apk_data_size; // Size of the DSO in the APK void *mmap_addr; // Address at which the assembly data was mmapped - uint8_t *data_addr; // Address at which the assembly data is available. It may be the same as `mmap_addr` if the + const uint8_t *data_addr; // Address at which the assembly data is available. It may be the same as `mmap_addr` if the // data wasn't compressed, different otherwise. uint32_t data_size; };