diff --git a/Directory.Build.props b/Directory.Build.props index 64033ca5019..57752f62665 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -49,7 +49,7 @@ 6.12.0.148 6.0.0 6.0.0 - 2.13.1 + 2.16.1 2.14.1 5.8.9.2 diff --git a/build-tools/installers/create-installers.targets b/build-tools/installers/create-installers.targets index 7080dc29fde..11c3d1b652e 100644 --- a/build-tools/installers/create-installers.targets +++ b/build-tools/installers/create-installers.targets @@ -251,12 +251,16 @@ <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)ManifestOverlays\Timing.xml" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-arm64\libc.so" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-arm64\libm.so" /> + <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-arm64\libassembly-blob.so" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-arm\libc.so" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-arm\libm.so" /> + <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-arm\libassembly-blob.so" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-x64\libc.so" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-x64\libm.so" /> + <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-x64\libassembly-blob.so" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-x86\libc.so" /> <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-x86\libm.so" /> + <_MSBuildFiles Include="$(MicrosoftAndroidSdkOutDir)libstubs\android-x86\libassembly-blob.so" /> <_MSBuildTargetsSrcFiles Include="$(MSBuildTargetsSrcDir)\Xamarin.Android.AvailableItems.targets" /> diff --git a/build-tools/xaprepare/xaprepare/ConfigAndData/BuildAndroidPlatforms.cs b/build-tools/xaprepare/xaprepare/ConfigAndData/BuildAndroidPlatforms.cs index 442a897ef6a..24675bcba58 100644 --- a/build-tools/xaprepare/xaprepare/ConfigAndData/BuildAndroidPlatforms.cs +++ b/build-tools/xaprepare/xaprepare/ConfigAndData/BuildAndroidPlatforms.cs @@ -5,8 +5,8 @@ namespace Xamarin.Android.Prepare { class BuildAndroidPlatforms { - public const string AndroidNdkVersion = "26"; - public const string AndroidNdkPkgRevision = "26.0.10792818"; + public const string AndroidNdkVersion = "26b"; + public const string AndroidNdkPkgRevision = "26.1.10909125"; public const int NdkMinimumAPI = 21; public const int NdkMinimumAPILegacy32 = 21; diff --git a/src/Microsoft.Android.Sdk.ILLink/PreserveLists/System.Private.CoreLib.xml b/src/Microsoft.Android.Sdk.ILLink/PreserveLists/System.Private.CoreLib.xml new file mode 100644 index 00000000000..c5aea679db6 --- /dev/null +++ b/src/Microsoft.Android.Sdk.ILLink/PreserveLists/System.Private.CoreLib.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/src/Xamarin.Android.Build.Tasks/Linker/MonoDroid.Tuner/AddRidMetadataAttributeStep.cs b/src/Xamarin.Android.Build.Tasks/Linker/MonoDroid.Tuner/AddRidMetadataAttributeStep.cs new file mode 100644 index 00000000000..6ee4fe3d2e7 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Linker/MonoDroid.Tuner/AddRidMetadataAttributeStep.cs @@ -0,0 +1,100 @@ +using System; + +using Mono.Cecil; +using Mono.Linker; +using Mono.Linker.Steps; + +#if ILLINK +using Microsoft.Android.Sdk.ILLink; +#endif + +namespace MonoDroid.Tuner; + +public class AddRidMetadataAttributeStep : BaseStep +{ + protected override void ProcessAssembly (AssemblyDefinition assembly) + { + if (!Annotations.HasAction (assembly)) { + return; + } + + var action = Annotations.GetAction (assembly); + if (action == AssemblyAction.Skip || action == AssemblyAction.Delete) { + return; + } + + string? rid = null; +#if ILLINK + if (!Context.TryGetCustomData ("XARuntimeIdentifier", out rid)) { + throw new InvalidOperationException ("Missing XARuntimeIdentifier custom data"); + } +#endif + if (String.IsNullOrEmpty (rid)) { + throw new InvalidOperationException ("RID must have a non-empty value"); + } + + AssemblyDefinition corlib = GetCorlib (); + MethodDefinition assemblyMetadataAttributeCtor = FindAssemblyMetadataAttributeCtor (corlib); + TypeDefinition systemString = GetSystemString (corlib); + + var attr = new CustomAttribute (assembly.MainModule.ImportReference (assemblyMetadataAttributeCtor)); + attr.ConstructorArguments.Add (new CustomAttributeArgument (systemString, "XamarinAndroidAbi")); // key + + // TODO: figure out how to get the RID... + attr.ConstructorArguments.Add (new CustomAttributeArgument (systemString, rid)); // value + + assembly.CustomAttributes.Add (attr); + + if (action == AssemblyAction.Copy) { + Annotations.SetAction (assembly, AssemblyAction.Save); + } + } + + TypeDefinition GetSystemString (AssemblyDefinition asm) => FindType (asm, "System.String", required: true); + + AssemblyDefinition GetCorlib () + { + const string ImportAssembly = "System.Private.CoreLib"; + AssemblyDefinition? asm = Context.Resolve (AssemblyNameReference.Parse (ImportAssembly)); + if (asm == null) { + throw new InvalidOperationException ($"Unable to import assembly '{ImportAssembly}'"); + } + + return asm; + } + + MethodDefinition FindAssemblyMetadataAttributeCtor (AssemblyDefinition asm) + { + const string AttributeType = "System.Reflection.AssemblyMetadataAttribute"; + + TypeDefinition assemblyMetadataAttribute = FindType (asm, AttributeType, required: true); + foreach (MethodDefinition md in assemblyMetadataAttribute!.Methods) { + if (!md.IsConstructor) { + continue; + } + + return md; + } + + throw new InvalidOperationException ($"Unable to find the {AttributeType} type constructor"); + } + + TypeDefinition? FindType (AssemblyDefinition asm, string typeName, bool required) + { + foreach (ModuleDefinition md in asm.Modules) { + foreach (TypeDefinition et in md.Types) { + if (String.Compare (typeName, et.FullName, StringComparison.Ordinal) != 0) { + continue; + } + + return et; + } + } + + if (required) { + throw new InvalidOperationException ($"Internal error: required type '{typeName}' in assembly {asm} not found"); + } + + return null; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.After.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.After.targets index 662aff0130b..aa9d99dd4c8 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.After.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.After.targets @@ -26,4 +26,5 @@ This file is imported *after* the Microsoft.NET.Sdk/Sdk.targets. + diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.BuildOrder.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.BuildOrder.targets index 346c89aa99b..4b47587b3b4 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.BuildOrder.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.BuildOrder.targets @@ -77,6 +77,35 @@ properties that determine build ordering. _IncludeNativeSystemLibraries; _CheckGoogleSdkRequirements; + + <_GenerateJavaStubsDependsOnTargets> + _SetLatestTargetFrameworkVersion; + _PrepareAssemblies; + _PrepareNativeAssemblySources; + $(_AfterPrepareAssemblies); + + + <_GeneratePackageManagerJavaDependsOn> + _GenerateJavaStubs; + _RunAotForAllRIDs; + _ManifestMerger; + _ConvertCustomView; + $(_AfterConvertCustomView); + _AddStaticResources; + $(_AfterAddStaticResources); + _PrepareAssemblies; + _PrepareEnvironmentAssemblySources; + _GenerateEnvironmentFiles; + _GenerateAndroidRemapNativeCode; + _GenerateEmptyAndroidRemapNativeCode; + _IncludeNativeSystemLibraries; + + + <_GenerateAndroidRemapNativeCodeDependsOn> + _ConvertAndroidMamMappingFileToXml; + _CollectAndroidRemapMembers; + _PrepareAndroidRemapNativeAssemblySources + diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.ILLink.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.ILLink.targets index 7400de192ff..0461633d57b 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.ILLink.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.ILLink.targets @@ -40,6 +40,7 @@ This file contains the .NET 5-specific targets to customize ILLink https://github.com/dotnet/sdk/blob/a5393731b5b7b225692fff121f747fbbc9e8b140/src/Tasks/Microsoft.NET.Build.Tasks/targets/Microsoft.NET.ILLink.targets#L147 --> <_TrimmerCustomData Include="XATargetFrameworkDirectories" Value="$(_XATargetFrameworkDirectories)" /> + <_TrimmerCustomData Include="XARuntimeIdentifier" Value="$(RuntimeIdentifier)" Condition=" '$(_AndroidAddRuntimeIdentifierToAssemblies)' == 'true' " /> <_TrimmerCustomData Condition=" '$(_ProguardProjectConfiguration)' != '' " Include="ProguardConfiguration" @@ -94,6 +95,12 @@ This file contains the .NET 5-specific targets to customize ILLink BeforeStep="MarkStep" Type="MonoDroid.Tuner.FixLegacyResourceDesignerStep" /> + <_TrimmerCustomSteps + Condition=" '$(_AndroidAddRuntimeIdentifierToAssemblies)' == 'true' " + Include="$(_AndroidLinkerCustomStepAssembly)" + AfterStep="CleanStep" + Type="MonoDroid.Tuner.AddRidMetadataAttributeStep" + /> <_PreserveLists Include="$(MSBuildThisFileDirectory)..\PreserveLists\*.xml" /> + + + + + + + + <_AndroidUseAssemblySharedLibraries Condition=" '$(EmbedAssembliesIntoApk)' != 'true' or '$(AndroidIncludeDebugSymbols)' == 'true' ">false + <_AndroidUseAssemblySharedLibraries Condition=" '$(_AndroidUseAssemblySharedLibraries)' == '' ">true + + <_AndroidApplicationSharedLibraryPath>$(IntermediateOutputPath)app_shared_libraries\ + <_CompressedAssembliesOutputDir>$(IntermediateOutputPath)android\lz4-2 + + <_AndroidKeepStandaloneAssemblySources Condition=" '$(_AndroidKeepStandaloneAssemblySources)' == '' ">false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <_NativeAssemblyTarget Include="@(_TypeMapAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> + %(_TypeMapAssemblySource.abi) + + + <_NativeAssemblyTarget Include="@(_EnvironmentAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> + %(_EnvironmentAssemblySource.abi) + + + <_NativeAssemblyTarget Include="@(_MarshalMethodsAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> + %(_MarshalMethodsAssemblySource.abi) + + + <_NativeAssemblyTarget Include="@(_AndroidRemapAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> + %(_AndroidRemapAssemblySource.abi) + + + <_NativeAssemblyTarget Include="@(_AssemblyDSOSourceApplication->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> + %(_AssemblyDSOSourceApplication.abi) + + + + + + + <_ApplicationSharedLibrary Include="$(_AndroidApplicationSharedLibraryPath)%(_BuildTargetAbis.Identity)\libxamarin-app.so"> + %(_BuildTargetAbis.Identity) + + + + + + + + + + + + + + + + + + + + + + + + + <_CreateApplicationSharedLibrariesDependsOn> + _PrepareAssemblyDSOSources; + _CompileApplicationNativeAssemblySources; + _PrepareApplicationSharedLibraryItems + + + + + + + + + + diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index fbcaa1e0944..aa9cf1882f5 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -83,8 +83,6 @@ public class BuildApk : AndroidTask public string TlsProvider { get; set; } public string UncompressedFileExtensions { get; set; } - // Make it required after https://github.com/xamarin/monodroid/pull/1094 is merged - //[Required] public bool EnableCompression { get; set; } public bool IncludeWrapSh { get; set; } @@ -95,6 +93,8 @@ public class BuildApk : AndroidTask public bool UseAssemblyStore { get; set; } + public bool UseAssemblySharedLibraries { get; set; } + public string ZipFlushFilesLimit { get; set; } public string ZipFlushSizeLimit { get; set; } @@ -130,7 +130,7 @@ protected virtual void FixupArchive (ZipArchiveEx zip) { } List excludePatterns = new List (); - void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOutputPath, bool debug, bool compress, IDictionary compressedAssembliesInfo, string assemblyStoreApkName) + void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOutputPath, bool debug, string assemblyStoreApkName) { ArchiveFileList files = new ArchiveFileList (); bool refresh = true; @@ -197,7 +197,7 @@ void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOut } if (EmbedAssemblies) { - AddAssemblies (apk, debug, compress, compressedAssembliesInfo, assemblyStoreApkName); + AddAssemblies (apk, debug, assemblyStoreApkName); apk.Flush (); } @@ -312,18 +312,8 @@ public override bool RunTask () } bool debug = _Debug; - bool compress = !debug && EnableCompression; - IDictionary compressedAssembliesInfo = null; - - if (compress) { - string key = CompressedAssemblyInfo.GetKey (ProjectFullPath); - Log.LogDebugMessage ($"Retrieving assembly compression info with key '{key}'"); - compressedAssembliesInfo = BuildEngine4.UnregisterTaskObjectAssemblyLocal> (key, RegisteredTaskObjectLifetime.Build); - if (compressedAssembliesInfo == null) - throw new InvalidOperationException ($"Assembly compression info not found for key '{key}'. Compression will not be performed."); - } - ExecuteWithAbi (SupportedAbis, ApkInputPath, ApkOutputPath, debug, compress, compressedAssembliesInfo, assemblyStoreApkName: null); + ExecuteWithAbi (SupportedAbis, ApkInputPath, ApkOutputPath, debug, assemblyStoreApkName: null); outputFiles.Add (ApkOutputPath); if (CreatePackagePerAbi && SupportedAbis.Length > 1) { foreach (var abi in SupportedAbis) { @@ -332,7 +322,7 @@ public override bool RunTask () var apk = Path.GetFileNameWithoutExtension (ApkOutputPath); ExecuteWithAbi (new [] { abi }, String.Format ("{0}-{1}", ApkInputPath, abi), Path.Combine (path, String.Format ("{0}-{1}.apk", apk, abi)), - debug, compress, compressedAssembliesInfo, assemblyStoreApkName: abi); + debug, assemblyStoreApkName: abi); outputFiles.Add (Path.Combine (path, String.Format ("{0}-{1}.apk", apk, abi))); } } @@ -362,34 +352,14 @@ static Regex FileGlobToRegEx (string fileGlob, RegexOptions options) return new Regex (sb.ToString (), options); } - void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary compressedAssembliesInfo, string assemblyStoreApkName) + void AddAssemblies (ZipArchiveEx apk, bool debug, string assemblyStoreApkName) { - string sourcePath; - AssemblyCompression.AssemblyData compressedAssembly = null; - string compressedOutputDir = Path.GetFullPath (Path.Combine (Path.GetDirectoryName (ApkOutputPath), "..", "lz4")); - AssemblyStoreGenerator storeGenerator; - - if (UseAssemblyStore) { - storeGenerator = new AssemblyStoreGenerator (AssembliesPath, Log); - } else { - storeGenerator = null; + if (UseAssemblySharedLibraries) { + // All the assemblies are in shared libraries + return; } - AssemblyStoreAssemblyInfo storeAssembly = null; - - // - // NOTE - // - // The very first store (ID 0) **must** contain an index of all the assemblies included in the application, even if they - // are included in other APKs than the base one. The ID 0 store **must** be placed in the base assembly - // - - // Currently, all the assembly stores end up in the "base" apk (the APK name is the key in the dictionary below) but the code is ready for the time when we - // partition assemblies into "feature" APKs - const string DefaultBaseApkName = "base"; - if (String.IsNullOrEmpty (assemblyStoreApkName)) { - assemblyStoreApkName = DefaultBaseApkName; - } + string sourcePath; // Add user assemblies AddAssembliesFromCollection (ResolvedUserAssemblies); @@ -397,37 +367,6 @@ void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary> assemblyStorePaths = storeGenerator.Generate (Path.GetDirectoryName (ApkOutputPath)); - if (assemblyStorePaths == null) { - throw new InvalidOperationException ("Assembly store generator did not generate any stores"); - } - - if (!assemblyStorePaths.TryGetValue (assemblyStoreApkName, out List baseAssemblyStores) || baseAssemblyStores == null || baseAssemblyStores.Count == 0) { - throw new InvalidOperationException ("Assembly store generator didn't generate the required base stores"); - } - - string assemblyStorePrefix = $"{assemblyStoreApkName}_"; - foreach (string assemblyStorePath in baseAssemblyStores) { - string inArchiveName = Path.GetFileName (assemblyStorePath); - - if (inArchiveName.StartsWith (assemblyStorePrefix, StringComparison.Ordinal)) { - inArchiveName = inArchiveName.Substring (assemblyStorePrefix.Length); - } - - CompressionMethod compressionMethod; - if (inArchiveName.EndsWith (".manifest", StringComparison.Ordinal)) { - compressionMethod = CompressionMethod.Default; - } else { - compressionMethod = UncompressedMethod; - } - - AddFileToArchiveIfNewer (apk, assemblyStorePath, AssembliesPath + inArchiveName, compressionMethod); - } - void AddAssembliesFromCollection (ITaskItem[] assemblies) { foreach (ITaskItem assembly in assemblies) { @@ -440,23 +379,15 @@ void AddAssembliesFromCollection (ITaskItem[] assemblies) Log.LogCodedWarning ("XA0107", assembly.ItemSpec, 0, Properties.Resources.XA0107, assembly.ItemSpec); } - sourcePath = CompressAssembly (assembly); + sourcePath = assembly.ItemSpec; // Add assembly var assemblyPath = GetAssemblyPath (assembly, frameworkAssembly: false); - if (UseAssemblyStore) { - storeAssembly = new AssemblyStoreAssemblyInfo (sourcePath, assemblyPath, assembly.GetMetadata ("Abi")); - } else { - AddFileToArchiveIfNewer (apk, sourcePath, assemblyPath + Path.GetFileName (assembly.ItemSpec), compressionMethod: UncompressedMethod); - } + AddFileToArchiveIfNewer (apk, sourcePath, assemblyPath + Path.GetFileName (assembly.ItemSpec), compressionMethod: UncompressedMethod); // Try to add config if exists var config = Path.ChangeExtension (assembly.ItemSpec, "dll.config"); - if (UseAssemblyStore) { - storeAssembly.SetConfigPath (config); - } else { - AddAssemblyConfigEntry (apk, assemblyPath, config); - } + AddAssemblyConfigEntry (apk, assemblyPath, config); // Try to add symbols if Debug if (debug) { @@ -468,72 +399,11 @@ void AddAssembliesFromCollection (ITaskItem[] assemblies) } if (!String.IsNullOrEmpty (symbolsPath)) { - if (UseAssemblyStore) { - storeAssembly.SetDebugInfoPath (symbolsPath); - } else { - AddFileToArchiveIfNewer (apk, symbolsPath, assemblyPath + Path.GetFileName (symbols), compressionMethod: UncompressedMethod); - } + AddFileToArchiveIfNewer (apk, symbolsPath, assemblyPath + Path.GetFileName (symbols), compressionMethod: UncompressedMethod); } } - - if (UseAssemblyStore) { - storeGenerator.Add (assemblyStoreApkName, storeAssembly); - } } } - - void EnsureCompressedAssemblyData (string sourcePath, uint descriptorIndex) - { - if (compressedAssembly == null) - compressedAssembly = new AssemblyCompression.AssemblyData (sourcePath, descriptorIndex); - else - compressedAssembly.SetData (sourcePath, descriptorIndex); - } - - string CompressAssembly (ITaskItem assembly) - { - if (!compress) { - return assembly.ItemSpec; - } - - if (bool.TryParse (assembly.GetMetadata ("AndroidSkipCompression"), out bool value) && value) { - Log.LogDebugMessage ($"Skipping compression of {assembly.ItemSpec} due to 'AndroidSkipCompression' == 'true' "); - return assembly.ItemSpec; - } - - var key = CompressedAssemblyInfo.GetDictionaryKey (assembly); - if (compressedAssembliesInfo.TryGetValue (key, out CompressedAssemblyInfo info) && info != null) { - EnsureCompressedAssemblyData (assembly.ItemSpec, info.DescriptorIndex); - string assemblyOutputDir; - string subDirectory = assembly.GetMetadata ("DestinationSubDirectory"); - if (!String.IsNullOrEmpty (subDirectory)) - assemblyOutputDir = Path.Combine (compressedOutputDir, subDirectory); - else - assemblyOutputDir = compressedOutputDir; - AssemblyCompression.CompressionResult result = AssemblyCompression.Compress (compressedAssembly, assemblyOutputDir); - if (result != AssemblyCompression.CompressionResult.Success) { - switch (result) { - case AssemblyCompression.CompressionResult.EncodingFailed: - Log.LogMessage ($"Failed to compress {assembly.ItemSpec}"); - break; - - case AssemblyCompression.CompressionResult.InputTooBig: - Log.LogMessage ($"Input assembly {assembly.ItemSpec} exceeds maximum input size"); - break; - - default: - Log.LogMessage ($"Unknown error compressing {assembly.ItemSpec}"); - break; - } - return assembly.ItemSpec; - } - return compressedAssembly.DestinationPath; - } else { - Log.LogDebugMessage ($"Assembly missing from {nameof (CompressedAssemblyInfo)}: {key}"); - } - - return assembly.ItemSpec; - } } bool AddFileToArchiveIfNewer (ZipArchiveEx apk, string file, string inArchivePath, CompressionMethod compressionMethod = CompressionMethod.Default) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/CompileNativeAssembly.cs b/src/Xamarin.Android.Build.Tasks/Tasks/CompileNativeAssembly.cs index 2dca284c933..45d81a6b1c5 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/CompileNativeAssembly.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/CompileNativeAssembly.cs @@ -1,14 +1,7 @@ -using System; using System.Collections.Generic; -using System.Diagnostics; using System.IO; -using System.Threading; using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; - -using Xamarin.Android.Tools; -using Xamarin.Build; using Microsoft.Android.Build.Tasks; namespace Xamarin.Android.Tasks @@ -17,13 +10,6 @@ public class CompileNativeAssembly : AndroidAsyncTask { public override string TaskPrefix => "CNA"; - sealed class Config - { - public string AssemblerPath; - public string AssemblerOptions; - public string InputSource; - } - [Required] public ITaskItem[] Sources { get; set; } @@ -38,109 +24,29 @@ sealed class Config public override System.Threading.Tasks.Task RunTaskAsync () { - return this.WhenAll (GetAssemblerConfigs (), RunAssembler); + return this.WhenAll (GetAssemblerConfigs (), (NativeCompilationHelper.AssemblerConfig config) => NativeCompilationHelper.RunAssembler (config)); } - void RunAssembler (Config config) + IEnumerable GetAssemblerConfigs () { - var stdout_completed = new ManualResetEvent (false); - var stderr_completed = new ManualResetEvent (false); - var psi = new ProcessStartInfo () { - FileName = config.AssemblerPath, - Arguments = config.AssemblerOptions, - UseShellExecute = false, - RedirectStandardOutput = true, - RedirectStandardError = true, - CreateNoWindow = true, - WindowStyle = ProcessWindowStyle.Hidden, - WorkingDirectory = WorkingDirectory, - }; - - string assemblerName = Path.GetFileName (config.AssemblerPath); - LogDebugMessage ($"[LLVM llc] {psi.FileName} {psi.Arguments}"); - - var stdoutLines = new List (); - var stderrLines = new List (); - - using (var proc = new Process ()) { - proc.OutputDataReceived += (s, e) => { - if (e.Data != null) { - OnOutputData (assemblerName, s, e); - stdoutLines.Add (e.Data); - } else - stdout_completed.Set (); - }; - - proc.ErrorDataReceived += (s, e) => { - if (e.Data != null) { - OnErrorData (assemblerName, s, e); - stderrLines.Add (e.Data); - } else - stderr_completed.Set (); - }; - - proc.StartInfo = psi; - proc.Start (); - proc.BeginOutputReadLine (); - proc.BeginErrorReadLine (); - CancellationToken.Register (() => { try { proc.Kill (); } catch (Exception) { } }); - proc.WaitForExit (); - - if (psi.RedirectStandardError) - stderr_completed.WaitOne (TimeSpan.FromSeconds (30)); - - if (psi.RedirectStandardOutput) - stdout_completed.WaitOne (TimeSpan.FromSeconds (30)); - - if (proc.ExitCode != 0) { - var sb = MonoAndroidHelper.MergeStdoutAndStderrMessages (stdoutLines, stderrLines); - LogCodedError ("XA3006", Properties.Resources.XA3006, Path.GetFileName (config.InputSource), sb.ToString ()); - Cancel (); - } - } - } - - IEnumerable GetAssemblerConfigs () - { - const string assemblerOptions = - "-O2 " + - "--debugger-tune=lldb " + // NDK uses lldb now - "--debugify-level=location+variables " + - "--fatal-warnings " + - "--filetype=obj " + - "--relocation-model=pic"; - string llcPath = Path.Combine (AndroidBinUtilsDirectory, "llc"); + string assemblerPath = NativeCompilationHelper.GetAssemblerPath (AndroidBinUtilsDirectory); + string workingDirectory = Path.GetFullPath (WorkingDirectory); foreach (ITaskItem item in Sources) { // We don't need the directory since our WorkingDirectory is where all the sources are string sourceFile = Path.GetFileName (item.ItemSpec); - string outputFile = QuoteFileName (sourceFile.Replace (".ll", ".o")); - string executableDir = Path.GetDirectoryName (llcPath); - string executableName = MonoAndroidHelper.GetExecutablePath (executableDir, Path.GetFileName (llcPath)); - yield return new Config { - InputSource = item.ItemSpec, - AssemblerPath = Path.Combine (executableDir, executableName), - AssemblerOptions = $"{assemblerOptions} -o={outputFile} {QuoteFileName (sourceFile)}", + yield return new NativeCompilationHelper.AssemblerConfig ( + log: Log, + targetArch: MonoAndroidHelper.AbiToTargetArch (item.GetMetadata ("abi")), + assemblerPath: assemblerPath, + inputSource: sourceFile, + workingDirectory: workingDirectory + ) { + CancellationToken = CancellationToken, + Cancel = () => Cancel (), }; } } - - void OnOutputData (string assemblerName, object sender, DataReceivedEventArgs e) - { - LogDebugMessage ($"[{assemblerName} stdout] {e.Data}"); - } - - void OnErrorData (string assemblerName, object sender, DataReceivedEventArgs e) - { - LogMessage ($"[{assemblerName} stderr] {e.Data}", MessageImportance.High); - } - - static string QuoteFileName (string fileName) - { - var builder = new CommandLineBuilder (); - builder.AppendFileNameIfNotNull (fileName); - return builder.ToString (); - } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs deleted file mode 100644 index 48f849596e0..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs +++ /dev/null @@ -1,101 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using Microsoft.Build.Framework; -using Microsoft.Android.Build.Tasks; - -namespace Xamarin.Android.Tasks -{ - public class GenerateCompressedAssembliesNativeSourceFiles : AndroidTask - { - public override string TaskPrefix => "GCANSF"; - - [Required] - public ITaskItem[] ResolvedAssemblies { get; set; } - - [Required] - public string [] SupportedAbis { get; set; } - - [Required] - public string EnvironmentOutputDirectory { get; set; } - - [Required] - public bool Debug { get; set; } - - [Required] - public bool EnableCompression { get; set; } - - [Required] - public string ProjectFullPath { get; set; } - - public override bool RunTask () - { - GenerateCompressedAssemblySources (); - return !Log.HasLoggedErrors; - } - - void GenerateCompressedAssemblySources () - { - if (Debug || !EnableCompression) { - Generate (null); - return; - } - - var assemblies = new SortedDictionary (StringComparer.Ordinal); - foreach (ITaskItem assembly in ResolvedAssemblies) { - if (bool.TryParse (assembly.GetMetadata ("AndroidSkipAddToPackage"), out bool value) && value) { - continue; - } - - var assemblyKey = CompressedAssemblyInfo.GetDictionaryKey (assembly); - if (assemblies.ContainsKey (assemblyKey)) { - Log.LogDebugMessage ($"Skipping duplicate assembly: {assembly.ItemSpec}"); - continue; - } - - var fi = new FileInfo (assembly.ItemSpec); - if (!fi.Exists) { - Log.LogError ($"Assembly {assembly.ItemSpec} does not exist"); - continue; - } - - assemblies.Add (assemblyKey, new CompressedAssemblyInfo (checked((uint)fi.Length))); - } - - uint index = 0; - foreach (var kvp in assemblies) { - kvp.Value.DescriptorIndex = index++; - } - - string key = CompressedAssemblyInfo.GetKey (ProjectFullPath); - Log.LogDebugMessage ($"Storing compression assemblies info with key '{key}'"); - BuildEngine4.RegisterTaskObjectAssemblyLocal (key, assemblies, RegisteredTaskObjectLifetime.Build); - Generate (assemblies); - - void Generate (IDictionary dict) - { - var composer = new CompressedAssembliesNativeAssemblyGenerator (dict); - LLVMIR.LlvmIrModule compressedAssemblies = composer.Construct (); - - foreach (string abi in SupportedAbis) { - string baseAsmFilePath = Path.Combine (EnvironmentOutputDirectory, $"compressed_assemblies.{abi.ToLowerInvariant ()}"); - string llvmIrFilePath = $"{baseAsmFilePath}.ll"; - - using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) { - try { - composer.Generate (compressedAssemblies, GeneratePackageManagerJava.GetAndroidTargetArchForAbi (abi), sw, llvmIrFilePath); - } catch { - throw; - } finally { - sw.Flush (); - } - - if (Files.CopyIfStreamChanged (sw.BaseStream, llvmIrFilePath)) { - Log.LogDebugMessage ($"File {llvmIrFilePath} was regenerated"); - } - } - } - } - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 80f1f25b279..c25d4b3432f 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -2,14 +2,12 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.IO; using System.IO.MemoryMappedFiles; using System.Linq; using System.Reflection; using System.Text; using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; using Mono.Cecil; @@ -27,6 +25,22 @@ namespace Xamarin.Android.Tasks public class GenerateJavaStubs : AndroidTask { + sealed class RunState + { + public XAAssemblyResolver Resolver { get; set; } + public ICollection JavaTypeAssemblies { get; set; } + public ICollection UserAssemblies { get; set; } + public InputAssemblySet AssemblySet { get; set; } + public bool UseMarshalMethods { get; set; } + public AndroidTargetArch TargetArch { get; set; } = AndroidTargetArch.None; + + /// + /// If `true`, generate code/data that doesn't depend on a specific RID (e.g. ACW maps or JCWs) + /// To be used once per multi-RID runs. + /// + public bool GenerateRidAgnosticParts { get; set; } + } + public const string MarshalMethodsRegisterTaskKey = ".:!MarshalMethods!:."; public override string TaskPrefix => "GJS"; @@ -101,12 +115,7 @@ public class GenerateJavaStubs : AndroidTask public override bool RunTask () { try { - bool useMarshalMethods = !Debug && EnableMarshalMethods; - // We're going to do 3 steps here instead of separate tasks so - // we can share the list of JLO TypeDefinitions between them - using (XAAssemblyResolver res = MakeResolver (useMarshalMethods)) { - Run (res, useMarshalMethods); - } + Run (); } catch (XamarinAndroidException e) { Log.LogCodedError (string.Format ("XA{0:0000}", e.Code), e.MessageWithoutCode); if (MonoAndroidHelper.LogInternalExceptions) @@ -141,23 +150,151 @@ XAAssemblyResolver MakeResolver (bool useMarshalMethods) return res; } - void Run (XAAssemblyResolver res, bool useMarshalMethods) + void Run () { - PackageNamingPolicy pnp; - JavaNativeTypeManager.PackageNamingPolicy = Enum.TryParse (PackageNamingPolicy, out pnp) ? pnp : PackageNamingPolicyEnum.LowercaseCrc64; + if (Debug) { + if (LinkingEnabled) { + RunDebugWithLinking (); + } else { + RunDebugNoLinking (); + } + return; + } - Dictionary>? abiSpecificAssembliesByPath = null; - if (useMarshalMethods) { - abiSpecificAssembliesByPath = new Dictionary> (StringComparer.Ordinal); + bool useMarshalMethods = !Debug && EnableMarshalMethods; + if (LinkingEnabled) { + RunReleaseWithLinking (useMarshalMethods); + } else { + RunReleaseNoLinking (useMarshalMethods); } + } + + // * We have one set of assemblies in general, some RID-specific ones (e.g. `System.Private.CoreLib`, potentially others). + // * Typemaps don't use MVIDs or metadata tokens, so we can process one each of the RID-specific ones together with the others + // * Marshal methods are never used + void RunDebugNoLinking () + { + LogRunMode ("Debug, no linking"); + + XAAssemblyResolver resolver = MakeResolver (useMarshalMethods: false); + var assemblies = CollectInterestingAssemblies (new RidAgnosticInputAssemblySet (), (AndroidTargetArch arch) => resolver); + var state = new RunState { + UseMarshalMethods = false, + AssemblySet = assemblies, + JavaTypeAssemblies = assemblies.JavaTypeAssemblies, + UserAssemblies = assemblies.UserAssemblies, + GenerateRidAgnosticParts = true, + Resolver = resolver, + TargetArch = AndroidTargetArch.None, + }; + DoRun (state, out ApplicationConfigTaskState appConfState); + RegisterApplicationConfigState (appConfState); + } - // Put every assembly we'll need in the resolver - bool hasExportReference = false; - bool haveMonoAndroid = false; - var allTypemapAssemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); - var userAssemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); + // * We have as many sets of assemblies as there are RIDs, all assemblies are RID-specific + // * Typemaps don't use MVIDs or metadata tokens, so we can process a single set of RID-specific assemblies + // * Marshal methods are never used + void RunDebugWithLinking () + { + LogRunMode ("Debug, with linking"); + + XAAssemblyResolver resolver = MakeResolver (useMarshalMethods: false); + var assemblies = CollectInterestingAssemblies (new RidSpecificInputAssemblySet (), (AndroidTargetArch arch) => resolver); + + AndroidTargetArch firstArch = assemblies.JavaTypeAssemblies.Keys.First (); + var state = new RunState { + UseMarshalMethods = false, + AssemblySet = assemblies, + JavaTypeAssemblies = assemblies.JavaTypeAssemblies[firstArch].Values, + UserAssemblies = assemblies.UserAssemblies[firstArch].Values, + GenerateRidAgnosticParts = true, + Resolver = resolver, + TargetArch = AndroidTargetArch.None, + }; + DoRun (state, out ApplicationConfigTaskState appConfState); + RegisterApplicationConfigState (appConfState); + } + + // * We have one set of assemblies in general, some RID-specific ones (e.g. `System.Private.CoreLib`, potentially others). + // * Typemaps use MVIDs and metadata tokens, so we need to process all assemblies as per-RID ones (different MVIDs in the + // actually RID-specific assemblies may affect sorting of the RID-agnostic ones) + // * Marshal methods may be used + void RunReleaseNoLinking (bool useMarshalMethods) + { + LogRunMode ("Release, no linking"); - foreach (var assembly in ResolvedAssemblies) { + Dictionary resolvers = MakeResolvers (useMarshalMethods); + + // All the RID-agnostic asseemblies will use resolvers of this architecture. This is because the RidAwareInputAssemblySet does not store + // such assemblies separately, but it copies them to **all** the target RIDs. This, in turn, is done because of typemaps and marshal methods + // which process data in a way that requires proper sorting of assemblies per MVID and it requires valid type and method token IDs. + AndroidTargetArch firstArch = resolvers.First ().Key; + resolvers.Add (AndroidTargetArch.None, resolvers[firstArch]); + var assemblies = CollectInterestingAssemblies (new RidAwareInputAssemblySet (resolvers.Keys), (AndroidTargetArch arch) => resolvers[arch]); + RunReleaseCommon (useMarshalMethods, assemblies, resolvers); + } + + // * We have as many sets of assemblies as there are RIDs, all assemblies are RID-specific + // * Typemaps use MVIDs and metadata tokens, so we need per-RID set processing + // * Marshal methods may be used + void RunReleaseWithLinking (bool useMarshalMethods) + { + LogRunMode ("Release, with linking"); + + Dictionary resolvers = MakeResolvers (useMarshalMethods); + var assemblies = CollectInterestingAssemblies (new RidSpecificInputAssemblySet (), (AndroidTargetArch arch) => resolvers[arch]); + RunReleaseCommon (useMarshalMethods, assemblies, resolvers); + } + + void RunReleaseCommon (bool useMarshalMethods, RidSensitiveInputAssemblySet assemblies, Dictionary resolvers) + { + bool first = true; + + foreach (var kvp in resolvers) { + var state = new RunState { + UseMarshalMethods = useMarshalMethods, + AssemblySet = assemblies, + JavaTypeAssemblies = assemblies.JavaTypeAssemblies[kvp.Key].Values, + UserAssemblies = assemblies.UserAssemblies[kvp.Key].Values, + GenerateRidAgnosticParts = first, + Resolver = kvp.Value, + TargetArch = kvp.Key, + }; + + DoRun (state, out ApplicationConfigTaskState appConfState); + if (first) { + RegisterApplicationConfigState (appConfState); + first = false; + } + } + } + + Dictionary MakeResolvers (bool useMarshalMethods) + { + var resolvers = new Dictionary (); + foreach (string abi in SupportedAbis) { + // Each ABI gets its own resolver in this mode... + XAAssemblyResolver resolver = MakeResolver (useMarshalMethods); + resolvers.Add (MonoAndroidHelper.AbiToTargetArch (abi), resolver); + } + + return resolvers; + } + + void RegisterApplicationConfigState (ApplicationConfigTaskState appConfState) + { + BuildEngine4.RegisterTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (ApplicationConfigTaskState.RegisterTaskObjectKey), appConfState, RegisteredTaskObjectLifetime.Build); + } + + void LogRunMode (string mode) + { + Log.LogDebugMessage ($"GenerateJavaStubs mode: {mode}"); + } + + T CollectInterestingAssemblies (T assemblies, Func getResolver) where T: InputAssemblySet + { + AndroidTargetArch targetArch; + foreach (ITaskItem assembly in ResolvedAssemblies) { bool value; if (bool.TryParse (assembly.GetMetadata (AndroidSkipJavaStubGeneration), out value) && value) { Log.LogDebugMessage ($"Skipping Java Stub Generation for {assembly.ItemSpec}"); @@ -166,173 +303,177 @@ void Run (XAAssemblyResolver res, bool useMarshalMethods) bool addAssembly = false; string fileName = Path.GetFileName (assembly.ItemSpec); - if (!hasExportReference && String.Compare ("Mono.Android.Export.dll", fileName, StringComparison.OrdinalIgnoreCase) == 0) { - hasExportReference = true; + if (String.Compare ("Mono.Android.Export.dll", fileName, StringComparison.OrdinalIgnoreCase) == 0) { addAssembly = true; - } else if (!haveMonoAndroid && String.Compare ("Mono.Android.dll", fileName, StringComparison.OrdinalIgnoreCase) == 0) { - haveMonoAndroid = true; + } else if (String.Compare ("Mono.Android.dll", fileName, StringComparison.OrdinalIgnoreCase) == 0) { addAssembly = true; } else if (MonoAndroidHelper.FrameworkAssembliesToTreatAsUserAssemblies.Contains (fileName)) { if (!bool.TryParse (assembly.GetMetadata (AndroidSkipJavaStubGeneration), out value) || !value) { string name = Path.GetFileNameWithoutExtension (fileName); - if (!userAssemblies.ContainsKey (name)) - userAssemblies.Add (name, assembly.ItemSpec); + assemblies.AddUserAssembly (assembly); addAssembly = true; } + } else if (MonoAndroidHelper.IsSatelliteAssembly (assembly)) { + continue; } if (addAssembly) { - MaybeAddAbiSpecifcAssembly (assembly, fileName); - if (!allTypemapAssemblies.ContainsKey (assembly.ItemSpec)) { - allTypemapAssemblies.Add (assembly.ItemSpec, assembly); - } + assemblies.AddJavaTypeAssembly (assembly); } - res.Load (MonoAndroidHelper.GetTargetArch (assembly), assembly.ItemSpec); + targetArch = MonoAndroidHelper.GetTargetArch (assembly); + + // We don't check whether we have a resolver for `targetArch` on purpose, if it throws then it means we have a bug which + // should be fixed since there shouldn't be any assemblies passed to this task that belong in ABIs other than those + // specified in `SupportedAbis` (and, perhaps, a RID-agnostic one) + try { + getResolver (targetArch).Load (targetArch, assembly.ItemSpec); + } catch (Exception ex) { + throw new InvalidOperationException ($"Internal error: failed to get resolver for assembly {assembly.ItemSpec}, target architecture '{targetArch}'", ex); + } } // However we only want to look for JLO types in user code for Java stub code generation - foreach (var asm in ResolvedUserAssemblies) { - if (bool.TryParse (asm.GetMetadata (AndroidSkipJavaStubGeneration), out bool value) && value) { - Log.LogDebugMessage ($"Skipping Java Stub Generation for {asm.ItemSpec}"); + foreach (ITaskItem assembly in ResolvedUserAssemblies) { + if (bool.TryParse (assembly.GetMetadata (AndroidSkipJavaStubGeneration), out bool value) && value) { + Log.LogDebugMessage ($"Skipping Java Stub Generation for {assembly.ItemSpec}"); continue; } - res.Load (MonoAndroidHelper.GetTargetArch (asm), asm.ItemSpec); - MaybeAddAbiSpecifcAssembly (asm, Path.GetFileName (asm.ItemSpec)); - if (!allTypemapAssemblies.ContainsKey (asm.ItemSpec)) { - allTypemapAssemblies.Add (asm.ItemSpec, asm); - } - string name = Path.GetFileNameWithoutExtension (asm.ItemSpec); - if (!userAssemblies.ContainsKey (name)) - userAssemblies.Add (name, asm.ItemSpec); + targetArch = MonoAndroidHelper.GetTargetArch (assembly); + getResolver (targetArch).Load (targetArch, assembly.ItemSpec); + + assemblies.AddJavaTypeAssembly (assembly); + assemblies.AddUserAssembly (assembly); + } + + return assemblies; + } + + void DoRun (RunState state, out ApplicationConfigTaskState? appConfState) + { + Log.LogDebugMessage ($"DoRun for arch {state.TargetArch}"); + Log.LogDebugMessage ("Java type assemblies:"); + foreach (ITaskItem assembly in state.JavaTypeAssemblies) { + Log.LogDebugMessage ($" {assembly.ItemSpec}"); + } + + Log.LogDebugMessage ("User assemblies:"); + foreach (ITaskItem assembly in state.UserAssemblies) { + Log.LogDebugMessage ($" {assembly.ItemSpec}"); } + PackageNamingPolicy pnp; + JavaNativeTypeManager.PackageNamingPolicy = Enum.TryParse (PackageNamingPolicy, out pnp) ? pnp : PackageNamingPolicyEnum.LowercaseCrc64; // Step 1 - Find all the JLO types var cache = new TypeDefinitionCache (); var scanner = new XAJavaTypeScanner (Log, cache) { ErrorOnCustomJavaObject = ErrorOnCustomJavaObject, }; - List allJavaTypes = scanner.GetJavaTypes (allTypemapAssemblies.Values, res); - var javaTypes = new List (); + ICollection allJavaTypes = scanner.GetJavaTypes (state.JavaTypeAssemblies, state.Resolver); + var javaTypes = new List (); - foreach (JavaType jt in allJavaTypes) { + foreach (TypeDefinition javaType in allJavaTypes) { // Whem marshal methods are in use we do not want to skip non-user assemblies (such as Mono.Android) - we need to generate JCWs for them during // application build, unlike in Debug configuration or when marshal methods are disabled, in which case we use JCWs generated during Xamarin.Android // build and stored in a jar file. - if ((!useMarshalMethods && !userAssemblies.ContainsKey (jt.Type.Module.Assembly.Name.Name)) || JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (jt.Type, cache)) { + if ((!state.UseMarshalMethods && !state.AssemblySet.IsUserAssembly (javaType.Module.Assembly.Name.Name)) || JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (javaType, cache)) { continue; } - javaTypes.Add (jt); + javaTypes.Add (javaType); } - MarshalMethodsClassifier classifier = null; - if (useMarshalMethods) { - classifier = new MarshalMethodsClassifier (cache, res, Log); + MarshalMethodsClassifier? classifier = null; + if (state.UseMarshalMethods) { + classifier = new MarshalMethodsClassifier (cache, state.Resolver, Log); } - // Step 2 - Generate Java stub code - var success = CreateJavaSources (javaTypes, cache, classifier, useMarshalMethods); - if (!success) - return; + // TODO: JCWs don't need to be generated for every RID, but we do need the classifier for the marshal methods + // rewriter and generator. Add a mode to only classify marshal methods without generating the files. + // For now, always generate the JCWs if marshal methods are enabled + if (state.UseMarshalMethods || state.GenerateRidAgnosticParts) { + // Step 2 - Generate Java stub code + bool success = CreateJavaSources (javaTypes, cache, classifier, state.UseMarshalMethods); + if (!success) { + appConfState = null; + return; // TODO: throw? Return `false`? + } + } - if (useMarshalMethods) { + if (state.UseMarshalMethods) { // We need to parse the environment files supplied by the user to see if they want to use broken exception transitions. This information is needed // in order to properly generate wrapper methods in the marshal methods assembly rewriter. // We don't care about those generated by us, since they won't contain the `XA_BROKEN_EXCEPTION_TRANSITIONS` variable we look for. var environmentParser = new EnvironmentFilesParser (); - - Dictionary assemblyPaths = AddMethodsFromAbiSpecificAssemblies (classifier, res, abiSpecificAssembliesByPath); - - var rewriter = new MarshalMethodsAssemblyRewriter (classifier.MarshalMethods, classifier.Assemblies, assemblyPaths, Log); - rewriter.Rewrite (res, environmentParser.AreBrokenExceptionTransitionsEnabled (Environments)); + var rewriter = new MarshalMethodsAssemblyRewriter (classifier.MarshalMethods, classifier.Assemblies, Log); + rewriter.Rewrite (state.Resolver, environmentParser.AreBrokenExceptionTransitionsEnabled (Environments)); } // Step 3 - Generate type maps // Type mappings need to use all the assemblies, always. - WriteTypeMappings (allJavaTypes, cache); + WriteTypeMappings (state.TargetArch, allJavaTypes, cache, out appConfState); - // We need to save a map of .NET type -> ACW type for resource file fixups - var managed = new Dictionary (javaTypes.Count, StringComparer.Ordinal); - var java = new Dictionary (javaTypes.Count, StringComparer.Ordinal); + if (state.GenerateRidAgnosticParts) { + WriteAcwMaps (javaTypes, cache); - var managedConflicts = new Dictionary> (0, StringComparer.Ordinal); - var javaConflicts = new Dictionary> (0, StringComparer.Ordinal); + // Step 4 - Merge [Activity] and friends into AndroidManifest.xml + UpdateAndroidManifest (state, cache, allJavaTypes); + CreateAdditionalJavaSources (javaTypes, cache, classifier); + } - using (var acw_map = MemoryStreamPool.Shared.CreateStreamWriter ()) { - foreach (JavaType jt in javaTypes) { - TypeDefinition type = jt.Type; - string managedKey = type.FullName.Replace ('/', '.'); - string javaKey = JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.'); + if (state.UseMarshalMethods) { + classifier.AddSpecialCaseMethods (); - acw_map.Write (type.GetPartialAssemblyQualifiedName (cache)); - acw_map.Write (';'); - acw_map.Write (javaKey); - acw_map.WriteLine (); - - TypeDefinition conflict; - bool hasConflict = false; - if (managed.TryGetValue (managedKey, out conflict)) { - if (!conflict.Module.Name.Equals (type.Module.Name)) { - if (!managedConflicts.TryGetValue (managedKey, out var list)) - managedConflicts.Add (managedKey, list = new List { conflict.GetPartialAssemblyName (cache) }); - list.Add (type.GetPartialAssemblyName (cache)); - } - hasConflict = true; - } - if (java.TryGetValue (javaKey, out conflict)) { - if (!conflict.Module.Name.Equals (type.Module.Name)) { - if (!javaConflicts.TryGetValue (javaKey, out var list)) - javaConflicts.Add (javaKey, list = new List { conflict.GetAssemblyQualifiedName (cache) }); - list.Add (type.GetAssemblyQualifiedName (cache)); - success = false; - } - hasConflict = true; - } - if (!hasConflict) { - managed.Add (managedKey, type); - java.Add (javaKey, type); - - acw_map.Write (managedKey); - acw_map.Write (';'); - acw_map.Write (javaKey); - acw_map.WriteLine (); - - acw_map.Write (JavaNativeTypeManager.ToCompatJniName (type, cache).Replace ('/', '.')); - acw_map.Write (';'); - acw_map.Write (javaKey); - acw_map.WriteLine (); - } + Log.LogDebugMessage ($"Number of generated marshal methods: {classifier.MarshalMethods.Count}"); + + if (classifier.RejectedMethodCount > 0) { + Log.LogWarning ($"Number of methods in the project that will be registered dynamically: {classifier.RejectedMethodCount}"); } - acw_map.Flush (); - Files.CopyIfStreamChanged (acw_map.BaseStream, AcwMapFile); + if (classifier.WrappedMethodCount > 0) { + // TODO: change to LogWarning once the generator can output code which requires no non-blittable wrappers + Log.LogDebugMessage ($"Number of methods in the project that need marshal method wrappers: {classifier.WrappedMethodCount}"); + } } + } - foreach (var kvp in managedConflicts) { - Log.LogCodedWarning ("XA4214", Properties.Resources.XA4214, kvp.Key, string.Join (", ", kvp.Value)); - Log.LogCodedWarning ("XA4214", Properties.Resources.XA4214_Result, kvp.Key, kvp.Value [0]); - } + void CreateAdditionalJavaSources (ICollection javaTypes, TypeDefinitionCache cache, MarshalMethodsClassifier? classifier) + { + StringWriter regCallsWriter = new StringWriter (); + regCallsWriter.WriteLine ("\t\t// Application and Instrumentation ACWs must be registered first."); + foreach (TypeDefinition type in javaTypes) { + if (JavaNativeTypeManager.IsApplication (type, cache) || JavaNativeTypeManager.IsInstrumentation (type, cache)) { + if (classifier != null && !classifier.FoundDynamicallyRegisteredMethods (type)) { + continue; + } - foreach (var kvp in javaConflicts) { - Log.LogCodedError ("XA4215", Properties.Resources.XA4215, kvp.Key); - foreach (var typeName in kvp.Value) - Log.LogCodedError ("XA4215", Properties.Resources.XA4215_Details, kvp.Key, typeName); + string javaKey = JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.'); + regCallsWriter.WriteLine ("\t\tmono.android.Runtime.register (\"{0}\", {1}.class, {1}.__md_methods);", + type.GetAssemblyQualifiedName (cache), javaKey); + } } + regCallsWriter.Close (); + + var real_app_dir = Path.Combine (OutputDirectory, "src", "mono", "android", "app"); + string applicationTemplateFile = "ApplicationRegistration.java"; + SaveResource (applicationTemplateFile, applicationTemplateFile, real_app_dir, + template => template.Replace ("// REGISTER_APPLICATION_AND_INSTRUMENTATION_CLASSES_HERE", regCallsWriter.ToString ())); + } - // Step 3 - Merge [Activity] and friends into AndroidManifest.xml + void UpdateAndroidManifest (RunState state, TypeDefinitionCache cache, ICollection allJavaTypes) + { var manifest = new ManifestDocument (ManifestTemplate) { - PackageName = PackageName, - VersionName = VersionName, - ApplicationLabel = ApplicationLabel ?? PackageName, - Placeholders = ManifestPlaceholders, - Resolver = res, - SdkDir = AndroidSdkDir, - TargetSdkVersion = AndroidSdkPlatform, - MinSdkVersion = MonoAndroidHelper.ConvertSupportedOSPlatformVersionToApiLevel (SupportedOSPlatformVersion).ToString (), - Debug = Debug, - MultiDex = MultiDex, - NeedsInternet = NeedsInternet, + PackageName = PackageName, + VersionName = VersionName, + ApplicationLabel = ApplicationLabel ?? PackageName, + Placeholders = ManifestPlaceholders, + Resolver = state.Resolver, + SdkDir = AndroidSdkDir, + TargetSdkVersion = AndroidSdkPlatform, + MinSdkVersion = MonoAndroidHelper.ConvertSupportedOSPlatformVersionToApiLevel (SupportedOSPlatformVersion).ToString (), + Debug = Debug, + MultiDex = MultiDex, + NeedsInternet = NeedsInternet, InstantRunEnabled = InstantRunEnabled }; // Only set manifest.VersionCode if there is no existing value in AndroidManifest.xml. @@ -341,7 +482,10 @@ void Run (XAAssemblyResolver res, bool useMarshalMethods) } else if (!string.IsNullOrEmpty (VersionCode)) { manifest.VersionCode = VersionCode; } - manifest.Assemblies.AddRange (userAssemblies.Values); + + foreach (ITaskItem assembly in state.UserAssemblies) { + manifest.Assemblies.Add (Path.GetFileName (assembly.ItemSpec)); + } if (!String.IsNullOrWhiteSpace (CheckedBuild)) { // We don't validate CheckedBuild value here, this will be done in BuildApk. We just know that if it's @@ -356,70 +500,12 @@ void Run (XAAssemblyResolver res, bool useMarshalMethods) if (manifest.SaveIfChanged (Log, MergedAndroidManifestOutput)) { Log.LogDebugMessage ($"Saving: {MergedAndroidManifestOutput}"); } + } - // Create additional runtime provider java sources. - string providerTemplateFile = "MonoRuntimeProvider.Bundled.java"; - string providerTemplate = GetResource (providerTemplateFile); - - foreach (var provider in additionalProviders) { - var contents = providerTemplate.Replace ("MonoRuntimeProvider", provider); - var real_provider = Path.Combine (OutputDirectory, "src", "mono", provider + ".java"); - Files.CopyIfStringChanged (contents, real_provider); - } - - // Create additional application java sources. - StringWriter regCallsWriter = new StringWriter (); - regCallsWriter.WriteLine ("\t\t// Application and Instrumentation ACWs must be registered first."); - foreach (JavaType jt in javaTypes) { - TypeDefinition type = jt.Type; - if (JavaNativeTypeManager.IsApplication (type, cache) || JavaNativeTypeManager.IsInstrumentation (type, cache)) { - if (classifier != null && !classifier.FoundDynamicallyRegisteredMethods (type)) { - continue; - } - - string javaKey = JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.'); - regCallsWriter.WriteLine ("\t\tmono.android.Runtime.register (\"{0}\", {1}.class, {1}.__md_methods);", - type.GetAssemblyQualifiedName (cache), javaKey); - } - } - regCallsWriter.Close (); - - var real_app_dir = Path.Combine (OutputDirectory, "src", "mono", "android", "app"); - string applicationTemplateFile = "ApplicationRegistration.java"; - SaveResource (applicationTemplateFile, applicationTemplateFile, real_app_dir, - template => template.Replace ("// REGISTER_APPLICATION_AND_INSTRUMENTATION_CLASSES_HERE", regCallsWriter.ToString ())); - - if (useMarshalMethods) { - classifier.AddSpecialCaseMethods (); - - Log.LogDebugMessage ($"Number of generated marshal methods: {classifier.MarshalMethods.Count}"); - - if (classifier.RejectedMethodCount > 0) { - Log.LogWarning ($"Number of methods in the project that will be registered dynamically: {classifier.RejectedMethodCount}"); - } - - if (classifier.WrappedMethodCount > 0) { - // TODO: change to LogWarning once the generator can output code which requires no non-blittable wrappers - Log.LogDebugMessage ($"Number of methods in the project that need marshal method wrappers: {classifier.WrappedMethodCount}"); - } - } - - void MaybeAddAbiSpecifcAssembly (ITaskItem assembly, string fileName) - { - if (abiSpecificAssembliesByPath == null) { - return; - } - - string? abi = assembly.GetMetadata ("Abi"); - if (!String.IsNullOrEmpty (abi)) { - if (!abiSpecificAssembliesByPath.TryGetValue (fileName, out List? items)) { - items = new List (); - abiSpecificAssembliesByPath.Add (fileName, items); - } - - items.Add (assembly); - } - } + void WriteAcwMaps (List javaTypes, TypeDefinitionCache cache) + { + var writer = new AcwMapWriter (Log, AcwMapFile); + writer.Write (javaTypes, cache); } AssemblyDefinition LoadAssembly (string path, XAAssemblyResolver? resolver = null) @@ -452,7 +538,7 @@ AssemblyDefinition LoadAssembly (string path, XAAssemblyResolver? resolver = nul } } - bool CreateJavaSources (IEnumerable newJavaTypes, TypeDefinitionCache cache, MarshalMethodsClassifier classifier, bool useMarshalMethods) + bool CreateJavaSources (IEnumerable newJavaTypes, TypeDefinitionCache cache, MarshalMethodsClassifier classifier, bool useMarshalMethods) { if (useMarshalMethods && classifier == null) { throw new ArgumentNullException (nameof (classifier)); @@ -464,8 +550,7 @@ bool CreateJavaSources (IEnumerable newJavaTypes, TypeDefinitionCache bool generateOnCreateOverrides = int.Parse (AndroidSdkPlatform) <= 10; bool ok = true; - foreach (JavaType jt in newJavaTypes) { - TypeDefinition t = jt.Type; // JCW generator doesn't care about ABI-specific types or token ids + foreach (TypeDefinition t in newJavaTypes) { if (t.IsInterface) { // Interfaces are in typemap but they shouldn't have JCW generated for them continue; @@ -568,14 +653,14 @@ void SaveResource (string resource, string filename, string destDir, Func types, TypeDefinitionCache cache) + void WriteTypeMappings (AndroidTargetArch targetArch, ICollection types, TypeDefinitionCache cache, out ApplicationConfigTaskState appConfState) { - var tmg = new TypeMapGenerator ((string message) => Log.LogDebugMessage (message), SupportedAbis); - if (!tmg.Generate (Debug, SkipJniAddNativeMethodRegistrationAttributeScan, types, cache, TypemapOutputDirectory, GenerateNativeAssembly, out ApplicationConfigTaskState appConfState)) { + Log.LogDebugMessage ($"Generating typemaps for arch {targetArch}, {types.Count} types"); + var tmg = new TypeMapGenerator (targetArch, Log, SupportedAbis); + if (!tmg.Generate (Debug, SkipJniAddNativeMethodRegistrationAttributeScan, types, cache, TypemapOutputDirectory, GenerateNativeAssembly, out appConfState)) { throw new XamarinAndroidException (4308, Properties.Resources.XA4308); } GeneratedBinaryTypeMaps = tmg.GeneratedBinaryTypeMaps.ToArray (); - BuildEngine4.RegisterTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (ApplicationConfigTaskState.RegisterTaskObjectKey), appConfState, RegisteredTaskObjectLifetime.Build); } /// diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJniRemappingNativeCode.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJniRemappingNativeCode.cs index b89ce87d26d..b8eec268ddb 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJniRemappingNativeCode.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJniRemappingNativeCode.cs @@ -86,7 +86,7 @@ void Generate (JniRemappingAssemblyGenerator jniRemappingComposer, int typeRepla string llFilePath = $"{baseAsmFilePath}.ll"; using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) { - jniRemappingComposer.Generate (module, GeneratePackageManagerJava.GetAndroidTargetArchForAbi (abi), sw, llFilePath); + jniRemappingComposer.Generate (module, MonoAndroidHelper.AbiToTargetArch (abi), sw, llFilePath); sw.Flush (); Files.CopyIfStreamChanged (sw.BaseStream, llFilePath); } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index 670fcfe566e..87b22d35d81 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -38,6 +38,9 @@ public class GeneratePackageManagerJava : AndroidTask public bool UseAssemblyStore { get; set; } + [Required] + public bool UseAssemblySharedLibraries { get; set; } + [Required] public string OutputDirectory { get; set; } @@ -145,26 +148,6 @@ public override bool RunTask () return !Log.HasLoggedErrors; } - static internal AndroidTargetArch GetAndroidTargetArchForAbi (string abi) - { - switch (abi.Trim ()) { - case "armeabi-v7a": - return AndroidTargetArch.Arm; - - case "arm64-v8a": - return AndroidTargetArch.Arm64; - - case "x86": - return AndroidTargetArch.X86; - - case "x86_64": - return AndroidTargetArch.X86_64; - - default: - throw new InvalidOperationException ($"Unknown ABI {abi}"); - } - } - static readonly string[] defaultLogLevel = {"MONO_LOG_LEVEL", "info"}; static readonly string[] defaultMonoDebug = {"MONO_DEBUG", "gen-compact-seq-points"}; static readonly string[] defaultHttpMessageHandler = {"XA_HTTP_CLIENT_HANDLER_TYPE", "System.Net.Http.HttpClientHandler, System.Net.Http"}; @@ -246,7 +229,7 @@ void AddEnvironment () Encoding assemblyNameEncoding = Encoding.UTF8; Action updateNameWidth = (ITaskItem assembly) => { - if (UseAssemblyStore) { + if (UseAssemblyStore) { // TODO: modify for assemblies embedded in DSOs return; } @@ -269,7 +252,7 @@ void AddEnvironment () uniqueAssemblyNames.Add (assemblyName); } - if (!UseAssemblyStore) { + if (!UseAssemblyStore) { // TODO: modify for assemblies embedded in DSOs assemblyCount++; return; } @@ -316,7 +299,7 @@ void AddEnvironment () GetRequiredTokens (assembly.ItemSpec, out android_runtime_jnienv_class_token, out jnienv_initialize_method_token, out jnienv_registerjninatives_method_token); } - if (!UseAssemblyStore) { + if (!UseAssemblyStore) { // TODO: modify for assemblies embedded in DSOs int abiNameLength = 0; foreach (string abi in SupportedAbis) { if (abi.Length <= abiNameLength) { @@ -386,13 +369,8 @@ void AddEnvironment () HaveRuntimeConfigBlob = haveRuntimeConfigBlob, NumberOfAssembliesInApk = assemblyCount, BundledAssemblyNameWidth = assemblyNameWidth, - NumberOfAssemblyStoresInApks = 2, // Until feature APKs are a thing, we're going to have just two stores in each app - one for arch-agnostic - // and up to 4 other for arch-specific assemblies. Only **one** arch-specific store is ever loaded on the app - // runtime, thus the number 2 here. All architecture specific stores contain assemblies with the same names - // and in the same order. MonoComponents = (MonoComponent)monoComponents, NativeLibraries = uniqueNativeLibraries, - HaveAssemblyStore = UseAssemblyStore, AndroidRuntimeJNIEnvToken = android_runtime_jnienv_class_token, JNIEnvInitializeToken = jnienv_initialize_method_token, JNIEnvRegisterJniNativesToken = jnienv_registerjninatives_method_token, @@ -423,7 +401,7 @@ void AddEnvironment () string marshalMethodsBaseAsmFilePath = Path.Combine (EnvironmentOutputDirectory, $"marshal_methods.{targetAbi}"); string environmentLlFilePath = $"{environmentBaseAsmFilePath}.ll"; string marshalMethodsLlFilePath = $"{marshalMethodsBaseAsmFilePath}.ll"; - AndroidTargetArch targetArch = GetAndroidTargetArchForAbi (abi); + AndroidTargetArch targetArch = MonoAndroidHelper.AbiToTargetArch (abi); using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) { try { diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs b/src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs index 5f4b09ececa..4fc366e6325 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs @@ -136,7 +136,7 @@ IEnumerable GetLinkerConfigs () string stripSymbolsArg = DebugBuild ? String.Empty : " -s"; - string ld = Path.Combine (AndroidBinUtilsDirectory, MonoAndroidHelper.GetExecutablePath (AndroidBinUtilsDirectory, "ld")); + string ld = NativeCompilationHelper.GetLinkerPath (AndroidBinUtilsDirectory); var targetLinkerArgs = new List (); foreach (var kvp in abis) { string abi = kvp.Key; diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAbiItems.cs b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAbiItems.cs index d826e94110c..31b10540bc4 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAbiItems.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAbiItems.cs @@ -13,9 +13,9 @@ public class PrepareAbiItems : AndroidTask const string ArmV7a = "armeabi-v7a"; const string TypeMapBase = "typemaps"; const string EnvBase = "environment"; - const string CompressedAssembliesBase = "compressed_assemblies"; const string JniRemappingBase = "jni_remap"; const string MarshalMethodsBase = "marshal_methods"; + public const string AssemblyDSOBase = "assembly_dso"; public override string TaskPrefix => "PAI"; @@ -50,12 +50,12 @@ public override bool RunTask () baseName = TypeMapBase; } else if (String.Compare ("environment", Mode, StringComparison.OrdinalIgnoreCase) == 0) { baseName = EnvBase; - } else if (String.Compare ("compressed", Mode, StringComparison.OrdinalIgnoreCase) == 0) { - baseName = CompressedAssembliesBase; } else if (String.Compare ("jniremap", Mode, StringComparison.OrdinalIgnoreCase) == 0) { baseName = JniRemappingBase; } else if (String.Compare ("marshal_methods", Mode, StringComparison.OrdinalIgnoreCase) == 0) { baseName = MarshalMethodsBase; + } else if (String.Compare ("assembly_dsos", Mode, StringComparison.OrdinalIgnoreCase) == 0) { + baseName = AssemblyDSOBase; } else { Log.LogError ($"Unknown mode: {Mode}"); return false; @@ -63,7 +63,7 @@ public override bool RunTask () TaskItem item; foreach (string abi in BuildTargetAbis) { - item = new TaskItem (Path.Combine (NativeSourcesDir, $"{baseName}.{abi}.ll")); + item = new TaskItem (Path.Combine (NativeSourcesDir, MonoAndroidHelper.MakeNativeAssemblyFileName (baseName, abi))); item.SetMetadata ("abi", abi); sources.Add (item); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AcwMapWriter.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AcwMapWriter.cs new file mode 100644 index 00000000000..315387ecbbf --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AcwMapWriter.cs @@ -0,0 +1,92 @@ +using System; +using System.Collections.Generic; + +using Java.Interop.Tools.Cecil; +using Java.Interop.Tools.TypeNameMappings; +using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Mono.Cecil; + +namespace Xamarin.Android.Tasks; + +class AcwMapWriter +{ + readonly TaskLoggingHelper Log; + readonly string AcwMapFile; + + public AcwMapWriter (TaskLoggingHelper log, string acwMapFile) + { + Log = log; + AcwMapFile = acwMapFile; + } + + public void Write (ICollection javaTypes, TypeDefinitionCache cache) + { + // We need to save a map of .NET type -> ACW type for resource file fixups + var managed = new Dictionary (javaTypes.Count, StringComparer.Ordinal); + var java = new Dictionary (javaTypes.Count, StringComparer.Ordinal); + + var managedConflicts = new Dictionary> (0, StringComparer.Ordinal); + var javaConflicts = new Dictionary> (0, StringComparer.Ordinal); + + using (var acw_map = MemoryStreamPool.Shared.CreateStreamWriter ()) { + foreach (TypeDefinition type in javaTypes) { + string managedKey = type.FullName.Replace ('/', '.'); + string javaKey = JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.'); + + acw_map.Write (type.GetPartialAssemblyQualifiedName (cache)); + acw_map.Write (';'); + acw_map.Write (javaKey); + acw_map.WriteLine (); + + TypeDefinition conflict; + bool hasConflict = false; + if (managed.TryGetValue (managedKey, out conflict)) { + if (!conflict.Module.Name.Equals (type.Module.Name)) { + if (!managedConflicts.TryGetValue (managedKey, out var list)) + managedConflicts.Add (managedKey, list = new List { conflict.GetPartialAssemblyName (cache) }); + list.Add (type.GetPartialAssemblyName (cache)); + } + hasConflict = true; + } + if (java.TryGetValue (javaKey, out conflict)) { + if (!conflict.Module.Name.Equals (type.Module.Name)) { + if (!javaConflicts.TryGetValue (javaKey, out var list)) + javaConflicts.Add (javaKey, list = new List { conflict.GetAssemblyQualifiedName (cache) }); + list.Add (type.GetAssemblyQualifiedName (cache)); + } + hasConflict = true; + } + if (!hasConflict) { + managed.Add (managedKey, type); + java.Add (javaKey, type); + + acw_map.Write (managedKey); + acw_map.Write (';'); + acw_map.Write (javaKey); + acw_map.WriteLine (); + + acw_map.Write (JavaNativeTypeManager.ToCompatJniName (type, cache).Replace ('/', '.')); + acw_map.Write (';'); + acw_map.Write (javaKey); + acw_map.WriteLine (); + } + } + + acw_map.Flush (); + Files.CopyIfStreamChanged (acw_map.BaseStream, AcwMapFile); + } + + foreach (var kvp in managedConflicts) { + Log.LogCodedWarning ("XA4214", Properties.Resources.XA4214, kvp.Key, string.Join (", ", kvp.Value)); + Log.LogCodedWarning ("XA4214", Properties.Resources.XA4214_Result, kvp.Key, kvp.Value [0]); + } + + foreach (var kvp in javaConflicts) { + Log.LogCodedError ("XA4215", Properties.Resources.XA4215, kvp.Key); + foreach (var typeName in kvp.Value) + Log.LogCodedError ("XA4215", Properties.Resources.XA4215_Details, kvp.Key, typeName); + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfig.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfig.cs index 25550eb3473..a38d7e774b8 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfig.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfig.cs @@ -32,7 +32,6 @@ sealed class ApplicationConfig public bool instant_run_enabled ; public bool jni_add_native_method_registration_attribute_present; public bool have_runtime_config_blob; - public bool have_assemblies_blob; public bool marshal_methods_enabled; public byte bound_stream_io_exception_type; public uint package_naming_policy; @@ -40,13 +39,21 @@ sealed class ApplicationConfig public uint system_property_count; public uint number_of_assemblies_in_apk; public uint bundled_assembly_name_width; - public uint number_of_assembly_store_files; public uint number_of_dso_cache_entries; + + [NativeAssembler (NumberFormat = LLVMIR.LlvmIrVariableNumberFormat.Hexadecimal)] public uint android_runtime_jnienv_class_token; + + [NativeAssembler (NumberFormat = LLVMIR.LlvmIrVariableNumberFormat.Hexadecimal)] public uint jnienv_initialize_method_token; + + [NativeAssembler (NumberFormat = LLVMIR.LlvmIrVariableNumberFormat.Hexadecimal)] public uint jnienv_registerjninatives_method_token; + public uint jni_remapping_replacement_type_count; public uint jni_remapping_replacement_method_index_entry_count; + + [NativeAssembler (NumberFormat = LLVMIR.LlvmIrVariableNumberFormat.Hexadecimal)] public uint mono_components_mask; public string android_package_name = String.Empty; } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs index 2dcbac7b0e4..8be51dae250 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs @@ -28,7 +28,7 @@ public override string GetComment (object data, string fieldName) { var dso_entry = EnsureType (data); if (String.Compare ("hash", fieldName, StringComparison.Ordinal) == 0) { - return $" hash 0x{dso_entry.hash:x}, from name: {dso_entry.HashedName}"; + return $" hash, from name: {dso_entry.HashedName}"; } if (String.Compare ("name", fieldName, StringComparison.Ordinal) == 0) { @@ -57,49 +57,6 @@ sealed class DSOCacheEntry public IntPtr handle = IntPtr.Zero; } - // Order of fields and their type must correspond *exactly* to that in - // src/monodroid/jni/xamarin-app.hh AssemblyStoreAssemblyDescriptor structure - sealed class AssemblyStoreAssemblyDescriptor - { - public uint data_offset; - public uint data_size; - - public uint debug_data_offset; - public uint debug_data_size; - - public uint config_data_offset; - public uint config_data_size; - } - - // Order of fields and their type must correspond *exactly* to that in - // src/monodroid/jni/xamarin-app.hh AssemblyStoreSingleAssemblyRuntimeData structure - sealed class AssemblyStoreSingleAssemblyRuntimeData - { - [NativePointer] - public byte image_data; - - [NativePointer] - public byte debug_info_data; - - [NativePointer] - public byte config_data; - - [NativePointer] - public AssemblyStoreAssemblyDescriptor descriptor; - } - - // Order of fields and their type must correspond *exactly* to that in - // src/monodroid/jni/xamarin-app.hh AssemblyStoreRuntimeData structure - sealed class AssemblyStoreRuntimeData - { - [NativePointer] - public byte data_start; - public uint assembly_count; - - [NativePointer] - public AssemblyStoreAssemblyDescriptor assemblies; - } - sealed class XamarinAndroidBundledAssemblyContextDataProvider : NativeAssemblerStructContextDataProvider { public override ulong GetBufferSize (object data, string fieldName) @@ -157,9 +114,7 @@ sealed class XamarinAndroidBundledAssembly public bool InstantRunEnabled { get; set; } public bool JniAddNativeMethodRegistrationAttributePresent { get; set; } public bool HaveRuntimeConfigBlob { get; set; } - public bool HaveAssemblyStore { get; set; } public int NumberOfAssembliesInApk { get; set; } - public int NumberOfAssemblyStoresInApks { get; set; } public int BundledAssemblyNameWidth { get; set; } // including the trailing NUL public int AndroidRuntimeJNIEnvToken { get; set; } public int JNIEnvInitializeToken { get; set; } @@ -211,7 +166,6 @@ protected override void Construct (LlvmIrModule module) instant_run_enabled = InstantRunEnabled, jni_add_native_method_registration_attribute_present = JniAddNativeMethodRegistrationAttributePresent, have_runtime_config_blob = HaveRuntimeConfigBlob, - have_assemblies_blob = HaveAssemblyStore, marshal_methods_enabled = MarshalMethodsEnabled, bound_stream_io_exception_type = (byte)BoundExceptionType, package_naming_policy = (uint)PackageNamingPolicy, @@ -219,7 +173,6 @@ protected override void Construct (LlvmIrModule module) system_property_count = (uint)(systemProperties == null ? 0 : systemProperties.Count * 2), number_of_assemblies_in_apk = (uint)NumberOfAssembliesInApk, bundled_assembly_name_width = (uint)BundledAssemblyNameWidth, - number_of_assembly_store_files = (uint)NumberOfAssemblyStoresInApks, number_of_dso_cache_entries = (uint)dsoCache.Count, android_runtime_jnienv_class_token = (uint)AndroidRuntimeJNIEnvToken, jnienv_initialize_method_token = (uint)JNIEnvInitializeToken, @@ -235,24 +188,23 @@ protected override void Construct (LlvmIrModule module) var dso_cache = new LlvmIrGlobalVariable (dsoCache, "dso_cache", LlvmIrVariableOptions.GlobalWritable) { Comment = " DSO cache entries", BeforeWriteCallback = HashAndSortDSOCache, + NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal, }; module.Add (dso_cache); - if (!HaveAssemblyStore) { - xamarinAndroidBundledAssemblies = new List> (NumberOfAssembliesInApk); - - var emptyBundledAssemblyData = new XamarinAndroidBundledAssembly { - apk_fd = -1, - data_offset = 0, - data_size = 0, - data = 0, - name_length = (uint)BundledAssemblyNameWidth, - name = null, - }; + // TODO: don't generate if we're embedding assemblies into DSOs + xamarinAndroidBundledAssemblies = new List> (NumberOfAssembliesInApk); + var emptyBundledAssemblyData = new XamarinAndroidBundledAssembly { + apk_fd = -1, + data_offset = 0, + data_size = 0, + data = 0, + name_length = (uint)BundledAssemblyNameWidth, + name = null, + }; - for (int i = 0; i < NumberOfAssembliesInApk; i++) { - xamarinAndroidBundledAssemblies.Add (new StructureInstance (xamarinAndroidBundledAssemblyStructureInfo, emptyBundledAssemblyData)); - } + for (int i = 0; i < NumberOfAssembliesInApk; i++) { + xamarinAndroidBundledAssemblies.Add (new StructureInstance (xamarinAndroidBundledAssemblyStructureInfo, emptyBundledAssemblyData)); } string bundledBuffersSize = xamarinAndroidBundledAssemblies == null ? "empty (unused when assembly stores are enabled)" : $"{BundledAssemblyNameWidth} bytes long"; @@ -261,25 +213,6 @@ protected override void Construct (LlvmIrModule module) Comment = $" Bundled assembly name buffers, all {bundledBuffersSize}", }; module.Add (bundled_assemblies); - - AddAssemblyStores (module); - } - - void AddAssemblyStores (LlvmIrModule module) - { - ulong itemCount = (ulong)(HaveAssemblyStore ? NumberOfAssembliesInApk : 0); - var assembly_store_bundled_assemblies = new LlvmIrGlobalVariable (typeof(List>), "assembly_store_bundled_assemblies", LlvmIrVariableOptions.GlobalWritable) { - ZeroInitializeArray = true, - ArrayItemCount = itemCount, - }; - module.Add (assembly_store_bundled_assemblies); - - itemCount = (ulong)(HaveAssemblyStore ? NumberOfAssemblyStoresInApks : 0); - var assembly_stores = new LlvmIrGlobalVariable (typeof(List>), "assembly_stores", LlvmIrVariableOptions.GlobalWritable) { - ZeroInitializeArray = true, - ArrayItemCount = itemCount, - }; - module.Add (assembly_stores); } void HashAndSortDSOCache (LlvmIrVariable variable, LlvmIrModuleTarget target, object? state) @@ -371,9 +304,6 @@ void AddNameMutations (string name) void MapStructures (LlvmIrModule module) { applicationConfigStructureInfo = module.MapStructure (); - module.MapStructure (); - assemblyStoreSingleAssemblyRuntimeDataStructureinfo = module.MapStructure (); - assemblyStoreRuntimeDataStructureInfo = module.MapStructure (); xamarinAndroidBundledAssemblyStructureInfo = module.MapStructure (); dsoCacheEntryStructureInfo = module.MapStructure (); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyStore.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyStore.cs deleted file mode 100644 index a5b5811b7de..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyStore.cs +++ /dev/null @@ -1,112 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; - -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; - -namespace Xamarin.Android.Tasks -{ - class ArchAssemblyStore : AssemblyStore - { - readonly Dictionary> assemblies; - HashSet seenArchAssemblyNames; - - public ArchAssemblyStore (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log, uint id, AssemblyStoreGlobalIndex globalIndexCounter) - : base (apkName, archiveAssembliesPrefix, log, id, globalIndexCounter) - { - assemblies = new Dictionary> (StringComparer.OrdinalIgnoreCase); - } - - public override string WriteIndex (List globalIndex) - { - throw new InvalidOperationException ("Architecture-specific assembly blob cannot contain global assembly index"); - } - - public override void Add (AssemblyStoreAssemblyInfo blobAssembly) - { - if (String.IsNullOrEmpty (blobAssembly.Abi)) { - throw new InvalidOperationException ($"Architecture-agnostic assembly cannot be added to an architecture-specific blob ({blobAssembly.FilesystemAssemblyPath})"); - } - - if (!assemblies.ContainsKey (blobAssembly.Abi)) { - assemblies.Add (blobAssembly.Abi, new List ()); - } - - List blobAssemblies = assemblies[blobAssembly.Abi]; - blobAssemblies.Add (blobAssembly); - - if (seenArchAssemblyNames == null) { - seenArchAssemblyNames = new HashSet (StringComparer.Ordinal); - } - - string assemblyName = GetAssemblyName (blobAssembly); - if (seenArchAssemblyNames.Contains (assemblyName)) { - return; - } - - seenArchAssemblyNames.Add (assemblyName); - } - - public override void Generate (string outputDirectory, List globalIndex, List blobPaths) - { - if (assemblies.Count == 0) { - return; - } - - var assemblyNames = new Dictionary (); - foreach (var kvp in assemblies) { - string abi = kvp.Key; - List archAssemblies = kvp.Value; - - // All the architecture blobs must have assemblies in exactly the same order - archAssemblies.Sort ((AssemblyStoreAssemblyInfo a, AssemblyStoreAssemblyInfo b) => Path.GetFileName (a.FilesystemAssemblyPath).CompareTo (Path.GetFileName (b.FilesystemAssemblyPath))); - if (assemblyNames.Count == 0) { - for (int i = 0; i < archAssemblies.Count; i++) { - AssemblyStoreAssemblyInfo info = archAssemblies[i]; - assemblyNames.Add (i, Path.GetFileName (info.FilesystemAssemblyPath)); - } - continue; - } - - if (archAssemblies.Count != assemblyNames.Count) { - throw new InvalidOperationException ($"Assembly list for ABI '{abi}' has a different number of assemblies than other ABI lists (expected {assemblyNames.Count}, found {archAssemblies.Count}"); - } - - for (int i = 0; i < archAssemblies.Count; i++) { - AssemblyStoreAssemblyInfo info = archAssemblies[i]; - string fileName = Path.GetFileName (info.FilesystemAssemblyPath); - - if (assemblyNames[i] != fileName) { - throw new InvalidOperationException ($"Assembly list for ABI '{abi}' differs from other lists at index {i}. Expected '{assemblyNames[i]}', found '{fileName}'"); - } - } - } - - bool addToGlobalIndex = true; - foreach (var kvp in assemblies) { - string abi = kvp.Key; - List archAssemblies = kvp.Value; - - if (archAssemblies.Count == 0) { - continue; - } - - // Android uses underscores in place of dashes in ABI names, let's follow the convention - string androidAbi = abi.Replace ('-', '_'); - Generate (Path.Combine (outputDirectory, $"{ApkName}_{BlobPrefix}.{androidAbi}{BlobExtension}"), archAssemblies, globalIndex, blobPaths, addToGlobalIndex); - - // NOTE: not thread safe! The counter must grow monotonically but we also don't want to use different index values for the architecture-specific - // assemblies with the same names, that would only waste space in the generated `libxamarin-app.so`. To use the same index values for the same - // assemblies in different architectures we need to move the counter back here. - GlobalIndexCounter.Subtract ((uint)archAssemblies.Count); - - if (addToGlobalIndex) { - // We want the architecture-specific assemblies to be added to the global index only once - addToGlobalIndex = false; - } - } - - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs index 1972dba0d85..feea6f7ccb3 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs @@ -1,21 +1,25 @@ using System; using System.Buffers; +using System.Collections.Generic; using System.IO; using K4os.Compression.LZ4; +using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; namespace Xamarin.Android.Tasks { class AssemblyCompression { - public enum CompressionResult + enum CompressionResult { Success, InputTooBig, EncodingFailed, } - public sealed class AssemblyData + sealed class AssemblyData { public string SourcePath { get; internal set; } public uint DescriptorIndex { get; internal set; } @@ -47,15 +51,26 @@ public void SetData (string sourcePath, uint descriptorIndex) // two - it should be more than enough for most needs. //public const ulong InputAssemblySizeLimit = 60 * 1024 * 1024; - static readonly ArrayPool bytePool = ArrayPool.Shared; + readonly ArrayPool bytePool = ArrayPool.Shared; + readonly TaskLoggingHelper log; + readonly string compressedOutputDir; - public static CompressionResult Compress (AssemblyData data, string outputDirectory) + public AssemblyCompression (TaskLoggingHelper log, string compressedOutputDir) { - if (data == null) + this.log = log; + this.compressedOutputDir = compressedOutputDir; + } + + // TODO: consider using https://github.com/emmanuel-marty/lz4ultra + CompressionResult Compress (AssemblyData data, string outputDirectory) + { + if (data == null) { throw new ArgumentNullException (nameof (data)); + } - if (String.IsNullOrEmpty (outputDirectory)) + if (String.IsNullOrEmpty (outputDirectory)) { throw new ArgumentException ("must not be null or empty", nameof (outputDirectory)); + } Directory.CreateDirectory (outputDirectory); @@ -79,8 +94,9 @@ public static CompressionResult Compress (AssemblyData data, string outputDirect destBytes = bytePool.Rent (LZ4Codec.MaximumOutputSize (sourceBytes.Length)); int encodedLength = LZ4Codec.Encode (sourceBytes, 0, checked((int)fi.Length), destBytes, 0, destBytes.Length, LZ4Level.L09_HC); - if (encodedLength < 0) + if (encodedLength < 0) { return CompressionResult.EncodingFailed; + } data.DestinationSize = (uint)encodedLength; using (var fs = File.Open (data.DestinationPath, FileMode.Create, FileAccess.Write, FileShare.Read)) { @@ -94,13 +110,74 @@ public static CompressionResult Compress (AssemblyData data, string outputDirect } } } finally { - if (sourceBytes != null) + if (sourceBytes != null) { bytePool.Return (sourceBytes); - if (destBytes != null) + } + if (destBytes != null) { bytePool.Return (destBytes); + } } return CompressionResult.Success; } + + public (string outputPath, bool compressed) CompressAssembly (ITaskItem assembly, FileInfo inputInfo) + { + if (Boolean.TryParse (assembly.GetMetadata ("AndroidSkipCompression"), out bool value) && value) { + log.LogDebugMessage ($"Skipping compression of {assembly.ItemSpec} due to 'AndroidSkipCompression' == 'true' "); + return (assembly.ItemSpec, false); + } + + return CompressAssembly (assembly.ItemSpec, inputInfo, assembly.GetMetadata ("DestinationSubDirectory")); + } + + public (string outputPath, bool compressed) CompressAssembly (string assemblyPath, FileInfo inputInfo, string? subDirectory) + { + if (!inputInfo.Exists) { + throw new InvalidOperationException ($"File '{assemblyPath}' does not exist"); + } + + string assemblyOutputDir; + if (!String.IsNullOrEmpty (subDirectory)) { + assemblyOutputDir = Path.Combine (compressedOutputDir, subDirectory); + } else { + assemblyOutputDir = compressedOutputDir; + } + string outputPath = Path.Combine (assemblyOutputDir, $"{Path.GetFileName (assemblyPath)}.lz4"); + Directory.CreateDirectory (assemblyOutputDir); + + byte[]? sourceBytes = null; + byte[]? destBytes = null; + try { + int inputLength = checked((int)inputInfo.Length); + sourceBytes = bytePool.Rent (inputLength); + using (var fs = File.Open (assemblyPath, FileMode.Open, FileAccess.Read, FileShare.Read)) { + fs.Read (sourceBytes, 0, inputLength); + } + + destBytes = bytePool.Rent (LZ4Codec.MaximumOutputSize (sourceBytes.Length)); + int encodedLength = LZ4Codec.Encode (sourceBytes, 0, inputLength, destBytes, 0, destBytes.Length, LZ4Level.L09_HC); + if (encodedLength < 0) { + log.LogMessage ($"Failed to compress {assemblyPath}"); + return (assemblyPath, false); + } + + using (var fs = File.Open (outputPath, FileMode.Create, FileAccess.Write, FileShare.Read)) { + using (var bw = new BinaryWriter (fs)) { + bw.Write (destBytes, 0, encodedLength); + bw.Flush (); + } + } + } finally { + if (sourceBytes != null) { + bytePool.Return (sourceBytes); + } + if (destBytes != null) { + bytePool.Return (destBytes); + } + } + + return (outputPath, true); + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStore.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStore.cs deleted file mode 100644 index 378a9ee8c46..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStore.cs +++ /dev/null @@ -1,377 +0,0 @@ -using System; -using System.Buffers; -using System.Collections.Generic; -using System.IO; -using System.Text; - -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; - -namespace Xamarin.Android.Tasks -{ - abstract class AssemblyStore - { - // The two constants below must match their counterparts in src/monodroid/jni/xamarin-app.hh - const uint BlobMagic = 0x41424158; // 'XABA', little-endian, must match the BUNDLED_ASSEMBLIES_BLOB_MAGIC native constant - const uint BlobVersion = 1; // Must match the BUNDLED_ASSEMBLIES_BLOB_VERSION native constant - - // MUST be equal to the size of the BlobBundledAssembly struct in src/monodroid/jni/xamarin-app.hh - const uint BlobBundledAssemblyNativeStructSize = 6 * sizeof (uint); - - // MUST be equal to the size of the BlobHashEntry struct in src/monodroid/jni/xamarin-app.hh - const uint BlobHashEntryNativeStructSize = sizeof (ulong) + (3 * sizeof (uint)); - - // MUST be equal to the size of the BundledAssemblyBlobHeader struct in src/monodroid/jni/xamarin-app.hh - const uint BlobHeaderNativeStructSize = sizeof (uint) * 5; - - protected const string BlobPrefix = "assemblies"; - protected const string BlobExtension = ".blob"; - - static readonly ArrayPool bytePool = ArrayPool.Shared; - - string archiveAssembliesPrefix; - string indexBlobPath; - - protected string ApkName { get; } - protected TaskLoggingHelper Log { get; } - protected AssemblyStoreGlobalIndex GlobalIndexCounter { get; } - - public uint ID { get; } - public bool IsIndexStore => ID == 0; - - protected AssemblyStore (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log, uint id, AssemblyStoreGlobalIndex globalIndexCounter) - { - if (String.IsNullOrEmpty (archiveAssembliesPrefix)) { - throw new ArgumentException ("must not be null or empty", nameof (archiveAssembliesPrefix)); - } - - if (String.IsNullOrEmpty (apkName)) { - throw new ArgumentException ("must not be null or empty", nameof (apkName)); - } - - GlobalIndexCounter = globalIndexCounter ?? throw new ArgumentNullException (nameof (globalIndexCounter)); - ID = id; - - this.archiveAssembliesPrefix = archiveAssembliesPrefix; - ApkName = apkName; - Log = log; - } - - public abstract void Add (AssemblyStoreAssemblyInfo blobAssembly); - public abstract void Generate (string outputDirectory, List globalIndex, List blobPaths); - - public virtual string WriteIndex (List globalIndex) - { - if (!IsIndexStore) { - throw new InvalidOperationException ("Assembly index may be written only to blob with index 0"); - } - - if (String.IsNullOrEmpty (indexBlobPath)) { - throw new InvalidOperationException ("Index blob path not set, was Generate called properly?"); - } - - if (globalIndex == null) { - throw new ArgumentNullException (nameof (globalIndex)); - } - - string indexBlobHeaderPath = $"{indexBlobPath}.hdr"; - string indexBlobManifestPath = Path.ChangeExtension (indexBlobPath, "manifest"); - - using (var hfs = File.Open (indexBlobHeaderPath, FileMode.Create, FileAccess.Write, FileShare.None)) { - using (var writer = new BinaryWriter (hfs, Encoding.UTF8, leaveOpen: true)) { - WriteIndex (writer, indexBlobManifestPath, globalIndex); - writer.Flush (); - } - - using (var ifs = File.Open (indexBlobPath, FileMode.Open, FileAccess.Read, FileShare.Read)) { - ifs.CopyTo (hfs); - hfs.Flush (); - } - } - - File.Delete (indexBlobPath); - File.Move (indexBlobHeaderPath, indexBlobPath); - - return indexBlobManifestPath; - } - - void WriteIndex (BinaryWriter blobWriter, string manifestPath, List globalIndex) - { - using (var manifest = File.Open (manifestPath, FileMode.Create, FileAccess.Write)) { - using (var manifestWriter = new StreamWriter (manifest, new UTF8Encoding (false))) { - WriteIndex (blobWriter, manifestWriter, globalIndex); - manifestWriter.Flush (); - } - } - } - - void WriteIndex (BinaryWriter blobWriter, StreamWriter manifestWriter, List globalIndex) - { - uint localEntryCount = 0; - var localAssemblies = new List (); - - manifestWriter.WriteLine ("Hash 32 Hash 64 Blob ID Blob idx Name"); - - var seenHashes32 = new HashSet (); - var seenHashes64 = new HashSet (); - bool haveDuplicates = false; - foreach (AssemblyStoreIndexEntry assembly in globalIndex) { - if (assembly.StoreID == ID) { - localEntryCount++; - localAssemblies.Add (assembly); - } - - if (WarnAboutDuplicateHash ("32", assembly.Name, assembly.NameHash32, seenHashes32) || - WarnAboutDuplicateHash ("64", assembly.Name, assembly.NameHash64, seenHashes64)) { - haveDuplicates = true; - } - - manifestWriter.WriteLine ($"0x{assembly.NameHash32:x08} 0x{assembly.NameHash64:x016} {assembly.StoreID:d03} {assembly.LocalBlobIndex:d04} {assembly.Name}"); - } - - if (haveDuplicates) { - throw new InvalidOperationException ("Duplicate assemblies encountered"); - } - - uint globalAssemblyCount = (uint)globalIndex.Count; - - blobWriter.Seek (0, SeekOrigin.Begin); - WriteBlobHeader (blobWriter, localEntryCount, globalAssemblyCount); - - // Header and two tables of the same size, each for 32 and 64-bit hashes - uint offsetFixup = BlobHeaderNativeStructSize + (BlobHashEntryNativeStructSize * globalAssemblyCount * 2); - - WriteAssemblyDescriptors (blobWriter, localAssemblies, CalculateOffsetFixup ((uint)localAssemblies.Count, offsetFixup)); - - var sortedIndex = new List (globalIndex); - sortedIndex.Sort ((AssemblyStoreIndexEntry a, AssemblyStoreIndexEntry b) => a.NameHash32.CompareTo (b.NameHash32)); - foreach (AssemblyStoreIndexEntry entry in sortedIndex) { - WriteHash (entry, entry.NameHash32); - } - - sortedIndex.Sort ((AssemblyStoreIndexEntry a, AssemblyStoreIndexEntry b) => a.NameHash64.CompareTo (b.NameHash64)); - foreach (AssemblyStoreIndexEntry entry in sortedIndex) { - WriteHash (entry, entry.NameHash64); - } - - void WriteHash (AssemblyStoreIndexEntry entry, ulong hash) - { - blobWriter.Write (hash); - blobWriter.Write (entry.MappingIndex); - blobWriter.Write (entry.LocalBlobIndex); - blobWriter.Write (entry.StoreID); - } - - bool WarnAboutDuplicateHash (string bitness, string assemblyName, ulong hash, HashSet seenHashes) - { - if (seenHashes.Contains (hash)) { - Log.LogMessage (MessageImportance.High, $"Duplicate {bitness}-bit hash 0x{hash} encountered for assembly {assemblyName}"); - return true; - } - - seenHashes.Add (hash); - return false; - } - } - - protected string GetAssemblyName (AssemblyStoreAssemblyInfo assembly) - { - string assemblyName = Path.GetFileNameWithoutExtension (assembly.FilesystemAssemblyPath); - if (assemblyName.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) { - assemblyName = Path.GetFileNameWithoutExtension (assemblyName); - } - - return assemblyName; - } - - protected void Generate (string outputFilePath, List assemblies, List globalIndex, List blobPaths, bool addToGlobalIndex = true) - { - if (globalIndex == null) { - throw new ArgumentNullException (nameof (globalIndex)); - } - - if (blobPaths == null) { - throw new ArgumentNullException (nameof (blobPaths)); - } - - if (IsIndexStore) { - indexBlobPath = outputFilePath; - } - - blobPaths.Add (outputFilePath); - Log.LogMessage (MessageImportance.Low, $"AssemblyBlobGenerator: generating blob: {outputFilePath}"); - - using (var fs = File.Open (outputFilePath, FileMode.Create, FileAccess.Write, FileShare.Read)) { - using (var writer = new BinaryWriter (fs, Encoding.UTF8)) { - Generate (writer, assemblies, globalIndex, addToGlobalIndex); - writer.Flush (); - } - } - } - - void Generate (BinaryWriter writer, List assemblies, List globalIndex, bool addToGlobalIndex) - { - var localAssemblies = new List (); - - if (!IsIndexStore) { - // Index blob's header and data before the assemblies is handled in WriteIndex in a slightly different - // way. - uint nbytes = BlobHeaderNativeStructSize + (BlobBundledAssemblyNativeStructSize * (uint)assemblies.Count); - var zeros = bytePool.Rent ((int)nbytes); - writer.Write (zeros, 0, (int)nbytes); - bytePool.Return (zeros); - } - - foreach (AssemblyStoreAssemblyInfo assembly in assemblies) { - string assemblyName = GetAssemblyName (assembly); - string archivePath = assembly.ArchiveAssemblyPath; - if (archivePath.StartsWith (archiveAssembliesPrefix, StringComparison.OrdinalIgnoreCase)) { - archivePath = archivePath.Substring (archiveAssembliesPrefix.Length); - } - - if (!String.IsNullOrEmpty (assembly.Abi)) { - string abiPath = $"{assembly.Abi}/"; - if (archivePath.StartsWith (abiPath, StringComparison.Ordinal)) { - archivePath = archivePath.Substring (abiPath.Length); - } - } - - if (!String.IsNullOrEmpty (archivePath)) { - if (archivePath.EndsWith ("/", StringComparison.Ordinal)) { - assemblyName = $"{archivePath}{assemblyName}"; - } else { - assemblyName = $"{archivePath}/{assemblyName}"; - } - } - - AssemblyStoreIndexEntry entry = WriteAssembly (writer, assembly, assemblyName, (uint)localAssemblies.Count); - if (addToGlobalIndex) { - globalIndex.Add (entry); - } - localAssemblies.Add (entry); - } - - writer.Flush (); - - if (IsIndexStore) { - return; - } - - writer.Seek (0, SeekOrigin.Begin); - WriteBlobHeader (writer, (uint)localAssemblies.Count); - WriteAssemblyDescriptors (writer, localAssemblies); - } - - uint CalculateOffsetFixup (uint localAssemblyCount, uint extraOffset = 0) - { - return (BlobBundledAssemblyNativeStructSize * (uint)localAssemblyCount) + extraOffset; - } - - void WriteBlobHeader (BinaryWriter writer, uint localEntryCount, uint globalEntryCount = 0) - { - // Header, must be identical to the BundledAssemblyBlobHeader structure in src/monodroid/jni/xamarin-app.hh - writer.Write (BlobMagic); // magic - writer.Write (BlobVersion); // version - writer.Write (localEntryCount); // local_entry_count - writer.Write (globalEntryCount); // global_entry_count - writer.Write ((uint)ID); // blob_id - } - - void WriteAssemblyDescriptors (BinaryWriter writer, List assemblies, uint offsetFixup = 0) - { - // Each assembly must be identical to the BlobBundledAssembly structure in src/monodroid/jni/xamarin-app.hh - - foreach (AssemblyStoreIndexEntry assembly in assemblies) { - AdjustOffsets (assembly, offsetFixup); - - writer.Write (assembly.DataOffset); - writer.Write (assembly.DataSize); - - writer.Write (assembly.DebugDataOffset); - writer.Write (assembly.DebugDataSize); - - writer.Write (assembly.ConfigDataOffset); - writer.Write (assembly.ConfigDataSize); - } - } - - void AdjustOffsets (AssemblyStoreIndexEntry assembly, uint offsetFixup) - { - if (offsetFixup == 0) { - return; - } - - assembly.DataOffset += offsetFixup; - - if (assembly.DebugDataOffset > 0) { - assembly.DebugDataOffset += offsetFixup; - } - - if (assembly.ConfigDataOffset > 0) { - assembly.ConfigDataOffset += offsetFixup; - } - } - - AssemblyStoreIndexEntry WriteAssembly (BinaryWriter writer, AssemblyStoreAssemblyInfo assembly, string assemblyName, uint localBlobIndex) - { - uint offset; - uint size; - - (offset, size) = WriteFile (assembly.FilesystemAssemblyPath, true); - - // NOTE: globalAssemblIndex++ is not thread safe but it **must** increase monotonically (see also ArchAssemblyStore.Generate for a special case) - var ret = new AssemblyStoreIndexEntry (assemblyName, ID, GlobalIndexCounter.Increment (), localBlobIndex) { - DataOffset = offset, - DataSize = size, - }; - - (offset, size) = WriteFile (assembly.DebugInfoPath, required: false); - if (offset != 0 && size != 0) { - ret.DebugDataOffset = offset; - ret.DebugDataSize = size; - } - - // Config files must end with \0 (nul) - (offset, size) = WriteFile (assembly.ConfigPath, required: false, appendNul: true); - if (offset != 0 && size != 0) { - ret.ConfigDataOffset = offset; - ret.ConfigDataSize = size; - } - - return ret; - - (uint offset, uint size) WriteFile (string filePath, bool required, bool appendNul = false) - { - if (!File.Exists (filePath)) { - if (required) { - throw new InvalidOperationException ($"Required file '{filePath}' not found"); - } - - return (0, 0); - } - - var fi = new FileInfo (filePath); - if (fi.Length == 0) { - return (0, 0); - } - - if (fi.Length > UInt32.MaxValue || writer.BaseStream.Position + fi.Length > UInt32.MaxValue) { - throw new InvalidOperationException ($"Writing assembly '{filePath}' to assembly blob would exceed the maximum allowed data size."); - } - - uint offset = (uint)writer.BaseStream.Position; - using (var fs = File.Open (filePath, FileMode.Open, FileAccess.Read, FileShare.Read)) { - fs.CopyTo (writer.BaseStream); - } - - uint length = (uint)fi.Length; - if (appendNul) { - length++; - writer.Write ((byte)0); - } - - return (offset, length); - } - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs deleted file mode 100644 index c5c166fb787..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs +++ /dev/null @@ -1,48 +0,0 @@ -using System; -using System.IO; - -namespace Xamarin.Android.Tasks -{ - class AssemblyStoreAssemblyInfo - { - public string FilesystemAssemblyPath { get; } - public string ArchiveAssemblyPath { get; } - public string DebugInfoPath { get; private set; } - public string ConfigPath { get; private set; } - public string Abi { get; } - - public AssemblyStoreAssemblyInfo (string filesystemAssemblyPath, string archiveAssemblyPath, string abi) - { - if (String.IsNullOrEmpty (filesystemAssemblyPath)) { - throw new ArgumentException ("must not be null or empty", nameof (filesystemAssemblyPath)); - } - - if (String.IsNullOrEmpty (archiveAssemblyPath)) { - throw new ArgumentException ("must not be null or empty", nameof (archiveAssemblyPath)); - } - - FilesystemAssemblyPath = filesystemAssemblyPath; - ArchiveAssemblyPath = archiveAssemblyPath; - Abi = abi; - } - - public void SetDebugInfoPath (string path) - { - DebugInfoPath = GetExistingPath (path); - } - - public void SetConfigPath (string path) - { - ConfigPath = GetExistingPath (path); - } - - string GetExistingPath (string path) - { - if (String.IsNullOrEmpty (path) || !File.Exists (path)) { - return String.Empty; - } - - return path; - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs deleted file mode 100644 index d60af903bc9..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs +++ /dev/null @@ -1,133 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Text; - -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; - -namespace Xamarin.Android.Tasks -{ - class AssemblyStoreGenerator - { - sealed class Store - { - public AssemblyStore Common; - public AssemblyStore Arch; - } - - readonly string archiveAssembliesPrefix; - readonly TaskLoggingHelper log; - - // NOTE: when/if we have parallel BuildApk these should become concurrent collections - readonly Dictionary stores = new Dictionary (StringComparer.Ordinal); - - AssemblyStore indexStore; - - // IDs must be counted per AssemblyStoreGenerator instance because it's possible that a single build will create more than one instance of the class and each time - // the stores must be assigned IDs starting from 0, or there will be errors due to "missing" index store - readonly Dictionary apkIds = new Dictionary (StringComparer.Ordinal); - - // Global assembly index must be restarted from 0 for the same reasons as apkIds above and at the same time it must be unique for each assembly added to **any** - // assembly store, thus we need to keep the state here - AssemblyStoreGlobalIndex globalIndexCounter = new AssemblyStoreGlobalIndex (); - - public AssemblyStoreGenerator (string archiveAssembliesPrefix, TaskLoggingHelper log) - { - if (String.IsNullOrEmpty (archiveAssembliesPrefix)) { - throw new ArgumentException ("must not be null or empty", nameof (archiveAssembliesPrefix)); - } - - this.archiveAssembliesPrefix = archiveAssembliesPrefix; - this.log = log; - } - - public void Add (string apkName, AssemblyStoreAssemblyInfo storeAssembly) - { - if (String.IsNullOrEmpty (apkName)) { - throw new ArgumentException ("must not be null or empty", nameof (apkName)); - } - - Store store; - if (!stores.ContainsKey (apkName)) { - store = new Store { - Common = new CommonAssemblyStore (apkName, archiveAssembliesPrefix, log, GetNextStoreID (apkName), globalIndexCounter), - Arch = new ArchAssemblyStore (apkName, archiveAssembliesPrefix, log, GetNextStoreID (apkName), globalIndexCounter) - }; - - stores.Add (apkName, store); - SetIndexStore (store.Common); - SetIndexStore (store.Arch); - } - - store = stores[apkName]; - if (String.IsNullOrEmpty (storeAssembly.Abi)) { - store.Common.Add (storeAssembly); - } else { - store.Arch.Add (storeAssembly); - } - - void SetIndexStore (AssemblyStore b) - { - if (!b.IsIndexStore) { - return; - } - - if (indexStore != null) { - throw new InvalidOperationException ("Index store already set!"); - } - - indexStore = b; - } - } - - uint GetNextStoreID (string apkName) - { - // NOTE: NOT thread safe, if we ever have parallel runs of BuildApk this operation must either be atomic or protected with a lock - if (!apkIds.ContainsKey (apkName)) { - apkIds.Add (apkName, 0); - } - return apkIds[apkName]++; - } - - public Dictionary> Generate (string outputDirectory) - { - if (stores.Count == 0) { - return null; - } - - if (indexStore == null) { - throw new InvalidOperationException ("Index store not found"); - } - - var globalIndex = new List (); - var ret = new Dictionary> (StringComparer.Ordinal); - string indexStoreApkName = null; - foreach (var kvp in stores) { - string apkName = kvp.Key; - Store store = kvp.Value; - - if (!ret.ContainsKey (apkName)) { - ret.Add (apkName, new List ()); - } - - if (store.Common == indexStore || store.Arch == indexStore) { - indexStoreApkName = apkName; - } - - GenerateStore (store.Common, apkName); - GenerateStore (store.Arch, apkName); - } - - string manifestPath = indexStore.WriteIndex (globalIndex); - ret[indexStoreApkName].Add (manifestPath); - - return ret; - - void GenerateStore (AssemblyStore store, string apkName) - { - store.Generate (outputDirectory, globalIndex, ret[apkName]); - } - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGlobalIndex.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGlobalIndex.cs deleted file mode 100644 index 6ce93f11f9d..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGlobalIndex.cs +++ /dev/null @@ -1,29 +0,0 @@ -namespace Xamarin.Android.Tasks -{ - // This class may seem weird, but it's designed with the specific needs of AssemblyStore instances in mind and also prepared for thread-safe use in the future, should the - // need arise - sealed class AssemblyStoreGlobalIndex - { - uint value = 0; - - public uint Value => value; - - /// - /// Increments the counter and returns its previous value - /// - public uint Increment () - { - uint ret = value++; - return ret; - } - - public void Subtract (uint count) - { - if (value < count) { - return; - } - - value -= count; - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreIndexEntry.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreIndexEntry.cs deleted file mode 100644 index 51d2bd8ca77..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreIndexEntry.cs +++ /dev/null @@ -1,43 +0,0 @@ -using System; -using System.IO.Hashing; -using System.Text; - -namespace Xamarin.Android.Tasks -{ - class AssemblyStoreIndexEntry - { - public string Name { get; } - public uint StoreID { get; } - public uint MappingIndex { get; } - public uint LocalBlobIndex { get; } - - // Hash values must have the same type as they are inside a union in the native code - public ulong NameHash64 { get; } - public ulong NameHash32 { get; } - - public uint DataOffset { get; set; } - public uint DataSize { get; set; } - - public uint DebugDataOffset { get; set; } - public uint DebugDataSize { get; set; } - - public uint ConfigDataOffset { get; set; } - public uint ConfigDataSize { get; set; } - - public AssemblyStoreIndexEntry (string name, uint blobID, uint mappingIndex, uint localBlobIndex) - { - if (String.IsNullOrEmpty (name)) { - throw new ArgumentException ("must not be null or empty", nameof (name)); - } - - Name = name; - StoreID = blobID; - MappingIndex = mappingIndex; - LocalBlobIndex = localBlobIndex; - - byte[] nameBytes = Encoding.UTF8.GetBytes (name); - NameHash32 = XxHash32.HashToUInt32 (nameBytes); - NameHash64 = XxHash64.HashToUInt64 (nameBytes); - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyStore.cs b/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyStore.cs deleted file mode 100644 index 9709a200e5a..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyStore.cs +++ /dev/null @@ -1,39 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Text; - -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; - -namespace Xamarin.Android.Tasks -{ - class CommonAssemblyStore : AssemblyStore - { - readonly List assemblies; - - public CommonAssemblyStore (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log, uint id, AssemblyStoreGlobalIndex globalIndexCounter) - : base (apkName, archiveAssembliesPrefix, log, id, globalIndexCounter) - { - assemblies = new List (); - } - - public override void Add (AssemblyStoreAssemblyInfo blobAssembly) - { - if (!String.IsNullOrEmpty (blobAssembly.Abi)) { - throw new InvalidOperationException ($"Architecture-specific assembly cannot be added to an architecture-agnostic blob ({blobAssembly.FilesystemAssemblyPath})"); - } - - assemblies.Add (blobAssembly); - } - - public override void Generate (string outputDirectory, List globalIndex, List blobPaths) - { - if (assemblies.Count == 0) { - return; - } - - Generate (Path.Combine (outputDirectory, $"{ApkName}_{BlobPrefix}{BlobExtension}"), assemblies, globalIndex, blobPaths); - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssembliesNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssembliesNativeAssemblyGenerator.cs deleted file mode 100644 index e0b3740a453..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssembliesNativeAssemblyGenerator.cs +++ /dev/null @@ -1,146 +0,0 @@ -using System; -using System.Collections.Generic; - -using Xamarin.Android.Tasks.LLVMIR; - -namespace Xamarin.Android.Tasks -{ - partial class CompressedAssembliesNativeAssemblyGenerator : LlvmIrComposer - { - const string DescriptorsArraySymbolName = "compressed_assembly_descriptors"; - const string CompressedAssembliesSymbolName = "compressed_assemblies"; - - sealed class CompressedAssemblyDescriptorContextDataProvider : NativeAssemblerStructContextDataProvider - { - public override string? GetPointedToSymbolName (object data, string fieldName) - { - if (String.Compare ("data", fieldName, StringComparison.Ordinal) != 0) { - return null; - } - - var descriptor = EnsureType (data); - return descriptor.BufferSymbolName; - } - } - - // Order of fields and their type must correspond *exactly* to that in - // src/monodroid/jni/xamarin-app.hh CompressedAssemblyDescriptor structure - [NativeAssemblerStructContextDataProvider (typeof (CompressedAssemblyDescriptorContextDataProvider))] - sealed class CompressedAssemblyDescriptor - { - [NativeAssembler (Ignore = true)] - public string BufferSymbolName; - - public uint uncompressed_file_size; - public bool loaded; - - [NativeAssembler (UsesDataProvider = true), NativePointer (PointsToSymbol = "")] - public byte data; - }; - - sealed class CompressedAssembliesContextDataProvider : NativeAssemblerStructContextDataProvider - { - public override ulong GetBufferSize (object data, string fieldName) - { - if (String.Compare ("descriptors", fieldName, StringComparison.Ordinal) != 0) { - return 0; - } - - var cas = EnsureType (data); - return cas.count; - } - } - - // Order of fields and their type must correspond *exactly* to that in - // src/monodroid/jni/xamarin-app.hh CompressedAssemblies structure - [NativeAssemblerStructContextDataProvider (typeof (CompressedAssembliesContextDataProvider))] - sealed class CompressedAssemblies - { - public uint count; - - [NativeAssembler (UsesDataProvider = true), NativePointer (PointsToSymbol = DescriptorsArraySymbolName)] - public CompressedAssemblyDescriptor descriptors; - }; - - IDictionary assemblies; - StructureInfo compressedAssemblyDescriptorStructureInfo; - StructureInfo compressedAssembliesStructureInfo; - - public CompressedAssembliesNativeAssemblyGenerator (IDictionary assemblies) - { - this.assemblies = assemblies; - } - - void InitCompressedAssemblies (out List>? compressedAssemblyDescriptors, - out StructureInstance? compressedAssemblies, - out List? buffers) - { - if (assemblies == null || assemblies.Count == 0) { - compressedAssemblyDescriptors = null; - compressedAssemblies = null; - buffers = null; - return; - } - - ulong counter = 0; - compressedAssemblyDescriptors = new List> (assemblies.Count); - buffers = new List (assemblies.Count); - foreach (var kvp in assemblies) { - string assemblyName = kvp.Key; - CompressedAssemblyInfo info = kvp.Value; - - string bufferName = $"__compressedAssemblyData_{counter++}"; - var descriptor = new CompressedAssemblyDescriptor { - BufferSymbolName = bufferName, - uncompressed_file_size = info.FileSize, - loaded = false, - data = 0 - }; - - var bufferVar = new LlvmIrGlobalVariable (typeof(List), bufferName, LlvmIrVariableOptions.LocalWritable) { - ZeroInitializeArray = true, - ArrayItemCount = descriptor.uncompressed_file_size, - }; - buffers.Add (bufferVar); - - compressedAssemblyDescriptors.Add (new StructureInstance (compressedAssemblyDescriptorStructureInfo, descriptor)); - } - - compressedAssemblies = new StructureInstance (compressedAssembliesStructureInfo, new CompressedAssemblies { count = (uint)assemblies.Count }); - } - - protected override void Construct (LlvmIrModule module) - { - MapStructures (module); - - List>? compressedAssemblyDescriptors; - StructureInstance? compressedAssemblies; - List? buffers; - - InitCompressedAssemblies (out compressedAssemblyDescriptors, out compressedAssemblies, out buffers); - - if (compressedAssemblyDescriptors == null) { - module.AddGlobalVariable ( - typeof(StructureInstance), - CompressedAssembliesSymbolName, - new StructureInstance (compressedAssembliesStructureInfo, new CompressedAssemblies ()) { IsZeroInitialized = true }, - LlvmIrVariableOptions.GlobalWritable - ); - return; - } - - module.AddGlobalVariable (CompressedAssembliesSymbolName, compressedAssemblies, LlvmIrVariableOptions.GlobalWritable); - module.AddGlobalVariable (DescriptorsArraySymbolName, compressedAssemblyDescriptors, LlvmIrVariableOptions.LocalWritable); - - module.Add (new LlvmIrGroupDelimiterVariable ()); - module.Add (buffers); - module.Add (new LlvmIrGroupDelimiterVariable ()); - } - - void MapStructures (LlvmIrModule module) - { - compressedAssemblyDescriptorStructureInfo = module.MapStructure (); - compressedAssembliesStructureInfo = module.MapStructure (); - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssemblyInfo.cs b/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssemblyInfo.cs deleted file mode 100644 index b3f775a96b7..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssemblyInfo.cs +++ /dev/null @@ -1,40 +0,0 @@ -using System; -using System.IO; -using Microsoft.Build.Framework; - -namespace Xamarin.Android.Tasks -{ - class CompressedAssemblyInfo - { - const string CompressedAssembliesInfoKey = "__CompressedAssembliesInfo"; - - public uint FileSize { get; } - public uint DescriptorIndex { get; set; } - - public CompressedAssemblyInfo (uint fileSize) - { - FileSize = fileSize; - DescriptorIndex = 0; - } - - public static string GetKey (string projectFullPath) - { - if (String.IsNullOrEmpty (projectFullPath)) - throw new ArgumentException ("must be a non-empty string", nameof (projectFullPath)); - - return $"{CompressedAssembliesInfoKey}:{projectFullPath}"; - } - - public static string GetDictionaryKey (ITaskItem assembly) - { - // Prefer %(DestinationSubPath) if set - var path = assembly.GetMetadata ("DestinationSubPath"); - if (!string.IsNullOrEmpty (path)) { - return path; - } - // MSBuild sometimes only sets %(DestinationSubDirectory) - var subDirectory = assembly.GetMetadata ("DestinationSubDirectory"); - return Path.Combine (subDirectory, Path.GetFileName (assembly.ItemSpec)); - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/DSOAssemblyInfo.cs b/src/Xamarin.Android.Build.Tasks/Utilities/DSOAssemblyInfo.cs new file mode 100644 index 00000000000..8563bc81d6c --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/DSOAssemblyInfo.cs @@ -0,0 +1,55 @@ +using System; + +namespace Xamarin.Android.Tasks; + +class DSOAssemblyInfo +{ + /// + /// Size of the loadable assembly data (after decompression, if compression is enabled). + /// + public uint DataSize { get; } + + /// + /// Size of the compressed assembly data or `0` if assembly is uncompressed. + /// + public uint CompressedDataSize { get; } + + /// + /// The file data comes from, either the original assembly or its compressed copy + /// + public string InputFile { get; } + + /// + /// Name of the assembly, including culture prefix if it's a satellite assembly. Must include the extension. + /// + public string Name { get; } + + /// + /// Indicates whether assembly data is stored in a standalone shared library. + /// + public bool IsStandalone { get; } + + public string? StandaloneDSOName { get; } + public uint? AssemblyLoadInfoIndex { get; set; } + public ulong? AssemblyDataSymbolOffset { get; set; } + + /// + /// is the original assembly name, including culture prefix (e.g. `en_US/`) if it is a + /// satellite assembly. should be the full path to the input file. + /// gives the original file size, while specifies + /// data size after compression, or `0` if file isn't compressed. + /// + public DSOAssemblyInfo (string name, string inputFile, uint dataSize, uint compressedDataSize, bool isStandalone = false, string? dsoName = null) + { + if (isStandalone && String.IsNullOrEmpty (dsoName)) { + throw new ArgumentException ("must not be null or empty for standalone assembly", nameof (dsoName)); + } + + Name = name; + InputFile = inputFile; + DataSize = dataSize; + CompressedDataSize = compressedDataSize; + IsStandalone = isStandalone; + StandaloneDSOName = isStandalone ? dsoName : null; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/DSOMetadata.cs b/src/Xamarin.Android.Build.Tasks/Utilities/DSOMetadata.cs new file mode 100644 index 00000000000..129b7ebc60f --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/DSOMetadata.cs @@ -0,0 +1,16 @@ +namespace Xamarin.Android.Tasks; + +public static class DSOMetadata +{ + public const string Abi = "Abi"; + public const string AndroidSkipCompression = "AndroidSkipCompression"; + public const string AssemblyLoadInfoIndex = "AssemblyLoadInfoIndex"; + public const string Compressed = "Compressed"; + public const string DataSize = "DataSize"; + public const string DataSymbolOffset = "DataSymbolOffset"; + public const string InputAssemblyPath = "InputAssemblyPath"; + public const string OriginalAssemblyPath = "OriginalAssemblyPath"; + public const string SatelliteAssemblyCulture = "SatelliteAssemblyCulture"; + public const string SourceFileBaseName = "SourceFileBaseName"; + public const string UncompressedDataSize = "UncompressedDataSize"; +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ELFHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ELFHelper.cs index a7a00b4af60..c40d917edd1 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ELFHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ELFHelper.cs @@ -15,6 +15,40 @@ namespace Xamarin.Android.Tasks { static class ELFHelper { + public static (ulong? offset, ulong? size) GetExportedSymbolOffsetAndSize (TaskLoggingHelper log, string dsoPath, string symbolName) + { + if (String.IsNullOrEmpty (dsoPath) || !File.Exists (dsoPath)) { + throw new ArgumentException ("must not be null or empty and the file must exist", nameof (dsoPath)); + } + + IELF elf = ELFReader.Load (dsoPath); + ISymbolTable? symtab = GetSymbolTable (elf, ".dynsym"); + if (symtab == null) { + log.LogDebugMessage ($"Shared library '{dsoPath}' does not export any symbols"); + return (null, null); + } + + ISymbolEntry? symbol = null; + foreach (var entry in symtab.Entries) { + if (entry.Type == ELFSymbolType.Object && String.Compare (symbolName, entry.Name, StringComparison.Ordinal) == 0) { + symbol = entry; + break; + } + } + + if (symbol == null) { + log.LogDebugMessage ($"Shared library '{dsoPath}' does not export symbol '{symbolName}'"); + return (null, null); + } + + if (elf.Class == Class.Bit64) { + var sym64 = (SymbolEntry)symbol; + return (sym64.Value, sym64.Size); + } + var sym32 = (SymbolEntry)symbol; + return ((ulong)sym32.Value, (ulong)sym32.Size); + } + public static bool IsEmptyAOTLibrary (TaskLoggingHelper log, string path) { if (String.IsNullOrEmpty (path) || !File.Exists (path)) { diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/IInputAssemblySet.cs b/src/Xamarin.Android.Build.Tasks/Utilities/IInputAssemblySet.cs new file mode 100644 index 00000000000..9c4a7c16cab --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/IInputAssemblySet.cs @@ -0,0 +1,17 @@ +using System; +using System.IO; + +using Microsoft.Build.Framework; + +namespace Xamarin.Android.Tasks; + +abstract class InputAssemblySet +{ + public abstract void AddJavaTypeAssembly (ITaskItem assemblyItem); + public abstract void AddUserAssembly (ITaskItem assemblyItem); + public abstract bool IsUserAssembly (string name); + + protected static readonly StringComparer AssemblyNameStringComparer = StringComparer.OrdinalIgnoreCase; + + protected string GetUserAssemblyKey (ITaskItem assemblyItem) => Path.GetFileNameWithoutExtension (assemblyItem.ItemSpec); +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs index 8db94269f32..2baa5c0fcc6 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs @@ -32,11 +32,19 @@ public void Generate (LlvmIrModule module, AndroidTargetArch arch, StreamWriter LlvmIrGenerator generator = LlvmIrGenerator.Create (arch, fileName); generator.Generate (output, module); output.Flush (); + + CleanupAfterGeneration (arch); } - public static ulong GetXxHash (string str, bool is64Bit) + protected virtual void CleanupAfterGeneration (AndroidTargetArch arch) + {} + + public static byte[] StringToBytes (string str) => Encoding.UTF8.GetBytes (str); + + public static ulong GetXxHash (string str, bool is64Bit) => GetXxHash (StringToBytes (str), is64Bit); + + public static ulong GetXxHash (byte[] stringBytes, bool is64Bit) { - byte[] stringBytes = Encoding.UTF8.GetBytes (str); if (is64Bit) { return XxHash64.HashToUInt64 (stringBytes); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs index bb81b03db0e..c3102f3a5c3 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs @@ -29,6 +29,7 @@ sealed class GeneratorWriteContext public readonly LlvmIrMetadataManager MetadataManager; public string CurrentIndent { get; private set; } = String.Empty; public bool InVariableGroup { get; set; } + public LlvmIrVariableNumberFormat NumberFormat { get; set; } = LlvmIrVariableNumberFormat.Default; public GeneratorWriteContext (TextWriter writer, LlvmIrModule module, LlvmIrModuleTarget target, LlvmIrMetadataManager metadataManager) { @@ -80,31 +81,56 @@ sealed class BasicType public readonly string Name; public readonly ulong Size; public readonly bool IsNumeric; + public readonly bool IsUnsigned; + public readonly bool PreferHex; + public readonly string HexFormat; - public BasicType (string name, ulong size, bool isNumeric = true) + public BasicType (string name, ulong size, bool isNumeric = true, bool isUnsigned = false, bool? preferHex = null) { Name = name; Size = size; IsNumeric = isNumeric; + IsUnsigned = isUnsigned; + + // If hex preference isn't specified, we determine whether the type wants to be represented in + // the hexadecimal notation based on signedness. Unsigned types will be represented in hexadecimal, + // but signed types will remain decimal, as it's easier for humans to see the actual value of the + // variable, given this note from LLVM IR manual: + // + // Note that hexadecimal integers are sign extended from the number of active bits, i.e. the bit width minus the number of leading zeros. So ‘s0x0001’ of type ‘i16’ will be -1, not 1. + // + // See: https://llvm.org/docs/LangRef.html#simple-constants + // + if (preferHex.HasValue) { + PreferHex = preferHex.Value; + } else { + PreferHex = isUnsigned; + } + if (!PreferHex) { + HexFormat = String.Empty; + return; + } + + HexFormat = $"x{size * 2}"; } } public const string IRPointerType = "ptr"; static readonly Dictionary basicTypeMap = new Dictionary { - { typeof (bool), new ("i8", 1, isNumeric: false) }, - { typeof (byte), new ("i8", 1) }, - { typeof (char), new ("i16", 2) }, + { typeof (bool), new ("i1", 1, isNumeric: false, isUnsigned: true, preferHex: false) }, + { typeof (byte), new ("i8", 1, isUnsigned: true) }, + { typeof (char), new ("i16", 2, isUnsigned: true, preferHex: false) }, { typeof (sbyte), new ("i8", 1) }, { typeof (short), new ("i16", 2) }, - { typeof (ushort), new ("i16", 2) }, + { typeof (ushort), new ("i16", 2, isUnsigned: true) }, { typeof (int), new ("i32", 4) }, - { typeof (uint), new ("i32", 4) }, + { typeof (uint), new ("i32", 4, isUnsigned: true) }, { typeof (long), new ("i64", 8) }, - { typeof (ulong), new ("i64", 8) }, + { typeof (ulong), new ("i64", 8, isUnsigned: true) }, { typeof (float), new ("float", 4) }, { typeof (double), new ("double", 8) }, - { typeof (void), new ("void", 0, isNumeric: false) }, + { typeof (void), new ("void", 0, isNumeric: false, preferHex: false) }, }; public string FilePath { get; } @@ -191,6 +217,8 @@ void WriteGlobalVariables (GeneratorWriteContext context) } foreach (LlvmIrGlobalVariable gv in context.Module.GlobalVariables) { + context.NumberFormat = gv.NumberFormat; + if (gv is LlvmIrGroupDelimiterVariable groupDelimiter) { if (!context.InVariableGroup && !String.IsNullOrEmpty (groupDelimiter.Comment)) { context.Output.WriteLine (); @@ -240,8 +268,10 @@ void WriteGlobalVariable (GeneratorWriteContext context, LlvmIrGlobalVariable va context.Output.Write (", align "); ulong alignment; - if (typeInfo.IsAggregate) { - ulong count = GetAggregateValueElementCount (variable); + if (variable.Alignment.HasValue) { + alignment = variable.Alignment.Value; + } else if (typeInfo.IsAggregate) { + ulong count = GetAggregateValueElementCount (context, variable); alignment = (ulong)target.GetAggregateAlignment ((int)typeInfo.MaxFieldAlignment, count * typeInfo.Size); } else if (typeInfo.IsStructure) { alignment = (ulong)target.GetAggregateAlignment ((int)typeInfo.MaxFieldAlignment, typeInfo.Size); @@ -290,9 +320,9 @@ void WriteTypeAndValue (GeneratorWriteContext context, LlvmIrVariable variable, WriteValue (context, valueType, variable); } - ulong GetAggregateValueElementCount (LlvmIrVariable variable) => GetAggregateValueElementCount (variable.Type, variable.Value, variable as LlvmIrGlobalVariable); + ulong GetAggregateValueElementCount (GeneratorWriteContext context, LlvmIrVariable variable) => GetAggregateValueElementCount (context, variable.Type, variable.Value, variable as LlvmIrGlobalVariable); - ulong GetAggregateValueElementCount (Type type, object? value, LlvmIrGlobalVariable? globalVariable = null) + ulong GetAggregateValueElementCount (GeneratorWriteContext context, Type type, object? value, LlvmIrGlobalVariable? globalVariable = null) { if (!type.IsArray ()) { throw new InvalidOperationException ($"Internal error: unknown type {type} when trying to determine aggregate type element count"); @@ -300,6 +330,9 @@ ulong GetAggregateValueElementCount (Type type, object? value, LlvmIrGlobalVaria if (value == null) { if (globalVariable != null) { + if (globalVariable.ArrayDataProvider != null) { + return globalVariable.ArrayDataProvider.GetTotalDataSize (context.Target); + } return globalVariable.ArrayItemCount; } return 0; @@ -386,9 +419,9 @@ void WriteType (GeneratorWriteContext context, Type type, object? value, out Llv if (type.IsArray ()) { Type elementType = type.GetArrayElementType (); - ulong elementCount = GetAggregateValueElementCount (type, value, globalVariable); + ulong elementCount = GetAggregateValueElementCount (context, type, value, globalVariable); - WriteArrayType (context, elementType, elementCount, out typeInfo); + WriteArrayType (context, elementType, elementCount, globalVariable, out typeInfo); return; } @@ -404,6 +437,11 @@ void WriteType (GeneratorWriteContext context, Type type, object? value, out Llv } void WriteArrayType (GeneratorWriteContext context, Type elementType, ulong elementCount, out LlvmTypeInfo typeInfo) + { + WriteArrayType (context, elementType, elementCount, variable: null, out typeInfo); + } + + void WriteArrayType (GeneratorWriteContext context, Type elementType, ulong elementCount, LlvmIrGlobalVariable? variable, out LlvmTypeInfo typeInfo) { string irType; ulong size; @@ -420,6 +458,35 @@ void WriteArrayType (GeneratorWriteContext context, Type elementType, ulong elem } else { irType = GetIRType (elementType, out size, out isPointer); maxFieldAlignment = size; + + if (elementType.IsArray) { + if (variable == null) { + throw new InvalidOperationException ($"Internal error: array of arrays ({elementType}) requires variable to be defined"); + } + + // For the sake of simpler code, we currently assume that all the element arrays are of the same size, because that's the only scenario + // that we use at this time. + var value = variable.Value as ICollection; + if (value == null) { + throw new InvalidOperationException ($"Internal error: variable '{variable.Name}' of type '{variable.Type}' is required to have a value of type which implements the ICollection interface"); + } + + if (value.Count == 0) { + throw new InvalidOperationException ($"Internal error: variable '{variable.Name}' of type '{variable.Type}' is required to have a value which is a non-empty ICollection"); + } + + Array? firstItem = null; + foreach (object v in value) { + firstItem = (Array)v; + break; + } + + if (firstItem == null) { + throw new InvalidOperationException ($"Internal error: variable '{variable.Name}' of type '{variable.Type}' is required to have a value which is a non-empty ICollection with non-null elements"); + } + + irType = $"[{MonoAndroidHelper.CultureInvariantToString (firstItem.Length)} x {irType}]"; + } } typeInfo = new LlvmTypeInfo ( isPointer: isPointer, @@ -449,12 +516,17 @@ ulong GetStructureMaxFieldAlignment (StructureInfo si) void WriteValue (GeneratorWriteContext context, Type valueType, LlvmIrVariable variable) { + if (variable is LlvmIrGlobalVariable globalVariable && globalVariable.ArrayDataProvider != null) { + WriteStreamedArrayValue (context, globalVariable, globalVariable.ArrayDataProvider); + return; + } + if (variable.Type.IsArray ()) { bool zeroInitialize = false; if (variable is LlvmIrGlobalVariable gv) { zeroInitialize = gv.ZeroInitializeArray || variable.Value == null; } else { - zeroInitialize = GetAggregateValueElementCount (variable) == 0; + zeroInitialize = GetAggregateValueElementCount (context, variable) == 0; } if (zeroInitialize) { @@ -478,6 +550,29 @@ void AssertArraySize (StructureInstance si, StructureMemberInfo smi, ulong lengt throw new InvalidOperationException ($"Invalid array size in field '{smi.Info.Name}' of structure '{si.Info.Name}', expected {expectedLength}, found {length}"); } + void WriteInlineArray (GeneratorWriteContext context, byte[] bytes, bool encodeAsASCII) + { + if (encodeAsASCII) { + context.Output.Write ('c'); + context.Output.Write (QuoteString (bytes, bytes.Length, out _, nullTerminated: false)); + return; + } + + string irType = MapToIRType (typeof(byte)); + bool first = true; + context.Output.Write ("[ "); + foreach (byte b in bytes) { + if (!first) { + context.Output.Write (", "); + } else { + first = false; + } + + context.Output.Write ($"{irType} u0x{b:x02}"); + } + context.Output.Write (" ]"); + } + void WriteValue (GeneratorWriteContext context, StructureInstance structInstance, StructureMemberInfo smi, object? value) { if (smi.IsNativePointer) { @@ -495,8 +590,7 @@ void WriteValue (GeneratorWriteContext context, StructureInstance structInstance // Byte arrays are represented in the same way as strings, without the explicit NUL termination byte AssertArraySize (structInstance, smi, length, smi.ArrayElements); - context.Output.Write ('c'); - context.Output.Write (QuoteString (bytes, bytes.Length, out _, nullTerminated: false)); + WriteInlineArray (context, bytes, encodeAsASCII: false); return; } @@ -549,6 +643,27 @@ bool WriteNativePointerValue (GeneratorWriteContext context, StructureInstance s return false; } + string ToHex (BasicType basicTypeDesc, Type type, object? value) + { + const char prefixSigned = 's'; + const char prefixUnsigned = 'u'; + + string hex; + if (type == typeof(byte)) { + hex = ((byte)value).ToString (basicTypeDesc.HexFormat, CultureInfo.InvariantCulture); + } else if (type == typeof(ushort)) { + hex = ((ushort)value).ToString (basicTypeDesc.HexFormat, CultureInfo.InvariantCulture); + } else if (type == typeof(uint)) { + hex = ((uint)value).ToString (basicTypeDesc.HexFormat, CultureInfo.InvariantCulture); + } else if (type == typeof(ulong)) { + hex = ((ulong)value).ToString (basicTypeDesc.HexFormat, CultureInfo.InvariantCulture); + } else { + throw new NotImplementedException ($"Conversion to hexadecimal from type {type} is not implemented"); + }; + + return $"{(basicTypeDesc.IsUnsigned ? prefixUnsigned : prefixSigned)}0x{hex}"; + } + void WriteValue (GeneratorWriteContext context, Type type, object? value) { if (value is LlvmIrVariable variableRef) { @@ -556,14 +671,26 @@ void WriteValue (GeneratorWriteContext context, Type type, object? value) return; } - if (IsNumeric (type)) { - context.Output.Write (MonoAndroidHelper.CultureInvariantToString (value)); - return; - } + bool isBasic = basicTypeMap.TryGetValue (type, out BasicType basicTypeDesc); + if (isBasic) { + if (basicTypeDesc.IsNumeric) { + bool hex = context.NumberFormat switch { + LlvmIrVariableNumberFormat.Default => basicTypeDesc.PreferHex, + LlvmIrVariableNumberFormat.Decimal => false, + LlvmIrVariableNumberFormat.Hexadecimal => true, + _ => throw new InvalidOperationException ($"Internal error: number format {context.NumberFormat} is unsupported") + }; - if (type == typeof(bool)) { - context.Output.Write ((bool)value ? '1' : '0'); - return; + context.Output.Write ( + hex ? ToHex (basicTypeDesc, type, value) : MonoAndroidHelper.CultureInvariantToString (value) + ); + return; + } + + if (type == typeof(bool)) { + context.Output.Write ((bool)value ? "true" : "false"); + return; + } } if (IsStructureInstance (type)) { @@ -588,8 +715,13 @@ void WriteValue (GeneratorWriteContext context, Type type, object? value) return; } - if (type.IsInlineArray ()) { + if (type.IsArray) { + if (type == typeof(byte[])) { + WriteInlineArray (context, (byte[])value, encodeAsASCII: true); + return; + } + throw new NotSupportedException ($"Internal error: array of type {type} is unsupported"); } throw new NotSupportedException ($"Internal error: value type '{type}' is unsupported"); @@ -616,8 +748,20 @@ void WriteStructureValue (GeneratorWriteContext context, StructureInstance? inst context.Output.Write (' '); object? value = GetTypedMemberValue (context, info, smi, instance, smi.MemberType); + LlvmIrVariableNumberFormat numberFormat = smi.Info.GetNumberFormat (); + LlvmIrVariableNumberFormat? savedNumberFormat = null; + + if (numberFormat != LlvmIrVariableNumberFormat.Default && numberFormat != context.NumberFormat) { + savedNumberFormat = context.NumberFormat; + context.NumberFormat = numberFormat; + } + WriteValue (context, instance, smi, value); + if (savedNumberFormat.HasValue) { + context.NumberFormat = savedNumberFormat.Value; + } + if (i < lastMember) { context.Output.Write (", "); } @@ -628,9 +772,6 @@ void WriteStructureValue (GeneratorWriteContext context, StructureInstance? inst sb.Append (MapManagedTypeToNative (smi)); sb.Append (' '); sb.Append (smi.Info.Name); - if (value != null && smi.MemberType.IsPrimitive && smi.MemberType != typeof(bool)) { - sb.Append ($" (0x{value:x})"); - } comment = sb.ToString (); } WriteCommentLine (context, comment); @@ -641,75 +782,80 @@ void WriteStructureValue (GeneratorWriteContext context, StructureInstance? inst context.Output.Write ('}'); } - void WriteArrayValue (GeneratorWriteContext context, LlvmIrVariable variable) + void WriteArrayValueStart (GeneratorWriteContext context) { - ICollection entries; - if (variable.Type.ImplementsInterface (typeof(IDictionary))) { - var list = new List (); - foreach (var kvp in (IDictionary)variable.Value) { - list.Add (kvp.Key); - list.Add (kvp.Value); - } - entries = list; - } else { - entries = (ICollection)variable.Value; - } - - if (entries.Count == 0) { - context.Output.Write ("zeroinitializer"); - return; - } - context.Output.WriteLine ('['); context.IncreaseIndent (); + } - Type elementType = variable.Type.GetArrayElementType (); - bool writeIndices = (variable.WriteOptions & LlvmIrVariableWriteOptions.ArrayWriteIndexComments) == LlvmIrVariableWriteOptions.ArrayWriteIndexComments; - ulong counter = 0; - string? prevItemComment = null; - uint stride; + void WriteArrayValueEnd (GeneratorWriteContext context) + { + context.DecreaseIndent (); + context.Output.Write (']'); + } + uint GetArrayStride (LlvmIrVariable variable) + { if ((variable.WriteOptions & LlvmIrVariableWriteOptions.ArrayFormatInRows) == LlvmIrVariableWriteOptions.ArrayFormatInRows) { - stride = variable.ArrayStride > 0 ? variable.ArrayStride : 1; - } else { - stride = 1; + return variable.ArrayStride > 0 ? variable.ArrayStride : 1; } + return 1; + } + + void WriteArrayEntries (GeneratorWriteContext context, LlvmIrVariable variable, ICollection? entries, Type elementType, uint stride, bool writeIndices, bool terminateWithComma = false) + { bool first = true; + bool ignoreComments = stride > 1; + string? prevItemComment = null; + ulong counter = 0; - // TODO: implement output in rows - foreach (object entry in entries) { - if (!first) { - context.Output.Write (','); - WritePrevItemCommentOrNewline (); - } else { - first = false; - } + if (entries != null) { + foreach (object entry in entries) { + if (!first) { + context.Output.Write (','); + if (stride == 1 || counter % stride == 0) { + WritePrevItemCommentOrNewline (); + context.Output.Write (context.CurrentIndent); + } else { + context.Output.Write (' '); + } + } else { + context.Output.Write (context.CurrentIndent); + first = false; + } - prevItemComment = null; - if (variable.GetArrayItemCommentCallback != null) { - prevItemComment = variable.GetArrayItemCommentCallback (variable, target, counter, entry, variable.GetArrayItemCommentCallbackCallerState); - } + if (!ignoreComments) { + prevItemComment = null; + if (variable.GetArrayItemCommentCallback != null) { + prevItemComment = variable.GetArrayItemCommentCallback (variable, target, counter, entry, variable.GetArrayItemCommentCallbackCallerState); + } - if (writeIndices && String.IsNullOrEmpty (prevItemComment)) { - prevItemComment = $" {counter}"; - } + if (writeIndices && String.IsNullOrEmpty (prevItemComment)) { + prevItemComment = $" {counter}"; + } + } - counter++; - context.Output.Write (context.CurrentIndent); - WriteType (context, elementType, entry, out _); + counter++; + WriteType (context, elementType, entry, out _); - context.Output.Write (' '); - WriteValue (context, elementType, entry); + context.Output.Write (' '); + WriteValue (context, elementType, entry); + } } - WritePrevItemCommentOrNewline (); - context.DecreaseIndent (); - context.Output.Write (']'); + if (terminateWithComma) { + if (!ignoreComments) { + context.Output.WriteLine (); // must put comma outside the comment + context.Output.Write (context.CurrentIndent); + } + context.Output.Write (','); + } + WritePrevItemCommentOrNewline (); void WritePrevItemCommentOrNewline () { - if (!String.IsNullOrEmpty (prevItemComment)) { + if (!ignoreComments && !String.IsNullOrEmpty (prevItemComment)) { context.Output.Write (' '); WriteCommentLine (context, prevItemComment); } else { @@ -718,6 +864,97 @@ void WritePrevItemCommentOrNewline () } } + bool ArrayWantsToWriteIndices (LlvmIrVariable variable) => (variable.WriteOptions & LlvmIrVariableWriteOptions.ArrayWriteIndexComments) == LlvmIrVariableWriteOptions.ArrayWriteIndexComments; + + void WriteStreamedArrayValue (GeneratorWriteContext context, LlvmIrGlobalVariable variable, LlvmIrStreamedArrayDataProvider dataProvider) + { + ulong dataSizeSoFar = 0; + ulong totalDataSize = dataProvider.GetTotalDataSize (context.Target); + bool first = true; + + WriteArrayValueStart (context); + while (true) { + (LlvmIrStreamedArrayDataProviderState state, ICollection? data) = dataProvider.GetData (context.Target); + if (state == LlvmIrStreamedArrayDataProviderState.NextSectionNoData) { + continue; + } + + bool mustHaveData = state != LlvmIrStreamedArrayDataProviderState.LastSectionNoData; + if (mustHaveData) { + if (data.Count == 0) { + throw new InvalidOperationException ("Data must be provided for streamed arrays"); + } + + dataSizeSoFar += (ulong)data.Count; + if (dataSizeSoFar > totalDataSize) { + throw new InvalidOperationException ($"Data provider {dataProvider} is trying to write more data than declared"); + } + + if (first) { + first = false; + } else { + context.Output.WriteLine (); + } + string comment = dataProvider.GetSectionStartComment (context.Target); + + if (comment.Length > 0) { + context.Output.Write (context.CurrentIndent); + WriteCommentLine (context, comment); + } + } + + bool lastSection = state == LlvmIrStreamedArrayDataProviderState.LastSection || state == LlvmIrStreamedArrayDataProviderState.LastSectionNoData; + WriteArrayEntries ( + context, + variable, + data, + dataProvider.ArrayElementType, + GetArrayStride (variable), + writeIndices: false, + terminateWithComma: !lastSection + ); + + if (lastSection) { + break; + } + + } + WriteArrayValueEnd (context); + } + + void WriteArrayValue (GeneratorWriteContext context, LlvmIrVariable variable) + { + ICollection entries; + if (variable.Type.ImplementsInterface (typeof(IDictionary))) { + var list = new List (); + foreach (var kvp in (IDictionary)variable.Value) { + list.Add (kvp.Key); + list.Add (kvp.Value); + } + entries = list; + } else { + entries = (ICollection)variable.Value; + } + + if (entries.Count == 0) { + context.Output.Write ("zeroinitializer"); + return; + } + + WriteArrayValueStart (context); + + WriteArrayEntries ( + context, + variable, + entries, + variable.Type.GetArrayElementType (), + GetArrayStride (variable), + writeIndices: ArrayWantsToWriteIndices (variable) + ); + + WriteArrayValueEnd (context); + } + void WriteLinkage (GeneratorWriteContext context, LlvmIrLinkage linkage) { if (linkage == LlvmIrLinkage.Default) { @@ -1232,8 +1469,6 @@ static string MapManagedTypeToNative (StructureMemberInfo smi) return $"{nativeType}*"; } - static bool IsNumeric (Type type) => basicTypeMap.TryGetValue (type, out BasicType typeDesc) && typeDesc.IsNumeric; - object? GetTypedMemberValue (GeneratorWriteContext context, StructureInfo info, StructureMemberInfo smi, StructureInstance instance, Type expectedType, object? defaultValue = null) { object? value = smi.GetValue (instance.Obj); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs index 7f741490ce0..49524f0bf6c 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs @@ -1,4 +1,5 @@ using System; +using System.Collections; using System.Globalization; namespace Xamarin.Android.Tasks.LLVMIR; @@ -11,6 +12,13 @@ enum LlvmIrVariableWriteOptions ArrayFormatInRows = 0x0002, } +enum LlvmIrVariableNumberFormat +{ + Default, + Hexadecimal, + Decimal, +} + abstract class LlvmIrVariable : IEquatable { public abstract bool Global { get; } @@ -29,6 +37,8 @@ abstract class LlvmIrVariable : IEquatable public object? Value { get; set; } public string? Comment { get; set; } + public LlvmIrVariableNumberFormat NumberFormat { get; set; } = LlvmIrVariableNumberFormat.Decimal; + /// /// Both global and local variables will want their names to matter in equality checks, but function /// parameters must not take it into account, thus this property. If set to false, @@ -146,6 +156,50 @@ public void AssignNumber (ulong n) } } +enum LlvmIrStreamedArrayDataProviderState +{ + NextSection, + LastSection, + NextSectionNoData, + LastSectionNoData, +} + +abstract class LlvmIrStreamedArrayDataProvider +{ + /// + /// Type of every member of the array returned by . Generator will check + /// every member type against this property, allowing also derived types. + /// + public Type ArrayElementType { get; } + + protected LlvmIrStreamedArrayDataProvider (Type arrayElementType) + { + ArrayElementType = arrayElementType; + } + + /// + /// Whenever returns the generator will call this method to obtain the new section + /// comment, if any, to be output before the actual data. Returning `String.Empty` prevents the comment + /// from being added. + /// + public virtual string GetSectionStartComment (LlvmIrModuleTarget target) => String.Empty; + + /// + /// Provide the next chunk of data for the specified target (ABI). Implementations need to return at least one + /// non-empty collection of data. The returned collection **must** be exactly the size of contained data (e.g. it cannot be + /// a byte array rented from a byte pool, because these can be bigger than requested. When returning the last (or the only) section, + /// must have a value of . + /// Each section may be preceded by a comment, . + /// + public abstract (LlvmIrStreamedArrayDataProviderState status, ICollection data) GetData (LlvmIrModuleTarget target); + + /// + /// Provide the total data size for the specified target (ABI). This needs to be used instead of + /// because a variable instance is created once and shared by all targets, while per-target data sets might have different sizes. + /// + public abstract ulong GetTotalDataSize (LlvmIrModuleTarget target); +} + class LlvmIrGlobalVariable : LlvmIrVariable { /// @@ -162,9 +216,33 @@ class LlvmIrGlobalVariable : LlvmIrVariable /// public virtual LlvmIrVariableOptions? Options { get; set; } + /// + /// If set to `true`, initialize the array with a shortcut zero-initializer statement. Useful when pre-allocating + /// space for runtime use that won't be filled in with any data at the build time. + /// public bool ZeroInitializeArray { get; set; } + + /// + /// Specify number of items in an array. Used in cases when we want to pre-allocate an array without giving it any + /// value, thus making it impossible for the generator to discover the number of items automatically. This is useful + /// when using . This property is used **only** if the variable + /// is `null`. + /// public ulong ArrayItemCount { get; set; } + /// + /// If set, it will override any automatically calculated alignment for this variable + /// + public ulong? Alignment { get; set; } + + /// + /// If set, the provider will be called to obtain all the data to be placed in an array variable. The total amount + /// of data that will be returned by the provider **must** be specified in the property, + /// in order for the generator to properly declare the variable. The generator will verify that the amount of data + /// is exactly that much and throw an exception otherwise. + /// + public LlvmIrStreamedArrayDataProvider? ArrayDataProvider { get; set; } + /// /// Constructs a local variable. is translated to one of the LLVM IR first class types (see /// https://llvm.org/docs/LangRef.html#t-firstclass) only if it's an integral or floating point type. In all other cases it diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/MemberInfoUtilities.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/MemberInfoUtilities.cs index 431d92b6229..713b0c5a1a6 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/MemberInfoUtilities.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/MemberInfoUtilities.cs @@ -71,5 +71,15 @@ public static bool InlineArrayNeedsPadding (this MemberInfo mi) return attr.NeedsPadding; } + + public static LlvmIrVariableNumberFormat GetNumberFormat (this MemberInfo mi) + { + var attr = mi.GetCustomAttribute (); + if (attr == null) { + return LlvmIrVariableNumberFormat.Default; + } + + return attr.NumberFormat; + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/NativeAssemblerAttribute.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/NativeAssemblerAttribute.cs index 39fc5e094bb..c239255e1ac 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/NativeAssemblerAttribute.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/NativeAssemblerAttribute.cs @@ -33,6 +33,8 @@ class NativeAssemblerAttribute : Attribute /// size to which the member must be padded is specified by /// public bool NeedsPadding { get; set; } + + public LLVMIR.LlvmIrVariableNumberFormat NumberFormat { get; set; } = LLVMIR.LlvmIrVariableNumberFormat.Default; } [AttributeUsage (AttributeTargets.Class, Inherited = true)] diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs index 8a1b8996794..bbf926358dd 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs @@ -255,7 +255,7 @@ void ReorderActivityAliases (TaskLoggingHelper log, XElement app) } } - public IList Merge (TaskLoggingHelper log, TypeDefinitionCache cache, List subclasses, string applicationClass, bool embed, string bundledWearApplicationName, IEnumerable mergedManifestDocuments) + public IList Merge (TaskLoggingHelper log, TypeDefinitionCache cache, ICollection subclasses, string applicationClass, bool embed, string bundledWearApplicationName, IEnumerable mergedManifestDocuments) { var manifest = doc.Root; @@ -330,8 +330,7 @@ public IList Merge (TaskLoggingHelper log, TypeDefinitionCache cache, Li throw new InvalidOperationException (string.Format ("The targetSdkVersion ({0}) is not a valid API level", targetSdkVersion)); int targetSdkVersionValue = tryTargetSdkVersion.Value; - foreach (JavaType jt in subclasses) { - TypeDefinition t = jt.Type; + foreach (TypeDefinition t in subclasses) { if (t.IsAbstract) continue; @@ -568,7 +567,7 @@ Func GetGenerator (T return null; } - XElement CreateApplicationElement (XElement manifest, string applicationClass, List subclasses, TypeDefinitionCache cache) + XElement CreateApplicationElement (XElement manifest, string applicationClass, ICollection subclasses, TypeDefinitionCache cache) { var application = manifest.Descendants ("application").FirstOrDefault (); @@ -592,8 +591,7 @@ XElement CreateApplicationElement (XElement manifest, string applicationClass, L List typeAttr = new List (); List typeUsesLibraryAttr = new List (); List typeUsesConfigurationAttr = new List (); - foreach (JavaType jt in subclasses) { - TypeDefinition t = jt.Type; + foreach (TypeDefinition t in subclasses) { ApplicationAttribute aa = ApplicationAttribute.FromCustomAttributeProvider (t); if (aa == null) continue; @@ -925,7 +923,7 @@ void AddSupportsGLTextures (XElement application, TypeDefinitionCache cache) } } - void AddInstrumentations (XElement manifest, IList subclasses, int targetSdkVersion, TypeDefinitionCache cache) + void AddInstrumentations (XElement manifest, ICollection subclasses, int targetSdkVersion, TypeDefinitionCache cache) { var assemblyAttrs = Assemblies.SelectMany (path => InstrumentationAttribute.FromCustomAttributeProvider (Resolver.GetAssembly (path))); @@ -938,8 +936,7 @@ void AddInstrumentations (XElement manifest, IList subclasses, int tar manifest.Add (ia.ToElement (PackageName, cache)); } - foreach (JavaType jt in subclasses) { - TypeDefinition type = jt.Type; + foreach (TypeDefinition type in subclasses) { if (type.IsSubclassOf ("Android.App.Instrumentation", cache)) { var xe = InstrumentationFromTypeDefinition (type, JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.'), cache); if (xe != null) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs index 96063bf2126..cdb0f7a338a 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs @@ -26,7 +26,7 @@ sealed class AssemblyImports IDictionary assemblyPaths; TaskLoggingHelper log; - public MarshalMethodsAssemblyRewriter (IDictionary> methods, ICollection uniqueAssemblies, IDictionary assemblyPaths, TaskLoggingHelper log) + public MarshalMethodsAssemblyRewriter (IDictionary> methods, ICollection uniqueAssemblies, TaskLoggingHelper log) { this.assemblyPaths = assemblyPaths; this.methods = methods ?? throw new ArgumentNullException (nameof (methods)); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs index f87cf118971..da2f4a07b2a 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs @@ -922,6 +922,7 @@ void AddMarshalMethodNames (LlvmIrModule module, AssemblyCacheState acs) var mm_method_names_variable = new LlvmIrGlobalVariable (mm_method_names, "mm_method_names", LlvmIrVariableOptions.GlobalConstant) { BeforeWriteCallback = UpdateMarshalMethodNameIds, BeforeWriteCallbackCallerState = acs, + NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal, }; module.Add (mm_method_names_variable); @@ -1021,6 +1022,7 @@ void AddAssemblyImageCache (LlvmIrModule module, out AssemblyCacheState acs) BeforeWriteCallbackCallerState = acs, GetArrayItemCommentCallback = GetAssemblyImageCacheItemComment, GetArrayItemCommentCallbackCallerState = acs, + NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal, }; module.Add (assembly_image_cache_hashes); @@ -1066,7 +1068,7 @@ void UpdateAssemblyImageCacheHashes (LlvmIrVariable variable, LlvmIrModuleTarget i = acs.Hashes32[v32].index; } - return $" {index}: {name} => 0x{value:x} => {i}"; + return $" {index}: {name} => {i}"; } void UpdateAssemblyImageCacheIndices (LlvmIrVariable variable, LlvmIrModuleTarget target, object? callerState) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs index d72b347d5e9..d4e6a454111 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs @@ -543,6 +543,8 @@ public static AndroidTargetArch AbiToTargetArch (string abi) "arm64-v8a" => AndroidTargetArch.Arm64, "x86_64" => AndroidTargetArch.X86_64, "x86" => AndroidTargetArch.X86, + "" => throw new ArgumentException ("must not be empty string", nameof (abi)), + null => throw new ArgumentNullException (nameof (abi)), _ => throw new NotSupportedException ($"Internal error: unsupported ABI '{abi}'") }; } @@ -643,5 +645,41 @@ public static string GetNativeLibsRootDirectoryPath (string androidBinUtilsDirec string relPath = GetToolsRootDirectoryRelativePath (androidBinUtilsDirectory); return Path.GetFullPath (Path.Combine (androidBinUtilsDirectory, relPath, "lib")); } + + public static string MakeNativeAssemblyFileName (string baseName, string abi) => $"{baseName}.{abi}.ll"; + + public static bool IsSatelliteAssembly (ITaskItem item) + { + return IsNotEmptyMetadataAndMaybeMatches (item, "AssetType", "resources") && IsNotEmptyMetadataAndMaybeMatches (item, "Culture"); + + bool IsNotEmptyMetadataAndMaybeMatches (ITaskItem item, string metadataName, string? expectedValue = null) + { + string? data = item.GetMetadata (metadataName); + if (String.IsNullOrEmpty (data)) { + return false; + } + + if (String.IsNullOrEmpty (expectedValue)) { + return true; + } + + return String.Compare (data, expectedValue, StringComparison.Ordinal) == 0; + } + } + + public static HashSet MakeHashSet (ICollection items) + { + var ret = new HashSet (StringComparer.OrdinalIgnoreCase); + + if (items.Count == 0) { + return ret; + } + + foreach (ITaskItem item in items) { + ret.Add (item.ItemSpec); + } + + return ret; + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeCompilationHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeCompilationHelper.cs new file mode 100644 index 00000000000..4825718c507 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeCompilationHelper.cs @@ -0,0 +1,292 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Text; +using System.Threading; + +using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +class NativeCompilationHelper +{ + public abstract class Config + { + public readonly string ExecutablePath; + public readonly List ProcessOptions; + public readonly TaskLoggingHelper Log; + public readonly string OutputFile; + public readonly string? WorkingDirectory; + public readonly AndroidTargetArch TargetArch; + + public object? State { get; set; } + public CancellationToken? CancellationToken { get; set; } + public Action? Cancel { get; set; } + + protected Config (TaskLoggingHelper log, AndroidTargetArch targetArch, string executablePath, string outputFile, List processOptions, string? workingDirectory) + { + Log = log; + TargetArch = targetArch; + ExecutablePath = EnsureValidString (executablePath, nameof (executablePath)); + OutputFile = EnsureValidString (outputFile, nameof (outputFile)); + ProcessOptions = new List (processOptions); + WorkingDirectory = workingDirectory; + } + + string EnsureValidString (string v, string name) + { + if (String.IsNullOrEmpty (v)) { + throw new ArgumentException ("must not be null or empty", name); + } + + return v; + } + } + + public sealed class LinkerConfig : Config + { + public readonly List ObjectFilePaths; + public readonly List ExtraLibraries; + + public LinkerConfig (TaskLoggingHelper log, AndroidTargetArch targetArch, string linkerPath, string outputSharedLibrary, List? linkerOptions = null, string? workingDirectory = null) + : base ( + log, + targetArch, + linkerPath, + outputSharedLibrary, + linkerOptions != null ? linkerOptions : NativeCompilationHelper.CommonLinkerArgs, + workingDirectory + ) + { + AddArchOptions (ProcessOptions, targetArch); + ProcessOptions.Add ($"-soname {Path.GetFileName(OutputFile)}"); + ProcessOptions.Add ($"-o {QuoteFileName(OutputFile)}"); + + ObjectFilePaths = new List (); + ExtraLibraries = new List (); + } + + void AddArchOptions (List options, AndroidTargetArch arch) + { + string elfArch; + switch (arch) { + case AndroidTargetArch.Arm: + options.Add ("-X"); + elfArch = "armelf_linux_eabi"; + break; + + case AndroidTargetArch.Arm64: + options.Add ("--fix-cortex-a53-843419"); + elfArch = "aarch64linux"; + break; + + case AndroidTargetArch.X86: + elfArch = "elf_i386"; + break; + + case AndroidTargetArch.X86_64: + elfArch = "elf_x86_64"; + break; + + default: + throw new NotSupportedException ($"Unsupported Android target architecture '{arch}'"); + } + + options.Add ("-m"); + options.Add (elfArch); + } + } + + public sealed class AssemblerConfig : Config + { + public readonly string InputSource; + + public AssemblerConfig (TaskLoggingHelper log, AndroidTargetArch targetArch, string assemblerPath, string inputSource, string workingDirectory, List? assemblerOptions = null, string? outputFile = null) + : base ( + log, + targetArch, + assemblerPath, + String.IsNullOrEmpty (outputFile) ? Path.ChangeExtension (inputSource, ".o") : outputFile, + assemblerOptions != null ? assemblerOptions : NativeCompilationHelper.DefaultAssemblerOptions, + workingDirectory + ) + { + InputSource = inputSource; + + ProcessOptions.Add ($"-o={QuoteFileName(OutputFile)}"); + ProcessOptions.Add (QuoteFileName (InputSource)); + } + } + + public static readonly List DefaultAssemblerOptions = new List { + "-O2", + "--debugger-tune=lldb", + "--debugify-level=location+variables", + "--fatal-warnings", + "--filetype=obj", + "--relocation-model=pic", + }; + + public static readonly List CommonLinkerArgs = new List { + "--shared", + "--allow-shlib-undefined", + "--export-dynamic", + "-z relro", + "-z noexecstack", + "--enable-new-dtags", + "--build-id", + "--warn-shared-textrel", + "--fatal-warnings", + }; + + public static string GetAssemblerPath (string androidBinUtilsDirectory) + { + string llcPath = Path.Combine (androidBinUtilsDirectory, "llc"); + string executableDir = Path.GetDirectoryName (llcPath); + string executableName = MonoAndroidHelper.GetExecutablePath (executableDir, Path.GetFileName (llcPath)); + + return Path.Combine (executableDir, executableName); + } + + public static string GetLinkerPath (string androidBinUtilsDirectory) + { + return Path.Combine (androidBinUtilsDirectory, MonoAndroidHelper.GetExecutablePath (androidBinUtilsDirectory, "ld")); + } + + static string QuoteFileName (string fileName) + { + var builder = new CommandLineBuilder (); + builder.AppendFileNameIfNotNull (fileName); + return builder.ToString (); + } + + static void CreateOutputDirectory (string filePath) + { + string? dirPath = Path.GetDirectoryName (filePath); + if (!String.IsNullOrEmpty (dirPath)) { + Directory.CreateDirectory (dirPath); + } + } + + public static bool RunLinker (LinkerConfig config) + { + if (config.ObjectFilePaths.Count > 0) { + foreach (string objFile in config.ObjectFilePaths) { + config.ProcessOptions.Add (QuoteFileName (objFile)); + } + } + + if (config.ExtraLibraries.Count > 0) { + foreach (string libName in config.ObjectFilePaths) { + config.ProcessOptions.Add ($"-l{libName}"); + } + } + + ProcessStartInfo psi = CreateProcessStartInfo (config); + return RunProcess ( + config, + psi, + "LLVM ld", + (StringBuilder sb) => config.Log.LogCodedError ("XA3007", Properties.Resources.XA3007, Path.GetFileName (config.OutputFile), sb.ToString ()) + ); + } + + public static bool RunAssembler (AssemblerConfig config) + { + ProcessStartInfo psi = CreateProcessStartInfo (config); + return RunProcess ( + config, + psi, + "LLVM llc", + (StringBuilder sb) => config.Log.LogCodedError ("XA3006", Properties.Resources.XA3006, Path.GetFileName (config.InputSource), sb.ToString ()) + ); + } + + static bool RunProcess (Config config, ProcessStartInfo psi, string processDisplayName, Action onError) + { + CreateOutputDirectory (config.OutputFile); + + using var stdout_completed = new ManualResetEvent (false); + using var stderr_completed = new ManualResetEvent (false); + + string assemblerName = Path.GetFileName (config.ExecutablePath); + config.Log.LogDebugMessage ($"[{processDisplayName}] {psi.FileName} {psi.Arguments}"); + + var stdoutLines = new List (); + var stderrLines = new List (); + + using var proc = new Process (); + proc.OutputDataReceived += (s, e) => { + if (e.Data != null) { + OnOutputData (config.Log, assemblerName, s, e); + stdoutLines.Add (e.Data); + } else + stdout_completed.Set (); + }; + + proc.ErrorDataReceived += (s, e) => { + if (e.Data != null) { + OnErrorData (config.Log, assemblerName, s, e); + stderrLines.Add (e.Data); + } else + stderr_completed.Set (); + }; + + proc.StartInfo = psi; + proc.Start (); + proc.BeginOutputReadLine (); + proc.BeginErrorReadLine (); + config.CancellationToken?.Register (() => { try { proc.Kill (); } catch (Exception) { } }); + proc.WaitForExit (); + + if (psi.RedirectStandardError) + stderr_completed.WaitOne (TimeSpan.FromSeconds (30)); + + if (psi.RedirectStandardOutput) + stdout_completed.WaitOne (TimeSpan.FromSeconds (30)); + + if (proc.ExitCode != 0) { + var sb = MonoAndroidHelper.MergeStdoutAndStderrMessages (stdoutLines, stderrLines); + onError (sb); + if (config.Cancel != null) { + config.Cancel (); + } + return false; + } + + return true; + } + + static ProcessStartInfo CreateProcessStartInfo (Config config) + { + var psi = new ProcessStartInfo { + FileName = config.ExecutablePath, + Arguments = String.Join (" ", config.ProcessOptions), + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + WindowStyle = ProcessWindowStyle.Hidden, + }; + + if (!String.IsNullOrEmpty (config.WorkingDirectory)) { + psi.WorkingDirectory = config.WorkingDirectory; + } + + return psi; + } + + static void OnOutputData (TaskLoggingHelper log, string assemblerName, object sender, DataReceivedEventArgs e) + { + log.LogDebugMessage ($"[{assemblerName} stdout] {e.Data}"); + } + + static void OnErrorData (TaskLoggingHelper log, string assemblerName, object sender, DataReceivedEventArgs e) + { + log.LogMessage ($"[{assemblerName} stderr] {e.Data}", MessageImportance.High); + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/RidAgnosticInputAssemblySet.cs b/src/Xamarin.Android.Build.Tasks/Utilities/RidAgnosticInputAssemblySet.cs new file mode 100644 index 00000000000..511fe24164e --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/RidAgnosticInputAssemblySet.cs @@ -0,0 +1,36 @@ +using System; +using System.Collections.Generic; + +using Microsoft.Build.Framework; + +namespace Xamarin.Android.Tasks; + +class RidAgnosticInputAssemblySet : InputAssemblySet +{ + Dictionary javaTypeAssemblies = new (AssemblyNameStringComparer); + Dictionary userAssemblies = new (AssemblyNameStringComparer); + + public ICollection JavaTypeAssemblies => javaTypeAssemblies.Values; + public ICollection UserAssemblies => userAssemblies.Values; + + public override void AddJavaTypeAssembly (ITaskItem assemblyItem) + { + Add (assemblyItem.ItemSpec, assemblyItem, javaTypeAssemblies); + } + + public override void AddUserAssembly (ITaskItem assemblyItem) + { + Add (GetUserAssemblyKey (assemblyItem), assemblyItem, userAssemblies); + } + + public override bool IsUserAssembly (string name) => userAssemblies.ContainsKey (name); + + void Add (string key, ITaskItem assemblyItem, Dictionary dict) + { + if (dict.ContainsKey (key)) { + return; + } + + dict.Add (key, assemblyItem); + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/RidAwareInputAssemblySet.cs b/src/Xamarin.Android.Build.Tasks/Utilities/RidAwareInputAssemblySet.cs new file mode 100644 index 00000000000..f77cec38968 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/RidAwareInputAssemblySet.cs @@ -0,0 +1,34 @@ +using System.Collections.Generic; + +using Microsoft.Build.Framework; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +/// +/// Assembly input set where **some** assemblies are expected to be RID-specific but **all** of +/// them need to be placed in RID-specific sets. Any assembly that is not RID-specific will be +/// copied to all the RID-specific sets. This is to be used in `Release` builds when linking is +/// **disabled**, as these builds use assembly MVIDs and type+method metadata token IDs. +/// +class RidAwareInputAssemblySet : RidSensitiveInputAssemblySet +{ + List targetArches; + + public RidAwareInputAssemblySet (ICollection targetArches) + { + this.targetArches = new List (targetArches); + } + + protected override void Add (AndroidTargetArch arch, string key, ITaskItem assemblyItem, Dictionary> assemblies) + { + if (arch != AndroidTargetArch.None) { + base.Add (arch, key, assemblyItem, assemblies); + return; + } + + foreach (AndroidTargetArch targetArch in targetArches) { + base.Add (targetArch, key, assemblyItem, assemblies); + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/RidSensitiveInputAssemblySet.cs b/src/Xamarin.Android.Build.Tasks/Utilities/RidSensitiveInputAssemblySet.cs new file mode 100644 index 00000000000..6adce1df85e --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/RidSensitiveInputAssemblySet.cs @@ -0,0 +1,62 @@ +using System.Collections.Generic; + +using Microsoft.Build.Framework; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +abstract class RidSensitiveInputAssemblySet : InputAssemblySet +{ + // Both dictionaries must have the same keys present, even if the associated value is an empty dictionary + Dictionary> javaTypeAssemblies = new (); + Dictionary> userAssemblies = new (); + + public Dictionary> JavaTypeAssemblies => javaTypeAssemblies; + public Dictionary> UserAssemblies => userAssemblies; + + public override void AddJavaTypeAssembly (ITaskItem assemblyItem) + { + AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (assemblyItem); + Add (arch, assemblyItem.ItemSpec, assemblyItem, JavaTypeAssemblies); + EnsureArchKey (arch, userAssemblies); + } + + public override void AddUserAssembly (ITaskItem assemblyItem) + { + AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (assemblyItem); + Add (arch, GetUserAssemblyKey (assemblyItem), assemblyItem, UserAssemblies); + EnsureArchKey (arch, javaTypeAssemblies); + } + + public override bool IsUserAssembly (string name) + { + foreach (var kvp in userAssemblies) { + if (kvp.Value.ContainsKey (name)) { + return true; + } + } + + return false; + } + + protected virtual void Add (AndroidTargetArch targetArch, string key, ITaskItem assemblyItem, Dictionary> assemblies) + { + Dictionary dict = EnsureArchKey (targetArch, assemblies); + if (dict.ContainsKey (key)) { + return; + } + + dict.Add (key, assemblyItem); + } + + Dictionary EnsureArchKey (AndroidTargetArch targetArch, Dictionary> assemblies) + { + if (assemblies.TryGetValue (targetArch, out Dictionary dict)) { + return dict; + } + + dict = new (AssemblyNameStringComparer); + assemblies.Add (targetArch, dict); + return dict; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/RidSpecificInputAssemblySet.cs b/src/Xamarin.Android.Build.Tasks/Utilities/RidSpecificInputAssemblySet.cs new file mode 100644 index 00000000000..e5312fb2440 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/RidSpecificInputAssemblySet.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; + +using Microsoft.Build.Framework; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +/// +/// Assembly input set where **all** assemblies are expected to be RID-specific, any assembly which is +/// not will cause an exception. This is meant to be used whenever linking is enabled. +/// +class RidSpecificInputAssemblySet : RidSensitiveInputAssemblySet +{ + protected override void Add (AndroidTargetArch arch, string key, ITaskItem assemblyItem, Dictionary> assemblies) + { + if (arch == AndroidTargetArch.None) { + throw new InvalidOperationException ($"`Abi` metadata item is required for assembly '{assemblyItem.ItemSpec}'"); + } + + base.Add (arch, key, assemblyItem, assemblies); + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs index 3730176e751..a1081b3f35e 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs @@ -8,6 +8,7 @@ using Java.Interop.Tools.Cecil; using Mono.Cecil; using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Utilities; using Xamarin.Android.Tools; namespace Xamarin.Android.Tasks @@ -96,27 +97,13 @@ sealed class ReleaseGenerationState public readonly Dictionary KnownAssemblies; public readonly Dictionary MvidCache; - public readonly IDictionary> TempModules; - - // Just a convenient way to access one of the temp modules dictionaries, to be used when dealing with ABI-agnostic - // types in ProcessReleaseType. - public readonly Dictionary TempModulesAbiAgnostic; + public readonly Dictionary TempModules; public ReleaseGenerationState (string[] supportedAbis) { KnownAssemblies = new Dictionary (StringComparer.Ordinal); MvidCache = new Dictionary (); - - var tempModules = new Dictionary> (); - foreach (string abi in supportedAbis) { - var dict = new Dictionary (); - if (TempModulesAbiAgnostic == null) { - TempModulesAbiAgnostic = dict; - } - tempModules.Add (MonoAndroidHelper.AbiToTargetArch (abi), dict); - } - - TempModules = new ReadOnlyDictionary> (tempModules); + TempModules = new Dictionary (); } public void AddKnownAssembly (TypeDefinition td) @@ -133,7 +120,8 @@ public void AddKnownAssembly (TypeDefinition td) public string GetAssemblyName (TypeDefinition td) => td.Module.Assembly.FullName; } - Action logger; + AndroidTargetArch targetArch; + readonly TaskLoggingHelper Log; Encoding outputEncoding; byte[] moduleMagicString; byte[] typemapIndexMagicString; @@ -141,11 +129,10 @@ public void AddKnownAssembly (TypeDefinition td) public IList GeneratedBinaryTypeMaps { get; } = new List (); - public TypeMapGenerator (Action logger, string[] supportedAbis) + public TypeMapGenerator (AndroidTargetArch targetArch, TaskLoggingHelper log, string[] supportedAbis) { - this.logger = logger ?? throw new ArgumentNullException (nameof (logger)); - if (supportedAbis == null) - throw new ArgumentNullException (nameof (supportedAbis)); + this.targetArch = targetArch; + Log = log; this.supportedAbis = supportedAbis; outputEncoding = Files.UTF8withoutBOM; @@ -168,7 +155,7 @@ void UpdateApplicationConfig (TypeDefinition javaType, ApplicationConfigTaskStat } } - public bool Generate (bool debugBuild, bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, bool generateNativeAssembly, out ApplicationConfigTaskState appConfState) + public bool Generate (bool debugBuild, bool skipJniAddNativeMethodRegistrationAttributeScan, ICollection javaTypes, TypeDefinitionCache cache, string outputDirectory, bool generateNativeAssembly, out ApplicationConfigTaskState appConfState) { if (String.IsNullOrEmpty (outputDirectory)) throw new ArgumentException ("must not be null or empty", nameof (outputDirectory)); @@ -189,7 +176,7 @@ public bool Generate (bool debugBuild, bool skipJniAddNativeMethodRegistrationAt return GenerateRelease (skipJniAddNativeMethodRegistrationAttributeScan, javaTypes, cache, typemapsOutputDirectory, appConfState); } - bool GenerateDebug (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, bool generateNativeAssembly, ApplicationConfigTaskState appConfState) + bool GenerateDebug (bool skipJniAddNativeMethodRegistrationAttributeScan, ICollection javaTypes, TypeDefinitionCache cache, string outputDirectory, bool generateNativeAssembly, ApplicationConfigTaskState appConfState) { if (generateNativeAssembly) { return GenerateDebugNativeAssembly (skipJniAddNativeMethodRegistrationAttributeScan, javaTypes, cache, outputDirectory, appConfState); @@ -197,15 +184,14 @@ bool GenerateDebug (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, ApplicationConfigTaskState appConfState) + bool GenerateDebugFiles (bool skipJniAddNativeMethodRegistrationAttributeScan, ICollection javaTypes, TypeDefinitionCache cache, string outputDirectory, ApplicationConfigTaskState appConfState) { var modules = new Dictionary (StringComparer.Ordinal); int maxModuleFileNameWidth = 0; int maxModuleNameWidth = 0; var javaDuplicates = new Dictionary> (StringComparer.Ordinal); - foreach (JavaType jt in javaTypes) { - TypeDefinition td = jt.Type; + foreach (TypeDefinition td in javaTypes) { UpdateApplicationConfig (td, appConfState); string moduleName = td.Module.Assembly.Name.Name; ModuleDebugData module; @@ -263,14 +249,13 @@ bool GenerateDebugFiles (bool skipJniAddNativeMethodRegistrationAttributeScan, L return true; } - bool GenerateDebugNativeAssembly (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, ApplicationConfigTaskState appConfState) + bool GenerateDebugNativeAssembly (bool skipJniAddNativeMethodRegistrationAttributeScan, ICollection javaTypes, TypeDefinitionCache cache, string outputDirectory, ApplicationConfigTaskState appConfState) { var javaToManaged = new List (); var managedToJava = new List (); var javaDuplicates = new Dictionary> (StringComparer.Ordinal); - foreach (JavaType jt in javaTypes) { - TypeDefinition td = jt.Type; + foreach (TypeDefinition td in javaTypes) { UpdateApplicationConfig (td, appConfState); TypeMapDebugEntry entry = GetDebugEntry (td, cache); @@ -375,7 +360,7 @@ string GetManagedTypeName (TypeDefinition td) return $"{managedTypeName}, {td.Module.Assembly.Name.Name}"; } - void ProcessReleaseType (ReleaseGenerationState state, TypeDefinition td, AndroidTargetArch typeArch, ApplicationConfigTaskState appConfState, TypeDefinitionCache cache) + void ProcessReleaseType (ReleaseGenerationState state, TypeDefinition td, ApplicationConfigTaskState appConfState, TypeDefinitionCache cache) { UpdateApplicationConfig (td, appConfState); @@ -390,21 +375,7 @@ void ProcessReleaseType (ReleaseGenerationState state, TypeDefinition td, Androi state.MvidCache.Add (td.Module.Mvid, moduleUUID); } - bool abiAgnosticType = typeArch == AndroidTargetArch.None; - Dictionary tempModules; - if (abiAgnosticType) { - tempModules = state.TempModulesAbiAgnostic; - } else { - // It will throw if `typeArch` isn't in the dictionary. This is intentional, since we must have no TypeDefinition entries for architectures not - // mentioned in `supportedAbis`. - try { - tempModules = state.TempModules[typeArch]; - } catch (KeyNotFoundException ex) { - throw new InvalidOperationException ($"Internal error: cannot process type specific to architecture '{typeArch}', since that architecture isn't mentioned in the set of supported ABIs", ex); - } - } - - if (!tempModules.TryGetValue (moduleUUID, out ModuleReleaseData moduleData)) { + if (!state.TempModules.TryGetValue (moduleUUID, out ModuleReleaseData moduleData)) { moduleData = new ModuleReleaseData { Mvid = td.Module.Mvid, MvidBytes = moduleUUID, @@ -414,15 +385,8 @@ void ProcessReleaseType (ReleaseGenerationState state, TypeDefinition td, Androi DuplicateTypes = new List (), }; - if (abiAgnosticType) { - // ABI-agnostic types must be added to all the ABIs - foreach (var kvp in state.TempModules) { - kvp.Value.Add (moduleUUID, moduleData); - } - } else { - // ABI-specific types are added only to their respective tempModules - tempModules.Add (moduleUUID, moduleData); - } + // ABI-specific types are added only to their respective tempModules + state.TempModules.Add (moduleUUID, moduleData); } string javaName = Java.Interop.Tools.TypeNameMappings.JavaNativeTypeManager.ToJniName (td, cache); @@ -450,42 +414,31 @@ void ProcessReleaseType (ReleaseGenerationState state, TypeDefinition td, Androi } } - bool GenerateRelease (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, ApplicationConfigTaskState appConfState) + bool GenerateRelease (bool skipJniAddNativeMethodRegistrationAttributeScan, ICollection javaTypes, TypeDefinitionCache cache, string outputDirectory, ApplicationConfigTaskState appConfState) { var state = new ReleaseGenerationState (supportedAbis); - foreach (JavaType jt in javaTypes) { - if (!jt.IsABiSpecific) { - ProcessReleaseType (state, jt.Type, AndroidTargetArch.None, appConfState, cache); - continue; - } - - foreach (var kvp in jt.PerAbiTypes) { - ProcessReleaseType (state, kvp.Value, kvp.Key, appConfState, cache); - } + foreach (TypeDefinition td in javaTypes) { + ProcessReleaseType (state, td, appConfState, cache); } - foreach (var kvp in state.TempModules) { - AndroidTargetArch arch = kvp.Key; - Dictionary tempModules = kvp.Value; - ModuleReleaseData[] modules = tempModules.Values.ToArray (); - Array.Sort (modules, new ModuleUUIDArrayComparer ()); + ModuleReleaseData[] modules = state.TempModules.Values.ToArray (); + Array.Sort (modules, new ModuleUUIDArrayComparer ()); - foreach (ModuleReleaseData module in modules) { - if (module.TypesScratch.Count == 0) { - module.Types = Array.Empty (); - continue; - } - - // No need to sort here, the LLVM IR generator will compute hashes and sort - // the array on write. - module.Types = module.TypesScratch.Values.ToArray (); + foreach (ModuleReleaseData module in modules) { + if (module.TypesScratch.Count == 0) { + module.Types = Array.Empty (); + continue; } - var composer = new TypeMappingReleaseNativeAssemblyGenerator (new NativeTypeMappingData (modules)); - GenerateNativeAssembly (arch, composer, composer.Construct (), outputDirectory); + // No need to sort here, the LLVM IR generator will compute hashes and sort + // the array on write. + module.Types = module.TypesScratch.Values.ToArray (); } + var composer = new TypeMappingReleaseNativeAssemblyGenerator (new NativeTypeMappingData (modules)); + GenerateNativeAssembly (composer, composer.Construct (), outputDirectory); + return true; } @@ -494,30 +447,35 @@ bool ShouldSkipInJavaToManaged (TypeDefinition td) return td.IsInterface || td.HasGenericParameters; } - string GetOutputFilePath (string baseFileName, string abi) => $"{baseFileName}.{abi}.ll"; - - void GenerateNativeAssembly (AndroidTargetArch arch, LLVMIR.LlvmIrComposer composer, LLVMIR.LlvmIrModule typeMapModule, string baseFileName) - { - WriteNativeAssembly ( - arch, - composer, - typeMapModule, - GetOutputFilePath (baseFileName, ArchToAbi (arch)) - ); - } - void GenerateNativeAssembly (LLVMIR.LlvmIrComposer composer, LLVMIR.LlvmIrModule typeMapModule, string baseFileName) { + if (targetArch != AndroidTargetArch.None) { + DoWriteNativeAssembly (targetArch, MonoAndroidHelper.MakeNativeAssemblyFileName (baseFileName, ArchToAbi (targetArch))); + return; + } + foreach (string abi in supportedAbis) { - WriteNativeAssembly ( - GeneratePackageManagerJava.GetAndroidTargetArchForAbi (abi), - composer, - typeMapModule, - GetOutputFilePath (baseFileName, abi) - ); + DoWriteNativeAssembly (MonoAndroidHelper.AbiToTargetArch (abi), MonoAndroidHelper.MakeNativeAssemblyFileName (baseFileName, abi)); + } + + void DoWriteNativeAssembly (AndroidTargetArch arch, string outputFile) + { + WriteNativeAssembly (arch, composer, typeMapModule, outputFile); } } + // void GenerateNativeAssembly (LLVMIR.LlvmIrComposer composer, LLVMIR.LlvmIrModule typeMapModule, string baseFileName) + // { + // foreach (string abi in supportedAbis) { + // WriteNativeAssembly ( + // GeneratePackageManagerJava.GetAndroidTargetArchForAbi (abi), + // composer, + // typeMapModule, + // GetOutputFilePath (baseFileName, abi) + // ); + // } + // } + void WriteNativeAssembly (AndroidTargetArch arch, LLVMIR.LlvmIrComposer composer, LLVMIR.LlvmIrModule typeMapModule, string outputFile) { using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) { diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs index a6e4fd465df..84c0033e8a6 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs @@ -69,6 +69,7 @@ sealed class TypeMapModuleEntry [NativeAssembler (Ignore = true)] public TypeMapJava JavaTypeMapEntry; + [NativeAssembler (NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal)] public uint type_token_id; public uint java_map_index; } @@ -126,6 +127,8 @@ sealed class TypeMapJava public ulong JavaNameHash64; public uint module_index; + + [NativeAssembler (NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal)] public uint type_token_id; public uint java_name_index; } @@ -212,6 +215,7 @@ protected override void Construct (LlvmIrModule module) BeforeWriteCallbackCallerState = cs, GetArrayItemCommentCallback = GetJavaHashesItemComment, GetArrayItemCommentCallbackCallerState = cs, + NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal, }; map_java_hashes.WriteOptions &= ~LlvmIrVariableWriteOptions.ArrayWriteIndexComments; module.Add (map_java_hashes); @@ -258,7 +262,7 @@ uint GetJavaEntryIndex (TypeMapJava javaEntry) throw new InvalidOperationException ("Internal error: construction state expected but not found"); } - return $" {index}: 0x{value:x} => {cs.JavaMap[(int)index].Instance.JavaName}"; + return $" {index}: {cs.JavaMap[(int)index].Instance.JavaName}"; } void GenerateAndSortJavaHashes (LlvmIrVariable variable, LlvmIrModuleTarget target, object? callerState) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs index 5f965ab46ff..c6eb367f638 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs @@ -28,20 +28,6 @@ public JavaType (TypeDefinition type, IDictionary PerAbi; - - public bool IsAbiSpecific => !PerAbi.ContainsKey (AndroidTargetArch.None); - - public TypeData (TypeDefinition firstType) - { - FirstType = firstType; - PerAbi = new Dictionary (); - } - } - public bool ErrorOnCustomJavaObject { get; set; } TaskLoggingHelper log; @@ -53,51 +39,33 @@ public XAJavaTypeScanner (TaskLoggingHelper log, TypeDefinitionCache cache) this.cache = cache; } - public List GetJavaTypes (ICollection inputAssemblies, XAAssemblyResolver resolver) + public ICollection GetJavaTypes (ICollection inputAssemblies, XAAssemblyResolver resolver) { - var types = new Dictionary (StringComparer.Ordinal); + var types = new Dictionary (StringComparer.Ordinal); foreach (ITaskItem asmItem in inputAssemblies) { AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (asmItem); AssemblyDefinition asmdef = resolver.Load (arch, asmItem.ItemSpec); foreach (ModuleDefinition md in asmdef.Modules) { foreach (TypeDefinition td in md.Types) { - AddJavaType (td, types, arch); + AddJavaType (td, types); } } } - var ret = new List (); - foreach (var kvp in types) { - ret.Add (new JavaType (kvp.Value.FirstType, kvp.Value.IsAbiSpecific ? kvp.Value.PerAbi : null)); - } - - return ret; + return types.Values; } - void AddJavaType (TypeDefinition type, Dictionary types, AndroidTargetArch arch) + void AddJavaType (TypeDefinition type, Dictionary types) { if (type.IsSubclassOf ("Java.Lang.Object", cache) || type.IsSubclassOf ("Java.Lang.Throwable", cache) || (type.IsInterface && type.ImplementsInterface ("Java.Interop.IJavaPeerable", cache))) { // For subclasses of e.g. Android.App.Activity. string typeName = type.GetPartialAssemblyQualifiedName (cache); - if (!types.TryGetValue (typeName, out TypeData typeData)) { - typeData = new TypeData (type); - types.Add (typeName, typeData); - } - - if (typeData.PerAbi.ContainsKey (AndroidTargetArch.None)) { - if (arch == AndroidTargetArch.None) { - throw new InvalidOperationException ($"Duplicate type '{type.FullName}' in assembly {type.Module.FileName}"); - } - - throw new InvalidOperationException ($"Previously added type '{type.FullName}' was in ABI-agnostic assembly, new one comes from ABI {arch} assembly"); - } - - if (typeData.PerAbi.ContainsKey (arch)) { - throw new InvalidOperationException ($"Duplicate type '{type.FullName}' in assembly {type.Module.FileName}, for ABI {arch}"); + if (types.ContainsKey (typeName)) { + return; } - typeData.PerAbi.Add (arch, type); + types.Add (typeName, type); } else if (type.IsClass && !type.IsSubclassOf ("System.Exception", cache) && type.ImplementsInterface ("Android.Runtime.IJavaObject", cache)) { string message = $"XA4212: Type `{type.FullName}` implements `Android.Runtime.IJavaObject` but does not inherit `Java.Lang.Object` or `Java.Lang.Throwable`. This is not supported."; @@ -114,7 +82,7 @@ void AddJavaType (TypeDefinition type, Dictionary types, Andro } foreach (TypeDefinition nested in type.NestedTypes) { - AddJavaType (nested, types, arch); + AddJavaType (nested, types); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.targets index bb45a722e04..b61ceccbe8b 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.targets @@ -271,29 +271,29 @@ BeforeTargets="CopyFilesToOutputDirectory" Inputs="$(MSBuildAllProjects);@(IntermediateAssembly);@(InputAssemblies)" Outputs="$(IntermediateOutputPath)ILRepacker.stamp" > - - <_InputAssembliesThatExist Include="@(InputAssemblies)" Condition="Exists('%(Identity)')" /> - <_NetstandardPath Include="@(ReferencePath->'%(RootDir)%(Directory)')" Condition="'%(FileName)%(Extension)' == 'netstandard.dll'" /> - - - <_NetstandardDir>@(_NetstandardPath) - <_ILRepackArgs>/out:"$(MSBuildThisFileDirectory)$(IntermediateOutputPath)$(AssemblyName).dll" /internalize - <_ILRepackArgs>$(_ILRepackArgs) /keyfile:"$(XamarinAndroidSourcePath)product.snk" - <_ILRepackArgs>$(_ILRepackArgs) "$(MSBuildThisFileDirectory)$(IntermediateOutputPath)$(AssemblyName).dll" - <_ILRepackArgs>$(_ILRepackArgs) @(_InputAssembliesThatExist->'"%(Identity)"', ' ') - <_ILRepackArgs>$(_ILRepackArgs) /lib:"$(_NetstandardDir.TrimEnd('\'))" - - - - - - + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 330b30f9a7e..6c8c21e8a50 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -87,13 +87,9 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. - - - - - + - - - - - - - - - <_GenerateJavaStubsDependsOnTargets> - _SetLatestTargetFrameworkVersion; - _PrepareAssemblies; - _PrepareNativeAssemblySources; - $(_AfterPrepareAssemblies); - - - .so;$(AndroidStoreUncompressedFileExtensions) .dex;$(AndroidStoreUncompressedFileExtensions) - .blob;$(AndroidStoreUncompressedFileExtensions) - - - - - - - - - - - - <_GeneratedAndroidEnvironment Include="__XA_PACKAGE_NAMING_POLICY__=$(AndroidPackageNamingPolicy)" /> @@ -1658,49 +1602,6 @@ because xbuild doesn't support framework reference assemblies. - - - - - - - - - - - - <_GenerateAndroidRemapNativeCodeDependsOn> - _ConvertAndroidMamMappingFileToXml; - _CollectAndroidRemapMembers; - _PrepareAndroidRemapNativeAssemblySources - - - - - - - @@ -1712,24 +1613,6 @@ because xbuild doesn't support framework reference assemblies. - - <_GeneratePackageManagerJavaDependsOn> - _GenerateJavaStubs; - _RunAotForAllRIDs; - _ManifestMerger; - _ConvertCustomView; - $(_AfterConvertCustomView); - _AddStaticResources; - $(_AfterAddStaticResources); - _PrepareAssemblies; - _PrepareEnvironmentAssemblySources; - _GenerateEnvironmentFiles; - _GenerateAndroidRemapNativeCode; - _GenerateEmptyAndroidRemapNativeCode; - _IncludeNativeSystemLibraries; - - - + UseAssemblySharedLibraries="$(_AndroidUseAssemblySharedLibraries)"> @@ -1990,81 +1872,6 @@ because xbuild doesn't support framework reference assemblies. - - - <_NativeAssemblyTarget Include="@(_TypeMapAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> - %(_TypeMapAssemblySource.abi) - - <_NativeAssemblyTarget Include="@(_EnvironmentAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> - %(_EnvironmentAssemblySource.abi) - - <_NativeAssemblyTarget Include="@(_CompressedAssembliesAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> - %(_CompressedAssembliesAssemblySource.abi) - - <_CompressedNativeAssemblyTarget Include="@(_CompressedAssembliesAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> - %(_CompressedAssembliesAssemblySource.abi) - - <_NativeAssemblyTarget Include="@(_MarshalMethodsAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> - %(_MarshalMethodsAssemblySource.abi) - - <_NativeAssemblyTarget Include="@(_AndroidRemapAssemblySource->'$([System.IO.Path]::ChangeExtension('%(Identity)', '.o'))')"> - %(_AndroidRemapAssemblySource.abi) - - - - - - - - - - - - - - - - - - - - - <_ApplicationSharedLibrary Include="$(_AndroidApplicationSharedLibraryPath)%(_BuildTargetAbis.Identity)\libxamarin-app.so"> - %(_BuildTargetAbis.Identity) - - - - - - - - - - - @@ -2115,7 +1922,7 @@ because xbuild doesn't support framework reference assemblies. EmbedAssemblies="$(EmbedAssembliesIntoApk)" ResolvedUserAssemblies="@(_ShrunkUserAssemblies);@(_AndroidResolvedSatellitePaths)" ResolvedFrameworkAssemblies="@(_ShrunkFrameworkAssemblies)" - FrameworkNativeLibraries="@(FrameworkNativeLibrary)" + FrameworkNativeLibraries="@(FrameworkNativeLibrary);@(_StandaloneAssemblyDSO)" NativeLibraries="@(AndroidNativeLibrary)" ApplicationSharedLibraries="@(_ApplicationSharedLibrary)" AdditionalNativeLibraryReferences="@(_AdditionalNativeLibraryReferences)" @@ -2138,7 +1945,7 @@ because xbuild doesn't support framework reference assemblies. ExcludeFiles="@(AndroidPackagingOptionsExclude)" ZipFlushFilesLimit="$(_ZipFlushFilesLimit)" ZipFlushSizeLimit="$(_ZipFlushSizeLimit)" - UseAssemblyStore="$(AndroidUseAssemblyStore)"> + UseAssemblySharedLibraries="$(_AndroidUseAssemblySharedLibraries)"> + ZipFlushSizeLimit="$(_ZipFlushSizeLimit)"> libmonodroid_path {utils.path_combine (appDirs[SharedConstants::APP_DIRS_DATA_DIR_INDEX].get_cstr (), "libmonodroid.so")}; - log_debug (LOG_ASSEMBLY, "Checking if libmonodroid was unpacked to %s", libmonodroid_path.get ()); - if (!utils.file_exists (libmonodroid_path.get ())) { - log_debug (LOG_ASSEMBLY, "%s not found, assuming application/android:extractNativeLibs == false", libmonodroid_path.get ()); - set_embedded_dso_mode_enabled (true); - } else { - log_debug (LOG_ASSEMBLY, "Native libs extracted to %s, assuming application/android:extractNativeLibs == true", appDirs[SharedConstants::APP_DIRS_DATA_DIR_INDEX].get_cstr ()); - set_embedded_dso_mode_enabled (false); - } -} - void BasicAndroidSystem::setup_app_library_directories (jstring_array_wrapper& runtimeApks, jstring_array_wrapper& appDirs, bool have_split_apks) { - if (!is_embedded_dso_mode_enabled ()) { + if (!BasicAndroidSystem::is_embedded_dso_mode_enabled ()) { log_debug (LOG_DEFAULT, "Setting up for DSO lookup in app data directories"); BasicAndroidSystem::app_lib_directories_size = 1; BasicAndroidSystem::app_lib_directories = new const char*[app_lib_directories_size](); diff --git a/src/monodroid/jni/basic-android-system.hh b/src/monodroid/jni/basic-android-system.hh index a12ba47fa1d..c04233c9649 100644 --- a/src/monodroid/jni/basic-android-system.hh +++ b/src/monodroid/jni/basic-android-system.hh @@ -6,6 +6,8 @@ #include "cpu-arch.hh" #include "jni-wrappers.hh" +#include "strings.hh" +#include "basic-utilities.hh" namespace xamarin::android::internal { @@ -60,6 +62,29 @@ namespace xamarin::android::internal public: void setup_app_library_directories (jstring_array_wrapper& runtimeApks, jstring_array_wrapper& appDirs, bool have_split_apks); + force_inline + static void detect_embedded_dso_mode (jstring_array_wrapper& appDirs) noexcept + { + // appDirs[SharedConstants::APP_DIRS_DATA_DIR_INDEX] points to the native library directory + dynamic_local_string libmonodroid_path; + jstring_wrapper &libdir = appDirs[SharedConstants::APP_DIRS_DATA_DIR_INDEX]; + libmonodroid_path.append (libdir.get_cstr ()).append (MONODROID_PATH_SEPARATOR).append ("libmonodroid.so"); + + log_debug (LOG_ASSEMBLY, "Checking if libmonodroid was unpacked to %s", libmonodroid_path.get ()); + if (!BasicUtilities::file_exists (libmonodroid_path.get ())) { + log_debug (LOG_ASSEMBLY, "%s not found, assuming application/android:extractNativeLibs == false", libmonodroid_path.get ()); + embedded_dso_mode_enabled = true; + } else { + log_debug (LOG_ASSEMBLY, "Native libs extracted to %s, assuming application/android:extractNativeLibs == true", libdir.get_cstr ()); + embedded_dso_mode_enabled = false; + } + } + + static bool is_embedded_dso_mode_enabled () noexcept + { + return embedded_dso_mode_enabled; + } + const char* get_override_dir (size_t index) const { if (index >= MAX_OVERRIDES) @@ -76,13 +101,6 @@ namespace xamarin::android::internal override_dirs [index] = const_cast (dir); } - bool is_embedded_dso_mode_enabled () const - { - return embedded_dso_mode_enabled; - } - - void detect_embedded_dso_mode (jstring_array_wrapper& appDirs) noexcept; - char *get_runtime_libdir () const { return runtime_libdir; @@ -111,15 +129,10 @@ namespace xamarin::android::internal void setup_apk_directories (unsigned short running_on_cpu, jstring_array_wrapper &runtimeApks, bool have_split_apks) noexcept; char* determine_primary_override_dir (jstring_wrapper &home); - void set_embedded_dso_mode_enabled (bool yesno) noexcept - { - embedded_dso_mode_enabled = yesno; - } - private: - bool embedded_dso_mode_enabled = false; char *runtime_libdir = nullptr; char *primary_override_dir = nullptr; + inline static bool embedded_dso_mode_enabled = false; }; } #endif // !__BASIC_ANDROID_SYSTEM_HH diff --git a/src/monodroid/jni/basic-utilities.cc b/src/monodroid/jni/basic-utilities.cc index adf67f7a9c1..11aef82956a 100644 --- a/src/monodroid/jni/basic-utilities.cc +++ b/src/monodroid/jni/basic-utilities.cc @@ -117,7 +117,7 @@ BasicUtilities::set_user_executable ([[maybe_unused]] const char *path) } bool -BasicUtilities::file_exists (const char *file) +BasicUtilities::file_exists (const char *file) noexcept { monodroid_stat_t s; if (monodroid_stat (file, &s) == 0 && (s.st_mode & S_IFMT) == S_IFREG) @@ -221,7 +221,7 @@ BasicUtilities::monodroid_fopen (const char *filename, const char *mode) } int -BasicUtilities::monodroid_stat (const char *path, monodroid_stat_t *s) +BasicUtilities::monodroid_stat (const char *path, monodroid_stat_t *s) noexcept { int result; diff --git a/src/monodroid/jni/basic-utilities.hh b/src/monodroid/jni/basic-utilities.hh index d7b67d83014..5094aa9780f 100644 --- a/src/monodroid/jni/basic-utilities.hh +++ b/src/monodroid/jni/basic-utilities.hh @@ -27,7 +27,7 @@ namespace xamarin::android { public: FILE *monodroid_fopen (const char* filename, const char* mode); - int monodroid_stat (const char *path, monodroid_stat_t *s); + static int monodroid_stat (const char *path, monodroid_stat_t *s) noexcept; monodroid_dir_t *monodroid_opendir (const char *filename); int monodroid_closedir (monodroid_dir_t *dirp); int monodroid_dirent_hasextension (monodroid_dirent_t *e, const char *extension); @@ -40,7 +40,7 @@ namespace xamarin::android int create_directory (const char *pathname, mode_t mode); void set_world_accessable (const char *path); void set_user_executable (const char *path); - bool file_exists (const char *file); + static bool file_exists (const char *file) noexcept; bool directory_exists (const char *directory); bool file_copy (const char *to, const char *from); diff --git a/src/monodroid/jni/debug-app-helper.cc b/src/monodroid/jni/debug-app-helper.cc index e79c62b238d..17613725dd1 100644 --- a/src/monodroid/jni/debug-app-helper.cc +++ b/src/monodroid/jni/debug-app-helper.cc @@ -15,6 +15,7 @@ #include "debug-app-helper.hh" #include "shared-constants.hh" #include "jni-wrappers.hh" +#include "monodroid-glue-internal.hh" using namespace xamarin::android; using namespace xamarin::android::internal; @@ -74,7 +75,7 @@ Java_mono_android_DebugRuntime_init (JNIEnv *env, [[maybe_unused]] jclass klass, jstring_array_wrapper applicationDirs (env, appDirs); jstring_array_wrapper runtimeApks (env, runtimeApksJava); - androidSystem.detect_embedded_dso_mode (applicationDirs); + BasicAndroidSystem::detect_embedded_dso_mode (applicationDirs); androidSystem.set_primary_override_dir (applicationDirs [0]); androidSystem.set_override_dir (0, androidSystem.get_primary_override_dir ()); androidSystem.setup_app_library_directories (runtimeApks, applicationDirs, haveSplitApks); @@ -206,7 +207,7 @@ get_libmonosgen_path () // storage location before loading it. copy_native_libraries_to_internal_location (); - if (androidSystem.is_embedded_dso_mode_enabled ()) { + if (BasicAndroidSystem::is_embedded_dso_mode_enabled ()) { return SharedConstants::MONO_SGEN_SO; } diff --git a/src/monodroid/jni/embedded-assemblies-zip.cc b/src/monodroid/jni/embedded-assemblies-zip.cc index f7be28a631e..f7723973c51 100644 --- a/src/monodroid/jni/embedded-assemblies-zip.cc +++ b/src/monodroid/jni/embedded-assemblies-zip.cc @@ -1,14 +1,11 @@ #include #include -#include #include -#include #include #include #include "embedded-assemblies.hh" -#include "cpp-util.hh" #include "globals.hh" #include "xamarin-app.hh" @@ -61,12 +58,18 @@ EmbeddedAssemblies::zip_load_entry_common (size_t entry_index, std::vector 1, "assembly_dso_prefix must be longer than 1 byte"); + + if ( // library location is not overridable, so we can test for it beginning with a hardcoded 'l' + (entry_name.get ()[0] != 'l' || entry_name.length () < assembly_dso_prefix.size () || memcmp (assembly_dso_prefix.data (), entry_name.get (), assembly_dso_prefix.size () - 1) != 0) && + (entry_name.get ()[0] != state.prefix[0] || entry_name.length () < state.prefix_len || memcmp (state.prefix, entry_name.get (), state.prefix_len) != 0) + ) { return false; } + state.location = entry_name.get ()[0] == 'l' ? EntryLocation::Libs : EntryLocation::Assemblies; #if defined (NET) - if (application_config.have_runtime_config_blob && !runtime_config_blob_found) { + if (application_config.have_runtime_config_blob && !runtime_config_blob_found && state.location == EntryLocation::Assemblies) { if (utils.ends_with (entry_name, SharedConstants::RUNTIME_CONFIG_BLOB_NAME)) { runtime_config_blob_found = true; runtime_config_blob_mmap = md_mmap_apk_file (state.apk_fd, state.data_offset, state.file_size, entry_name.get ()); @@ -75,9 +78,9 @@ EmbeddedAssemblies::zip_load_entry_common (size_t entry_index, std::vector c continue; } -#if !defined(NET) - if (utils.ends_with (entry_name, ".config")) { - char *assembly_name = strdup (basename (entry_name.get ())); - // Remove '.config' suffix - *strrchr (assembly_name, '.') = '\0'; - - md_mmap_info map_info = md_mmap_apk_file (state.apk_fd, state.data_offset, state.file_size, entry_name.get ()); - mono_register_config_for_assembly (assembly_name, (const char*)map_info.area); - + if (state.location != EntryLocation::Assemblies || !utils.ends_with (entry_name, SharedConstants::DLL_EXTENSION)) { continue; } -#endif // ndef NET - - if (!utils.ends_with (entry_name, SharedConstants::DLL_EXTENSION)) - continue; #if defined (DEBUG) if (entry_is_overridden) @@ -163,88 +154,23 @@ EmbeddedAssemblies::zip_load_individual_assembly_entries (std::vector c set_assembly_entry_data (bundled_assemblies [bundled_assembly_index], state.apk_fd, state.data_offset, state.file_size, state.prefix_len, max_assembly_name_size, entry_name); bundled_assembly_index++; - number_of_found_assemblies = bundled_assembly_index; + number_of_found_assembly_dsos = bundled_assembly_index; } have_and_want_debug_symbols = register_debug_symbols && bundled_debug_data != nullptr; } +#if defined(RELEASE) force_inline void -EmbeddedAssemblies::map_assembly_store (dynamic_local_string const& entry_name, ZipEntryLoadState &state) noexcept -{ - if (number_of_mapped_assembly_stores >= application_config.number_of_assembly_store_files) { - log_fatal (LOG_ASSEMBLY, "Too many assembly stores. Expected at most %u", application_config.number_of_assembly_store_files); - Helpers::abort_application (); - } - - md_mmap_info assembly_store_map = md_mmap_apk_file (state.apk_fd, state.data_offset, state.file_size, entry_name.get ()); - auto header = static_cast(assembly_store_map.area); - - if (header->magic != ASSEMBLY_STORE_MAGIC) { - log_fatal (LOG_ASSEMBLY, "Assembly store '%s' is not a valid Xamarin.Android assembly store file", entry_name.get ()); - Helpers::abort_application (); - } - - if (header->version > ASSEMBLY_STORE_FORMAT_VERSION) { - log_fatal (LOG_ASSEMBLY, "Assembly store '%s' uses format v%u which is not understood by this version of Xamarin.Android", entry_name.get (), header->version); - Helpers::abort_application (); - } - - if (header->store_id >= application_config.number_of_assembly_store_files) { - log_fatal ( - LOG_ASSEMBLY, - "Assembly store '%s' index %u exceeds the number of stores known at application build time, %u", - entry_name.get (), - header->store_id, - application_config.number_of_assembly_store_files - ); - Helpers::abort_application (); - } - - AssemblyStoreRuntimeData &rd = assembly_stores[header->store_id]; - if (rd.data_start != nullptr) { - log_fatal (LOG_ASSEMBLY, "Assembly store '%s' has a duplicate ID (%u)", entry_name.get (), header->store_id); - Helpers::abort_application (); - } - - constexpr size_t header_size = sizeof(AssemblyStoreHeader); - - rd.data_start = static_cast(assembly_store_map.area); - rd.assembly_count = header->local_entry_count; - rd.assemblies = reinterpret_cast(rd.data_start + header_size); - - number_of_found_assemblies += rd.assembly_count; - - if (header->store_id == 0) { - constexpr size_t bundled_assembly_size = sizeof(AssemblyStoreAssemblyDescriptor); - constexpr size_t hash_entry_size = sizeof(AssemblyStoreHashEntry); - - index_assembly_store_header = header; - - size_t bytes_before_hashes = header_size + (bundled_assembly_size * header->local_entry_count); - if constexpr (std::is_same_v) { - assembly_store_hashes = reinterpret_cast(rd.data_start + bytes_before_hashes + (hash_entry_size * header->global_entry_count)); - } else { - assembly_store_hashes = reinterpret_cast(rd.data_start + bytes_before_hashes); - } - } - - number_of_mapped_assembly_stores++; - have_and_want_debug_symbols = register_debug_symbols; -} - -force_inline void -EmbeddedAssemblies::zip_load_assembly_store_entries (std::vector const& buf, uint32_t num_entries, ZipEntryLoadState &state) noexcept +EmbeddedAssemblies::zip_load_standalone_dso_entries (std::vector const& buf, uint32_t num_entries, ZipEntryLoadState &state) noexcept { if (all_required_zip_entries_found ()) { return; } dynamic_local_string entry_name; - bool common_assembly_store_found = false; - bool arch_assembly_store_found = false; - log_debug (LOG_ASSEMBLY, "Looking for assembly stores in APK (common: '%s'; arch-specific: '%s')", assembly_store_common_file_name.data (), assembly_store_arch_file_name.data ()); + log_debug (LOG_ASSEMBLY, "Looking for assembly DSOs in APK, at prefix %s", assembly_dso_prefix); for (size_t i = 0; i < num_entries; i++) { if (all_required_zip_entries_found ()) { need_to_scan_more_apks = false; @@ -252,21 +178,12 @@ EmbeddedAssemblies::zip_load_assembly_store_entries (std::vector const& } bool interesting_entry = zip_load_entry_common (i, buf, entry_name, state); - if (!interesting_entry) { + if (!interesting_entry || state.location != EntryLocation::Libs) { continue; } - - if (!common_assembly_store_found && utils.ends_with (entry_name, assembly_store_common_file_name)) { - common_assembly_store_found = true; - map_assembly_store (entry_name, state); - } - - if (!arch_assembly_store_found && utils.ends_with (entry_name, assembly_store_arch_file_name)) { - arch_assembly_store_found = true; - map_assembly_store (entry_name, state); - } } } +#endif void EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unused]] monodroid_should_register should_register) @@ -309,9 +226,7 @@ EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unus Helpers::abort_application (); } - if (application_config.have_assembly_store) { - zip_load_assembly_store_entries (buf, cd_entries, state); - } else { + { zip_load_individual_assembly_entries (buf, cd_entries, should_register, state); } } diff --git a/src/monodroid/jni/embedded-assemblies.cc b/src/monodroid/jni/embedded-assemblies.cc index 82b7607f437..1bd46cd3b15 100644 --- a/src/monodroid/jni/embedded-assemblies.cc +++ b/src/monodroid/jni/embedded-assemblies.cc @@ -39,6 +39,7 @@ #include "startup-aware-lock.hh" #include "timing-internal.hh" #include "search.hh" +#include "assembly-data-provider.hh" using namespace xamarin::android; using namespace xamarin::android::internal; @@ -85,59 +86,7 @@ force_inline void EmbeddedAssemblies::get_assembly_data (uint8_t *data, uint32_t data_size, [[maybe_unused]] const char *name, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept { #if defined (ANDROID) && defined (HAVE_LZ4) && defined (RELEASE) - auto header = reinterpret_cast(data); - if (header->magic == COMPRESSED_DATA_MAGIC) { - if (XA_UNLIKELY (compressed_assemblies.descriptors == nullptr)) { - log_fatal (LOG_ASSEMBLY, "Compressed assembly found but no descriptor defined"); - Helpers::abort_application (); - } - if (XA_UNLIKELY (header->descriptor_index >= compressed_assemblies.count)) { - log_fatal (LOG_ASSEMBLY, "Invalid compressed assembly descriptor index %u", header->descriptor_index); - Helpers::abort_application (); - } - - CompressedAssemblyDescriptor &cad = compressed_assemblies.descriptors[header->descriptor_index]; - assembly_data_size = data_size - sizeof(CompressedAssemblyHeader); - if (!cad.loaded) { - StartupAwareLock decompress_lock (assembly_decompress_mutex); - - if (cad.loaded) { - set_assembly_data_and_size (reinterpret_cast(cad.data), cad.uncompressed_file_size, assembly_data, assembly_data_size); - return; - } - - if (XA_UNLIKELY (cad.data == nullptr)) { - log_fatal (LOG_ASSEMBLY, "Invalid compressed assembly descriptor at %u: no data", header->descriptor_index); - Helpers::abort_application (); - } - - if (header->uncompressed_length != cad.uncompressed_file_size) { - if (header->uncompressed_length > cad.uncompressed_file_size) { - log_fatal (LOG_ASSEMBLY, "Compressed assembly '%s' is larger than when the application was built (expected at most %u, got %u). Assemblies don't grow just like that!", name, cad.uncompressed_file_size, header->uncompressed_length); - Helpers::abort_application (); - } else { - log_debug (LOG_ASSEMBLY, "Compressed assembly '%s' is smaller than when the application was built. Adjusting accordingly.", name); - } - cad.uncompressed_file_size = header->uncompressed_length; - } - - const char *data_start = reinterpret_cast(data + sizeof(CompressedAssemblyHeader)); - int ret = LZ4_decompress_safe (data_start, reinterpret_cast(cad.data), static_cast(assembly_data_size), static_cast(cad.uncompressed_file_size)); - - if (ret < 0) { - log_fatal (LOG_ASSEMBLY, "Decompression of assembly %s failed with code %d", name, ret); - Helpers::abort_application (); - } - - if (static_cast(ret) != cad.uncompressed_file_size) { - log_debug (LOG_ASSEMBLY, "Decompression of assembly %s yielded a different size (expected %lu, got %u)", name, cad.uncompressed_file_size, static_cast(ret)); - Helpers::abort_application (); - } - cad.loaded = true; - } - - set_assembly_data_and_size (reinterpret_cast(cad.data), cad.uncompressed_file_size, assembly_data, assembly_data_size); - } else + // TODO: implement storing info about standalone assembly DSO offset here #endif { set_assembly_data_and_size (data, data_size, assembly_data, assembly_data_size); @@ -150,12 +99,6 @@ EmbeddedAssemblies::get_assembly_data (XamarinAndroidBundledAssembly const& e, u get_assembly_data (e.data, e.data_size, e.name, assembly_data, assembly_data_size); } -force_inline void -EmbeddedAssemblies::get_assembly_data (AssemblyStoreSingleAssemblyRuntimeData const& e, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept -{ - get_assembly_data (e.image_data, e.descriptor->data_size, "", assembly_data, assembly_data_size); -} - template force_inline void EmbeddedAssemblies::map_runtime_file (XamarinAndroidBundledAssembly& file) noexcept @@ -282,6 +225,18 @@ EmbeddedAssemblies::load_bundled_assembly ( return a; } +#if defined(RELEASE) +template +force_inline MonoAssembly* +EmbeddedAssemblies::standalone_dso_open_from_bundles (dynamic_local_string const& name, TLoaderData loader_data) noexcept +{ + hash_t name_hash = xxhash::hash (name.get (), name.length ()); + log_debug (LOG_ASSEMBLY, "standalone_dso_open_from_bundles: looking for bundled name: '%s', hash 0x%zx", name.get (), name_hash); + + return nullptr; +} +#endif // def RELEASE + template force_inline MonoAssembly* EmbeddedAssemblies::individual_assemblies_open_from_bundles (dynamic_local_string& name, TLoaderData loader_data, bool ref_only) noexcept @@ -319,144 +274,6 @@ EmbeddedAssemblies::individual_assemblies_open_from_bundles (dynamic_local_strin return nullptr; } -force_inline const AssemblyStoreHashEntry* -EmbeddedAssemblies::find_assembly_store_entry ([[maybe_unused]] hash_t hash, [[maybe_unused]] const AssemblyStoreHashEntry *entries, [[maybe_unused]] size_t entry_count) noexcept -{ -#if !defined (__MINGW32__) || (defined (__MINGW32__) && __GNUC__ >= 10) - hash_t entry_hash; - const AssemblyStoreHashEntry *ret = nullptr; - - while (entry_count > 0) { - ret = entries + (entry_count / 2); - if constexpr (std::is_same_v) { - entry_hash = ret->hash64; - } else { - entry_hash = ret->hash32; - } - auto result = hash <=> entry_hash; - - if (result < 0) { - entry_count /= 2; - } else if (result > 0) { - entries = ret + 1; - entry_count -= entry_count / 2 + 1; - } else { - return ret; - } - } -#endif // ndef __MINGW32__ || (def __MINGW32__ && __GNUC__ >= 10) - - return nullptr; -} - -template -force_inline MonoAssembly* -EmbeddedAssemblies::assembly_store_open_from_bundles (dynamic_local_string& name, TLoaderData loader_data, bool ref_only) noexcept -{ - size_t len = name.length (); - bool have_dll_ext = utils.ends_with (name, SharedConstants::DLL_EXTENSION); - - if (have_dll_ext) { - len -= sizeof(SharedConstants::DLL_EXTENSION) - 1; - } - - hash_t name_hash = xxhash::hash (name.get (), len); - log_debug (LOG_ASSEMBLY, "assembly_store_open_from_bundles: looking for bundled name: '%s' (hash 0x%zx)", name.get (), name_hash); - - const AssemblyStoreHashEntry *hash_entry = find_assembly_store_entry (name_hash, assembly_store_hashes, application_config.number_of_assemblies_in_apk); - if (hash_entry == nullptr) { - log_warn (LOG_ASSEMBLY, "Assembly '%s' (hash 0x%zx) not found", name.get (), name_hash); - return nullptr; - } - - if (hash_entry->mapping_index >= application_config.number_of_assemblies_in_apk) { - log_fatal (LOG_ASSEMBLY, "Invalid assembly index %u, exceeds the maximum index of %u", hash_entry->mapping_index, application_config.number_of_assemblies_in_apk - 1); - Helpers::abort_application (); - } - - AssemblyStoreSingleAssemblyRuntimeData &assembly_runtime_info = assembly_store_bundled_assemblies[hash_entry->mapping_index]; - if (assembly_runtime_info.image_data == nullptr) { - if (hash_entry->store_id >= application_config.number_of_assembly_store_files) { - log_fatal (LOG_ASSEMBLY, "Invalid assembly store ID %u, exceeds the maximum of %u", hash_entry->store_id, application_config.number_of_assembly_store_files - 1); - Helpers::abort_application (); - } - - AssemblyStoreRuntimeData &rd = assembly_stores[hash_entry->store_id]; - if (hash_entry->local_store_index >= rd.assembly_count) { - log_fatal (LOG_ASSEMBLY, "Invalid index %u into local store assembly descriptor array", hash_entry->local_store_index); - Helpers::abort_application (); - } - - AssemblyStoreAssemblyDescriptor *bba = &rd.assemblies[hash_entry->local_store_index]; - - // The assignments here don't need to be atomic, the value will always be the same, so even if two threads - // arrive here at the same time, nothing bad will happen. - assembly_runtime_info.image_data = rd.data_start + bba->data_offset; - assembly_runtime_info.descriptor = bba; - - if (bba->debug_data_offset != 0) { - assembly_runtime_info.debug_info_data = rd.data_start + bba->debug_data_offset; - } -#if !defined (NET) - if (bba->config_data_size != 0) { - assembly_runtime_info.config_data = rd.data_start + bba->config_data_offset; - - // Mono takes ownership of the pointers - mono_register_config_for_assembly ( - utils.string_concat (name.get (), ".dll"), - utils.strdup_new (reinterpret_cast(assembly_runtime_info.config_data)) - ); - } -#endif // NET - - log_debug ( - LOG_ASSEMBLY, - "Mapped: image_data == %p; debug_info_data == %p; config_data == %p; descriptor == %p; data size == %u; debug data size == %u; config data size == %u; name == '%s'", - assembly_runtime_info.image_data, - assembly_runtime_info.debug_info_data, - assembly_runtime_info.config_data, - assembly_runtime_info.descriptor, - assembly_runtime_info.descriptor->data_size, - assembly_runtime_info.descriptor->debug_data_size, - assembly_runtime_info.descriptor->config_data_size, - name.get () - ); - } - - uint8_t *assembly_data; - uint32_t assembly_data_size; - - if (!have_dll_ext) { - // AOT needs this since Mono will form the DSO name by appending the .so suffix to the assembly name passed to - // functions below and `monodroid_dlopen` uses the `.dll.so` extension to check whether we're being asked to load - // the AOTd code for an assembly. - name.append (SharedConstants::DLL_EXTENSION); - } - - get_assembly_data (assembly_runtime_info, assembly_data, assembly_data_size); - MonoImage *image = MonoImageLoader::load (name, loader_data, name_hash, assembly_data, assembly_data_size); - if (image == nullptr) { - log_warn (LOG_ASSEMBLY, "Failed to load MonoImage of '%s'", name.get ()); - return nullptr; - } - - if (have_and_want_debug_symbols && assembly_runtime_info.debug_info_data != nullptr) { - log_debug (LOG_ASSEMBLY, "Registering debug data for assembly '%s'", name.get ()); - mono_debug_open_image_from_memory (image, reinterpret_cast (assembly_runtime_info.debug_info_data), static_cast(assembly_runtime_info.descriptor->debug_data_size)); - } - - MonoImageOpenStatus status; - MonoAssembly *a = mono_assembly_load_from_full (image, name.get (), &status, ref_only); - if (a == nullptr || status != MonoImageOpenStatus::MONO_IMAGE_OK) { - log_warn (LOG_ASSEMBLY, "Failed to load managed assembly '%s'. %s", name.get (), mono_image_strerror (status)); - return nullptr; - } - -#if !defined (NET) - mono_config_for_assembly (image); -#endif - return a; -} // TODO: need to forbid loading assemblies into non-default ALC if they contain marshal method callbacks. // The best way is probably to store the information in the assembly `MonoImage*` cache. We should @@ -475,13 +292,22 @@ EmbeddedAssemblies::open_from_bundles (MonoAssemblyName* aname, TLoaderData load } name.append_c (asmname); + size_t time_index; + if (XA_UNLIKELY (FastTiming::enabled ())) { + timing = new Timing (); + time_index = internal_timing->start_event (TimingEventKind::Unspecified); + } + MonoAssembly *a; - if (application_config.have_assembly_store) { - a = assembly_store_open_from_bundles (name, loader_data, ref_only); - } else { + { a = individual_assemblies_open_from_bundles (name, loader_data, ref_only); } + if (XA_UNLIKELY (FastTiming::enabled ())) { + internal_timing->end_event (time_index, true); + internal_timing->add_more_info (time_index, name.get ()); + } + if (a == nullptr) { log_warn (LOG_ASSEMBLY, "open_from_bundles: failed to load assembly %s", name.get ()); } @@ -899,7 +725,7 @@ EmbeddedAssemblies::typemap_managed_to_java (MonoReflectionType *reflection_type } EmbeddedAssemblies::md_mmap_info -EmbeddedAssemblies::md_mmap_apk_file (int fd, uint32_t offset, size_t size, const char* filename) +EmbeddedAssemblies::md_mmap_apk_file (int fd, uint32_t offset, size_t size, const char* filename) noexcept { md_mmap_info file_info; md_mmap_info mmap_info; @@ -930,15 +756,13 @@ EmbeddedAssemblies::md_mmap_apk_file (int fd, uint32_t offset, size_t size, cons void EmbeddedAssemblies::gather_bundled_assemblies_from_apk (const char* apk, monodroid_should_register should_register) { - int fd; - - if ((fd = open (apk, O_RDONLY)) < 0) { + if ((apk_fd = open (apk, O_RDONLY)) < 0) { log_error (LOG_DEFAULT, "ERROR: Unable to load application package %s.", apk); Helpers::abort_application (); } - log_info (LOG_ASSEMBLY, "APK %s FD: %d", apk, fd); + log_info (LOG_ASSEMBLY, "APK %s FD: %d", apk, apk_fd); - zip_load_entries (fd, apk, should_register); + zip_load_entries (apk_fd, apk, should_register); } #if defined (DEBUG) || !defined (ANDROID) @@ -1220,11 +1044,11 @@ EmbeddedAssemblies::try_load_typemaps_from_directory (const char *path) size_t EmbeddedAssemblies::register_from (const char *apk_file, monodroid_should_register should_register) { - size_t prev = number_of_found_assemblies; + size_t prev = number_of_found_assembly_dsos; gather_bundled_assemblies_from_apk (apk_file, should_register); - log_info (LOG_ASSEMBLY, "Package '%s' contains %i assemblies", apk_file, number_of_found_assemblies - prev); + log_info (LOG_ASSEMBLY, "Package '%s' contains %i assemblies", apk_file, number_of_found_assembly_dsos - prev); - return number_of_found_assemblies; + return number_of_found_assembly_dsos; } diff --git a/src/monodroid/jni/embedded-assemblies.hh b/src/monodroid/jni/embedded-assemblies.hh index 4b4c3324dae..af662e4a656 100644 --- a/src/monodroid/jni/embedded-assemblies.hh +++ b/src/monodroid/jni/embedded-assemblies.hh @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -18,6 +19,8 @@ #include #endif +#include "android-system.hh" +#include "assembly-data-provider.hh" #include "strings.hh" #include "xamarin-app.hh" #include "cpp-util.hh" @@ -35,6 +38,12 @@ struct TypeMapHeader; +namespace { + namespace detail { + class SO_AssemblyDataProvider; + } +} + namespace xamarin::android::internal { #if defined (DEBUG) || !defined (ANDROID) struct TypeMappingInfo; @@ -67,8 +76,16 @@ namespace xamarin::android::internal { #define LoaderData typename #endif + enum class EntryLocation + { + Assemblies, + Libs, + }; + class EmbeddedAssemblies final { + friend class detail::SO_AssemblyDataProvider; + struct md_mmap_info { void *area; size_t size; @@ -85,6 +102,7 @@ namespace xamarin::android::internal { uint32_t local_header_offset; uint32_t data_offset; uint32_t file_size; + EntryLocation location; }; private: @@ -97,11 +115,14 @@ namespace xamarin::android::internal { static constexpr char assemblies_prefix[] = "assemblies/"; static constexpr char zip_path_separator[] = "/"; - static constexpr char assembly_store_prefix[] = "assemblies"; - static constexpr char assembly_store_extension[] = ".blob"; - static constexpr auto assembly_store_common_file_name = concat_const ("/", assembly_store_prefix, assembly_store_extension); - static constexpr auto assembly_store_arch_file_name = concat_const ("/", assembly_store_prefix, ".", SharedConstants::android_abi, assembly_store_extension); + // if the `libXA` embedded assembly prefix changes here, it must be updated in + // src/src/Xamarin.Android.Build.Tasks/Tasks/PrepareAssemblyStandaloneDSOAbiItems.cs as well + static constexpr auto assembly_dso_prefix = concat_const ("lib/", SharedConstants::android_apk_abi, "/libXA"); + // This includes the `.0000.so` suffix each assembly DSO must have and at least one name character (in addition + // to the `libXA` prefix) + static constexpr size_t assembly_dso_min_length = assembly_dso_prefix.size () - 1 + 9; + static constexpr size_t assembly_index_start_offset = 7; #if defined (DEBUG) || !defined (ANDROID) static constexpr char override_typemap_entry_name[] = ".__override__"; @@ -112,10 +133,8 @@ namespace xamarin::android::internal { using monodroid_should_register = bool (*)(const char *filename); public: -#if defined (RELEASE) && defined (ANDROID) EmbeddedAssemblies () noexcept {} -#endif // def RELEASE && def ANDROID #if defined (DEBUG) || !defined (ANDROID) void try_load_typemaps_from_directory (const char *path); @@ -168,21 +187,15 @@ namespace xamarin::android::internal { return need_to_scan_more_apks; } - void ensure_valid_assembly_stores () const noexcept - { - if (!application_config.have_assembly_store) { - return; - } - - abort_unless (index_assembly_store_header != nullptr && assembly_store_hashes != nullptr, "Invalid or incomplete assembly store data"); - } - private: STATIC_IN_ANDROID_RELEASE const char* typemap_managed_to_java (MonoType *type, MonoClass *klass, const uint8_t *mvid) noexcept; STATIC_IN_ANDROID_RELEASE MonoReflectionType* typemap_java_to_managed (hash_t hash, const MonoString *java_type_name) noexcept; size_t register_from (const char *apk_file, monodroid_should_register should_register); void gather_bundled_assemblies_from_apk (const char* apk, monodroid_should_register should_register); + template + MonoAssembly* standalone_dso_open_from_bundles (dynamic_local_string const& name, TLoaderData loader_data) noexcept; + template MonoAssembly* individual_assemblies_open_from_bundles (dynamic_local_string& name, TLoaderData loader_data, bool ref_only) noexcept; @@ -216,7 +229,7 @@ namespace xamarin::android::internal { const TypeMapEntry *typemap_managed_to_java (const char *managed_type_name) noexcept; #endif // DEBUG || !ANDROID - static md_mmap_info md_mmap_apk_file (int fd, uint32_t offset, size_t size, const char* filename); + static md_mmap_info md_mmap_apk_file (int fd, uint32_t offset, size_t size, const char* filename) noexcept; static MonoAssembly* open_from_bundles_full (MonoAssemblyName *aname, char **assemblies_path, void *user_data); #if defined (NET) static MonoAssembly* open_from_bundles (MonoAssemblyLoadContextGCHandle alc_gchandle, MonoAssemblyName *aname, char **assemblies_path, void *user_data, MonoError *error); @@ -226,15 +239,16 @@ namespace xamarin::android::internal { void set_assembly_data_and_size (uint8_t* source_assembly_data, uint32_t source_assembly_data_size, uint8_t*& dest_assembly_data, uint32_t& dest_assembly_data_size) noexcept; void get_assembly_data (uint8_t *data, uint32_t data_size, const char *name, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept; void get_assembly_data (XamarinAndroidBundledAssembly const& e, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept; - void get_assembly_data (AssemblyStoreSingleAssemblyRuntimeData const& e, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept; - void zip_load_entries (int fd, const char *apk_name, monodroid_should_register should_register); void zip_load_individual_assembly_entries (std::vector const& buf, uint32_t num_entries, monodroid_should_register should_register, ZipEntryLoadState &state) noexcept; - void zip_load_assembly_store_entries (std::vector const& buf, uint32_t num_entries, ZipEntryLoadState &state) noexcept; bool zip_load_entry_common (size_t entry_index, std::vector const& buf, dynamic_local_string &entry_name, ZipEntryLoadState &state) noexcept; bool zip_read_cd_info (int fd, uint32_t& cd_offset, uint32_t& cd_size, uint16_t& cd_entries); bool zip_adjust_data_offset (int fd, ZipEntryLoadState &state); +#if defined (RELEASE) + void zip_load_standalone_dso_entries (std::vector const& buf, uint32_t num_entries, ZipEntryLoadState &state) noexcept; +#endif // def RELEASE + template bool zip_extract_cd_info (std::array const& buf, uint32_t& cd_offset, uint32_t& cd_size, uint16_t& cd_entries); @@ -279,11 +293,7 @@ namespace xamarin::android::internal { bool all_required_zip_entries_found () const noexcept { return - number_of_mapped_assembly_stores == application_config.number_of_assembly_store_files -#if defined (NET) - && ((application_config.have_runtime_config_blob && runtime_config_blob_found) || !application_config.have_runtime_config_blob) -#endif // NET - ; + ((application_config.have_runtime_config_blob && runtime_config_blob_found) || !application_config.have_runtime_config_blob); } static force_inline c_unique_ptr to_utf8 (const MonoString *s) noexcept @@ -307,7 +317,6 @@ namespace xamarin::android::internal { void set_assembly_entry_data (XamarinAndroidBundledAssembly &entry, int apk_fd, uint32_t data_offset, uint32_t data_size, uint32_t prefix_len, uint32_t max_name_size, dynamic_local_string const& entry_name) noexcept; void set_debug_entry_data (XamarinAndroidBundledAssembly &entry, int apk_fd, uint32_t data_offset, uint32_t data_size, uint32_t prefix_len, uint32_t max_name_size, dynamic_local_string const& entry_name) noexcept; void map_assembly_store (dynamic_local_string const& entry_name, ZipEntryLoadState &state) noexcept; - const AssemblyStoreHashEntry* find_assembly_store_entry (hash_t hash, const AssemblyStoreHashEntry *entries, size_t entry_count) noexcept; private: std::vector *bundled_debug_data = nullptr; @@ -316,7 +325,7 @@ namespace xamarin::android::internal { bool register_debug_symbols; bool have_and_want_debug_symbols; size_t bundled_assembly_index = 0; - size_t number_of_found_assemblies = 0; + size_t number_of_found_assembly_dsos = 0; #if defined (DEBUG) || !defined (ANDROID) TypeMappingInfo *java_to_managed_maps; @@ -329,12 +338,9 @@ namespace xamarin::android::internal { md_mmap_info runtime_config_blob_mmap{}; bool runtime_config_blob_found = false; #endif // def NET - uint32_t number_of_mapped_assembly_stores = 0; + uint32_t number_of_standalone_dsos = 0; bool need_to_scan_more_apks = true; - - AssemblyStoreHeader *index_assembly_store_header = nullptr; - AssemblyStoreHashEntry *assembly_store_hashes; - std::mutex assembly_decompress_mutex; + int apk_fd = -1; }; } diff --git a/src/monodroid/jni/globals.cc b/src/monodroid/jni/globals.cc index 697ef2d037c..a390e650c0f 100644 --- a/src/monodroid/jni/globals.cc +++ b/src/monodroid/jni/globals.cc @@ -9,7 +9,4 @@ OSBridge osBridge; EmbeddedAssemblies embeddedAssemblies; MonodroidRuntime monodroidRuntime; Timing *timing = nullptr; -#ifndef ANDROID -DesignerAssemblies designerAssemblies; -#endif Debug debug; diff --git a/src/monodroid/jni/globals.hh b/src/monodroid/jni/globals.hh index e163ffc2e00..17c9034a562 100644 --- a/src/monodroid/jni/globals.hh +++ b/src/monodroid/jni/globals.hh @@ -19,8 +19,4 @@ extern xamarin::android::internal::EmbeddedAssemblies embeddedAssemblies; extern xamarin::android::internal::MonodroidRuntime monodroidRuntime; extern xamarin::android::Timing *timing; -#ifndef ANDROID -extern xamarin::android::internal::DesignerAssemblies designerAssemblies; -#endif - #endif // !__GLOBALS_H diff --git a/src/monodroid/jni/logger.cc b/src/monodroid/jni/logger.cc index 57f1ea79957..24146e25779 100644 --- a/src/monodroid/jni/logger.cc +++ b/src/monodroid/jni/logger.cc @@ -16,6 +16,7 @@ #include "debug.hh" #include "util.hh" #include "globals.hh" +#include "strings.hh" #undef DO_LOG #define DO_LOG(_level_,_category_,_format_,_args_) \ @@ -145,11 +146,8 @@ set_category (const char (&name)[NameSize], string_segment& arg, unsigned int en } void -init_logging_categories (char*& mono_log_mask, char*& mono_log_level) +init_logging_categories (xamarin::android::internal::dynamic_local_string& mono_log_mask, xamarin::android::internal::dynamic_local_string& mono_log_level) noexcept { - mono_log_mask = nullptr; - mono_log_level = nullptr; - #if !ANDROID log_categories = LOG_DEFAULT; #endif @@ -254,14 +252,14 @@ init_logging_categories (char*& mono_log_mask, char*& mono_log_level) constexpr char MONO_LOG_MASK_ARG[] = "mono_log_mask="; constexpr size_t MONO_LOG_MASK_ARG_LEN = sizeof(MONO_LOG_MASK_ARG) - 1; if (param.starts_with (MONO_LOG_MASK_ARG)) { - mono_log_mask = utils.strdup_new (param, MONO_LOG_MASK_ARG_LEN); + mono_log_mask.append (param, MONO_LOG_MASK_ARG_LEN); continue; } constexpr char MONO_LOG_LEVEL_ARG[] = "mono_log_level="; constexpr size_t MONO_LOG_LEVEL_ARG_LEN = sizeof(MONO_LOG_LEVEL_ARG) - 1; if (param.starts_with (MONO_LOG_LEVEL_ARG)) { - mono_log_level = utils.strdup_new (param, MONO_LOG_LEVEL_ARG_LEN); + mono_log_level.append (param, MONO_LOG_LEVEL_ARG_LEN); continue; } diff --git a/src/monodroid/jni/logger.hh b/src/monodroid/jni/logger.hh index 2f65333751f..88fc0d26a4c 100644 --- a/src/monodroid/jni/logger.hh +++ b/src/monodroid/jni/logger.hh @@ -1,6 +1,9 @@ #ifndef __MONODROID_LOGGER_H__ #define __MONODROID_LOGGER_H__ +#include +#include + #include "java-interop-logger.h" #ifndef ANDROID @@ -17,7 +20,13 @@ typedef enum android_LogPriority { } android_LogPriority; #endif -void init_logging_categories (char*& mono_log_mask, char*& mono_log_level); +namespace xamarin::android::internal +{ + template + class dynamic_local_string; +} + +void init_logging_categories (xamarin::android::internal::dynamic_local_string& mono_log_mask, xamarin::android::internal::dynamic_local_string& mono_log_level) noexcept; void init_reference_logging (const char *override_dir); diff --git a/src/monodroid/jni/mono-image-loader.hh b/src/monodroid/jni/mono-image-loader.hh index 87bcbc2ed3a..fcd1b434e52 100644 --- a/src/monodroid/jni/mono-image-loader.hh +++ b/src/monodroid/jni/mono-image-loader.hh @@ -120,7 +120,7 @@ namespace xamarin::android::internal { #if defined (USE_CACHE) ssize_t index = find_index (hash); if (index < 0) { - log_warn (LOG_ASSEMBLY, "Failed to look up image index for hash 0x%zx", hash); + //log_warn (LOG_ASSEMBLY, "Failed to look up image index for hash 0x%zx", hash); return image; } diff --git a/src/monodroid/jni/monodroid-glue-internal.hh b/src/monodroid/jni/monodroid-glue-internal.hh index 84ac5f5188a..219be96f349 100644 --- a/src/monodroid/jni/monodroid-glue-internal.hh +++ b/src/monodroid/jni/monodroid-glue-internal.hh @@ -254,7 +254,7 @@ namespace xamarin::android::internal static void mono_log_standard_streams_handler (const char *str, mono_bool is_stdout); // A reference to unique_ptr is not the best practice ever, but it's faster this way - void setup_mono_tracing (std::unique_ptr const& mono_log_mask, bool have_log_assembly, bool have_log_gc); + void setup_mono_tracing (dynamic_local_string const& mono_log_mask, bool have_log_assembly, bool have_log_gc); void install_logging_handlers (); #endif // def ANDROID diff --git a/src/monodroid/jni/monodroid-glue.cc b/src/monodroid/jni/monodroid-glue.cc index d5f8f5ae86d..707438250b4 100644 --- a/src/monodroid/jni/monodroid-glue.cc +++ b/src/monodroid/jni/monodroid-glue.cc @@ -395,8 +395,6 @@ MonodroidRuntime::gather_bundled_assemblies (jstring_array_wrapper &runtimeApks, break; } } - - embeddedAssemblies.ensure_valid_assembly_stores (); } #if defined (DEBUG) && !defined (WINDOWS) @@ -853,15 +851,19 @@ MonodroidRuntime::create_domain (JNIEnv *env, jstring_array_wrapper &runtimeApks } #endif // def NET - if (user_assemblies_count == 0 && androidSystem.count_override_assemblies () == 0 && !is_running_on_desktop) { + if (user_assemblies_count == 0 && androidSystem.count_override_assemblies () == 0) { #if defined (DEBUG) log_fatal (LOG_DEFAULT, "No assemblies found in '%s' or '%s'. Assuming this is part of Fast Deployment. Exiting...", androidSystem.get_override_dir (0), (AndroidSystem::MAX_OVERRIDES > 1 && androidSystem.get_override_dir (1) != nullptr) ? androidSystem.get_override_dir (1) : ""); #else - log_fatal (LOG_DEFAULT, "No assemblies (or assembly blobs) were found in the application APK file(s)"); + log_fatal (LOG_DEFAULT, "No assemblies (or assembly shared libraries) were found in the application APK file(s)"); +#endif + log_fatal (LOG_DEFAULT, "Make sure that all entries (if any) in the APK directory named `assemblies/` are STORED (not compressed)"); +#if defined (RELEASE) + log_fatal (LOG_DEFAULT, "Make sure that all the shared libraries (if any) found in the APK directory named lib/%s are STORED, or that they are extracted to the filesystem", SharedConstants::android_apk_abi); + log_fatal (LOG_DEFAULT, "Check the 'android:extractNativeLibs=' attribute of the 'application' element in your AndroidManifest.xml file to see whether shared libraries are extracted or not"); #endif - log_fatal (LOG_DEFAULT, "Make sure that all entries in the APK directory named `assemblies/` are STORED (not compressed)"); log_fatal (LOG_DEFAULT, "If Android Gradle Plugin's minification feature is enabled, it is likely all the entries in `assemblies/` are compressed"); Helpers::abort_application (); @@ -2023,7 +2025,7 @@ MonodroidRuntime::get_my_location (bool remove_file_name) #if defined (ANDROID) force_inline void -MonodroidRuntime::setup_mono_tracing (std::unique_ptr const& mono_log_mask, bool have_log_assembly, bool have_log_gc) +MonodroidRuntime::setup_mono_tracing (dynamic_local_string const& mono_log_mask, bool have_log_assembly, bool have_log_gc) { constexpr char MASK_ASM[] = "asm"; constexpr size_t MASK_ASM_LEN = sizeof(MASK_ASM) - 1; @@ -2034,7 +2036,7 @@ MonodroidRuntime::setup_mono_tracing (std::unique_ptr const& mono_log_ma constexpr char COMMA[] = ","; dynamic_local_string log_mask; - if (mono_log_mask == nullptr || *mono_log_mask.get () == '\0') { + if (mono_log_mask.empty () || *mono_log_mask.get () == '\0') { if (have_log_assembly) { log_mask.append (MASK_ASM); log_mask.append (COMMA); @@ -2114,13 +2116,10 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl jobject loader, jobjectArray assembliesJava, jint apiLevel, jboolean isEmulator, jboolean haveSplitApks) { - char *mono_log_mask_raw = nullptr; - char *mono_log_level_raw = nullptr; + dynamic_local_string mono_log_mask; + dynamic_local_string mono_log_level; - init_logging_categories (mono_log_mask_raw, mono_log_level_raw); - - std::unique_ptr mono_log_mask (mono_log_mask_raw); - std::unique_ptr mono_log_level (mono_log_level_raw); + init_logging_categories (mono_log_mask, mono_log_level); // If fast logging is disabled, log messages immediately FastTiming::initialize ((log_timing_categories & LOG_TIMING_FAST_BARE) == 0); @@ -2134,6 +2133,8 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl jstring_array_wrapper applicationDirs (env, appDirs); jstring_wrapper &home = applicationDirs[SharedConstants::APP_DIRS_FILES_DIR_INDEX]; + BasicAndroidSystem::detect_embedded_dso_mode (applicationDirs); + #if defined (NET) mono_opt_aot_lazy_assembly_load = application_config.aot_lazy_load ? TRUE : FALSE; @@ -2155,7 +2156,6 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl #endif // def NET android_api_level = apiLevel; - androidSystem.detect_embedded_dso_mode (applicationDirs); androidSystem.set_running_in_emulator (isEmulator); java_TimeZone = utils.get_class_from_runtime_field (env, klass, "java_util_TimeZone", true); @@ -2189,7 +2189,7 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl bool have_log_assembly = (log_categories & LOG_ASSEMBLY) != 0; bool have_log_gc = (log_categories & LOG_GC) != 0; - if (mono_log_level == nullptr || *mono_log_level.get () == '\0') { + if (mono_log_level.empty () || *mono_log_level.get () == '\0') { mono_trace_set_level_string ((have_log_assembly || have_log_gc) ? "info" : "error"); } else { mono_trace_set_level_string (mono_log_level.get ()); diff --git a/src/monodroid/jni/search.hh b/src/monodroid/jni/search.hh index 554126d2bee..472330fcfa8 100644 --- a/src/monodroid/jni/search.hh +++ b/src/monodroid/jni/search.hh @@ -11,21 +11,33 @@ namespace xamarin::android::internal { class Search final { public: - force_inline static ssize_t binary_search (hash_t key, const hash_t *arr, size_t n) noexcept + template + force_inline static ssize_t binary_search (hash_t key, const T *arr, size_t n) noexcept { + static_assert (equal != nullptr, "equal is a required templae parameter"); + static_assert (less_than != nullptr, "less_than is a required templae parameter"); + ssize_t left = -1; ssize_t right = static_cast(n); while (right - left > 1) { ssize_t middle = (left + right) >> 1; - if (arr[middle] < key) { + if (less_than (arr[middle], key)) { left = middle; } else { right = middle; } } - return arr[right] == key ? right : -1; + return equal (arr[right], key) ? right : -1; + } + + force_inline static ssize_t binary_search (hash_t key, const hash_t *arr, size_t n) noexcept + { + auto equal = [](hash_t const& entry, hash_t key) -> bool { return entry == key; }; + auto less_than = [](hash_t const& entry, hash_t key) -> bool { return entry < key; }; + + return binary_search (key, arr, n); } force_inline static ptrdiff_t binary_search_branchless (hash_t x, const hash_t *arr, uint32_t len) noexcept diff --git a/src/monodroid/jni/shared-constants.hh b/src/monodroid/jni/shared-constants.hh index a88dbcdf569..b1fdc5189f3 100644 --- a/src/monodroid/jni/shared-constants.hh +++ b/src/monodroid/jni/shared-constants.hh @@ -1,6 +1,7 @@ #ifndef __SHARED_CONSTANTS_HH #define __SHARED_CONSTANTS_HH +#include #include "cpp-util.hh" namespace xamarin::android::internal @@ -17,6 +18,18 @@ namespace xamarin::android::internal class SharedConstants { public: +#if defined (RELEASE) + static constexpr bool is_release = true; +#else + static constexpr bool is_release = false; +#endif + +#if defined (RELEASE) + static constexpr bool is_debug = false; +#else + static constexpr bool is_debug = true; +#endif + #if defined (NET) static constexpr char MONO_ANDROID_RUNTIME_ASSEMBLY_NAME[] = "Mono.Android.Runtime"; #endif @@ -51,15 +64,19 @@ namespace xamarin::android::internal #if __arm__ static constexpr char android_abi[] = "armeabi_v7a"; + static constexpr char android_apk_abi[] = "armeabi_v7a"; static constexpr char runtime_identifier[] = "android-arm"; #elif __aarch64__ static constexpr char android_abi[] = "arm64_v8a"; + static constexpr char android_apk_abi[] { "arm64-v8a" }; static constexpr char runtime_identifier[] = "android-arm64"; #elif __x86_64__ static constexpr char android_abi[] = "x86_64"; + static constexpr char android_apk_abi[] = "x86_64"; static constexpr char runtime_identifier[] = "android-x64"; #elif __i386__ static constexpr char android_abi[] = "x86"; + static constexpr char android_apk_abi[] = "x86"; static constexpr char runtime_identifier[] = "android-x86"; #endif diff --git a/src/monodroid/jni/strings.hh b/src/monodroid/jni/strings.hh index bf41b519b44..13a64467b5d 100644 --- a/src/monodroid/jni/strings.hh +++ b/src/monodroid/jni/strings.hh @@ -444,6 +444,11 @@ namespace xamarin::android::internal return *this; } + force_inline string_base& append (const TChar *s) noexcept + { + return append (s, strlen (s)); + } + template force_inline string_base& append (internal::string_base const& str) noexcept { @@ -456,6 +461,11 @@ namespace xamarin::android::internal return append (s, Size - 1); } + force_inline string_base& append (string_segment const& s, size_t start_pos = 0) noexcept + { + return append (s.start () + start_pos); + } + force_inline string_base& append_c (const char *s) noexcept { if (s == nullptr) diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index d50fc39a683..676d98d3faa 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -11,10 +11,6 @@ #include "xxhash.hh" static constexpr uint64_t FORMAT_TAG = 0x015E6972616D58; -static constexpr uint32_t COMPRESSED_DATA_MAGIC = 0x5A4C4158; // 'XALZ', little-endian -static constexpr uint32_t ASSEMBLY_STORE_MAGIC = 0x41424158; // 'XABA', little-endian -static constexpr uint32_t ASSEMBLY_STORE_FORMAT_VERSION = 1; // Increase whenever an incompatible change is made to the - // assembly store format static constexpr uint32_t MODULE_MAGIC_NAMES = 0x53544158; // 'XATS', little-endian static constexpr uint32_t MODULE_INDEX_MAGIC = 0x49544158; // 'XATI', little-endian static constexpr uint8_t MODULE_FORMAT_VERSION = 2; // Keep in sync with the value in src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs @@ -81,26 +77,6 @@ struct TypeMapJava }; #endif -struct CompressedAssemblyHeader -{ - uint32_t magic; // COMPRESSED_DATA_MAGIC - uint32_t descriptor_index; - uint32_t uncompressed_length; -}; - -struct CompressedAssemblyDescriptor -{ - uint32_t uncompressed_file_size; - bool loaded; - uint8_t *data; -}; - -struct CompressedAssemblies -{ - uint32_t count; - CompressedAssemblyDescriptor *descriptors; -}; - struct XamarinAndroidBundledAssembly final { int32_t apk_fd; @@ -111,86 +87,6 @@ struct XamarinAndroidBundledAssembly final char *name; }; -// -// Assembly store format -// -// The separate hash indices for 32 and 64-bit hashes are required because they will be sorted differently. -// The 'index' field of each of the hashes{32,64} entry points not only into the `assemblies` array in the -// store but also into the `uint8_t*` `assembly_store_bundled_assemblies*` arrays. -// -// This way the `assemblies` array in the store can remain read only, because we write the "mapped" assembly -// pointer somewhere else. Otherwise we'd have to copy the `assemblies` array to a writable area of memory. -// -// Each store has a unique ID assigned, which is an index into an array of pointers to arrays which store -// individual assembly addresses. Only store with ID 0 comes with the hashes32 and hashes64 arrays. This is -// done to make it possible to use a single sorted array to find assemblies insted of each store having its -// own sorted array of hashes, which would require several binary searches instead of just one. -// -// AssemblyStoreHeader header; -// AssemblyStoreAssemblyDescriptor assemblies[header.local_entry_count]; -// AssemblyStoreHashEntry hashes32[header.global_entry_count]; // only in assembly store with ID 0 -// AssemblyStoreHashEntry hashes64[header.global_entry_count]; // only in assembly store with ID 0 -// [DATA] -// - -// -// The structures which are found in the store files must be packed to avoid problems when calculating offsets (runtime -// size of a structure can be different than the real data size) -// -struct [[gnu::packed]] AssemblyStoreHeader final -{ - uint32_t magic; - uint32_t version; - uint32_t local_entry_count; - uint32_t global_entry_count; - uint32_t store_id; -}; - -struct [[gnu::packed]] AssemblyStoreHashEntry final -{ - union { - uint64_t hash64; - uint32_t hash32; - }; - - // Index into the array with pointers to assembly data. - // It **must** be unique across all the stores from all the apks - uint32_t mapping_index; - - // Index into the array with assembly descriptors inside a store - uint32_t local_store_index; - - // Index into the array with assembly store mmap addresses - uint32_t store_id; -}; - -struct [[gnu::packed]] AssemblyStoreAssemblyDescriptor final -{ - uint32_t data_offset; - uint32_t data_size; - - uint32_t debug_data_offset; - uint32_t debug_data_size; - - uint32_t config_data_offset; - uint32_t config_data_size; -}; - -struct AssemblyStoreRuntimeData final -{ - uint8_t *data_start; - uint32_t assembly_count; - AssemblyStoreAssemblyDescriptor *assemblies; -}; - -struct AssemblyStoreSingleAssemblyRuntimeData final -{ - uint8_t *image_data; - uint8_t *debug_info_data; - uint8_t *config_data; - AssemblyStoreAssemblyDescriptor *descriptor; -}; - enum class MonoComponent : uint32_t { None = 0x00, @@ -209,7 +105,6 @@ struct ApplicationConfig bool instant_run_enabled; bool jni_add_native_method_registration_attribute_present; bool have_runtime_config_blob; - bool have_assembly_store; bool marshal_methods_enabled; uint8_t bound_exception_type; uint32_t package_naming_policy; @@ -217,7 +112,6 @@ struct ApplicationConfig uint32_t system_property_count; uint32_t number_of_assemblies_in_apk; uint32_t bundled_assembly_name_width; - uint32_t number_of_assembly_store_files; uint32_t number_of_dso_cache_entries; uint32_t android_runtime_jnienv_class_token; uint32_t jnienv_initialize_method_token; @@ -287,7 +181,6 @@ MONO_API MONO_API_EXPORT const TypeMapJava map_java[]; MONO_API MONO_API_EXPORT const xamarin::android::hash_t map_java_hashes[]; #endif -MONO_API MONO_API_EXPORT CompressedAssemblies compressed_assemblies; MONO_API MONO_API_EXPORT const ApplicationConfig application_config; MONO_API MONO_API_EXPORT const char* const app_environment_variables[]; MONO_API MONO_API_EXPORT const char* const app_system_properties[]; @@ -295,8 +188,6 @@ MONO_API MONO_API_EXPORT const char* const app_system_properties[]; MONO_API MONO_API_EXPORT const char* const mono_aot_mode_name; MONO_API MONO_API_EXPORT XamarinAndroidBundledAssembly bundled_assemblies[]; -MONO_API MONO_API_EXPORT AssemblyStoreSingleAssemblyRuntimeData assembly_store_bundled_assemblies[]; -MONO_API MONO_API_EXPORT AssemblyStoreRuntimeData assembly_stores[]; MONO_API MONO_API_EXPORT DSOCacheEntry dso_cache[]; diff --git a/src/monodroid/libstub/stub-assembly.cc b/src/monodroid/libstub/stub-assembly.cc new file mode 100644 index 00000000000..e11439c02af --- /dev/null +++ b/src/monodroid/libstub/stub-assembly.cc @@ -0,0 +1,2 @@ +extern "C" [[gnu::visibility("default")]] +bool assembly_blob = true;