diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs deleted file mode 100644 index bd6cc842f2d..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ /dev/null @@ -1,755 +0,0 @@ -// Copyright (C) 2011 Xamarin, Inc. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.IO; -using System.Reflection; -using System.Text; -using System.Text.RegularExpressions; - -using Microsoft.Build.Utilities; -using Microsoft.Build.Framework; - -using Java.Interop.Tools.Cecil; - -using ArchiveFileList = System.Collections.Generic.List<(string filePath, string archivePath)>; -using Mono.Cecil; -using Xamarin.Android.Tools; -using Xamarin.Tools.Zip; -using Microsoft.Android.Build.Tasks; - -namespace Xamarin.Android.Tasks -{ - public class BuildApk : AndroidTask - { - const string ArchiveAssembliesPath = "lib"; - const string ArchiveLibPath = "lib"; - - public override string TaskPrefix => "BLD"; - - public string AndroidNdkDirectory { get; set; } - - [Required] - public string ApkInputPath { get; set; } - - [Required] - public string ApkOutputPath { get; set; } - - [Required] - public string AppSharedLibrariesDir { get; set; } - - [Required] - public ITaskItem[] ResolvedUserAssemblies { get; set; } - - [Required] - public ITaskItem[] ResolvedFrameworkAssemblies { get; set; } - - public ITaskItem[] AdditionalNativeLibraryReferences { get; set; } - - public ITaskItem[] EmbeddedNativeLibraryAssemblies { get; set; } - - [Required] - public ITaskItem[] FrameworkNativeLibraries { get; set; } - - [Required] - public ITaskItem[] NativeLibraries { get; set; } - - [Required] - public ITaskItem[] ApplicationSharedLibraries { get; set; } - - public ITaskItem[] BundleNativeLibraries { get; set; } - - public ITaskItem[] TypeMappings { get; set; } - - [Required] - public ITaskItem [] DalvikClasses { get; set; } - - [Required] - public string [] SupportedAbis { get; set; } - - public bool EmbedAssemblies { get; set; } - - public bool BundleAssemblies { get; set; } - - public ITaskItem[] JavaSourceFiles { get; set; } - - public ITaskItem[] JavaLibraries { get; set; } - - public string[] DoNotPackageJavaLibraries { get; set; } - - public string [] ExcludeFiles { get; set; } - - public string [] IncludeFiles { get; set; } - - public string Debug { get; set; } - - public string AndroidSequencePointsMode { get; set; } - - public string TlsProvider { get; set; } - public string UncompressedFileExtensions { get; set; } - - // Make it required after https://github.com/xamarin/monodroid/pull/1094 is merged - //[Required] - public bool EnableCompression { get; set; } - - public bool IncludeWrapSh { get; set; } - - public string CheckedBuild { get; set; } - - public string RuntimeConfigBinFilePath { get; set; } - - public bool UseAssemblyStore { get; set; } - - public string ZipFlushFilesLimit { get; set; } - - public string ZipFlushSizeLimit { get; set; } - - public int ZipAlignmentPages { get; set; } = AndroidZipAlign.DefaultZipAlignment64Bit; - - [Required] - public string AndroidBinUtilsDirectory { get; set; } - - [Required] - public string IntermediateOutputPath { get; set; } - - [Required] - public string ProjectFullPath { get; set; } - - [Output] - public ITaskItem[] OutputFiles { get; set; } - - [Output] - public ITaskItem[] OutputApkFiles { get; set; } - - [Output] - public ITaskItem [] DSODirectoriesToDelete { get; set; } - - - bool _Debug { - get { - return string.Equals (Debug, "true", StringComparison.OrdinalIgnoreCase); - } - } - - SequencePointsMode sequencePointsMode = SequencePointsMode.None; - - public ITaskItem[] LibraryProjectJars { get; set; } - HashSet uncompressedFileExtensions; - - // Do not use trailing / in the path - public string RootPath { get; set; } = ""; - - public string DalvikPath { get; set; } = ""; - - protected virtual CompressionMethod UncompressedMethod => CompressionMethod.Store; - - protected virtual void FixupArchive (ZipArchiveFileListBuilder zip) { } - - List existingEntries = new List (); - - List excludePatterns = new List (); - - List includePatterns = new List (); - - void ExecuteWithAbi (DSOWrapperGenerator.Config dsoWrapperConfig, string [] supportedAbis, string apkInputPath, string apkOutputPath, bool debug, bool compress, IDictionary> compressedAssembliesInfo, string assemblyStoreApkName) - { - ArchiveFileList files = new ArchiveFileList (); - - using (var apk = new ZipArchiveFileListBuilder (apkOutputPath, File.Exists (apkOutputPath) ? FileMode.Open : FileMode.Create)) { - - // Add classes.dx - CompressionMethod dexCompressionMethod = GetCompressionMethod (".dex"); - foreach (var dex in DalvikClasses) { - string apkName = dex.GetMetadata ("ApkName"); - string dexPath = string.IsNullOrWhiteSpace (apkName) ? Path.GetFileName (dex.ItemSpec) : apkName; - AddFileToArchiveIfNewer (apk, dex.ItemSpec, DalvikPath + dexPath, compressionMethod: dexCompressionMethod); - apk.Flush (); - } - - if (EmbedAssemblies) { - AddAssemblies (dsoWrapperConfig, apk, debug, compress, compressedAssembliesInfo, assemblyStoreApkName); - apk.Flush (); - } - - AddRuntimeConfigBlob (dsoWrapperConfig, apk); - AddRuntimeLibraries (apk, supportedAbis); - apk.Flush(); - AddNativeLibraries (files, supportedAbis); - AddAdditionalNativeLibraries (files, supportedAbis); - - if (TypeMappings != null) { - foreach (ITaskItem typemap in TypeMappings) { - AddFileToArchiveIfNewer (apk, typemap.ItemSpec, RootPath + Path.GetFileName(typemap.ItemSpec), compressionMethod: UncompressedMethod); - } - } - - foreach (var file in files) { - var item = Path.Combine (file.archivePath.Replace (Path.DirectorySeparatorChar, '/')); - existingEntries.Remove (item); - CompressionMethod compressionMethod = GetCompressionMethod (file.filePath); - if (apk.SkipExistingFile (file.filePath, item, compressionMethod)) { - Log.LogDebugMessage ($"Skipping {file.filePath} as the archive file is up to date."); - continue; - } - Log.LogDebugMessage ("\tAdding {0}", file.filePath); - apk.AddFileAndFlush (file.filePath, item, compressionMethod: compressionMethod); - } - - var jarFiles = (JavaSourceFiles != null) ? JavaSourceFiles.Where (f => f.ItemSpec.EndsWith (".jar", StringComparison.OrdinalIgnoreCase)) : null; - if (jarFiles != null && JavaLibraries != null) - jarFiles = jarFiles.Concat (JavaLibraries); - else if (JavaLibraries != null) - jarFiles = JavaLibraries; - - var libraryProjectJars = MonoAndroidHelper.ExpandFiles (LibraryProjectJars) - .Where (jar => !MonoAndroidHelper.IsEmbeddedReferenceJar (jar)); - - var jarFilePaths = libraryProjectJars.Concat (jarFiles != null ? jarFiles.Select (j => j.ItemSpec) : Enumerable.Empty ()); - jarFilePaths = MonoAndroidHelper.DistinctFilesByContent (jarFilePaths); - - foreach (var jarFile in jarFilePaths) { - using (var stream = File.OpenRead (jarFile)) - using (var jar = ZipArchive.Open (stream)) { - foreach (var jarItem in jar) { - if (jarItem.IsDirectory) - continue; - var name = jarItem.FullName; - if (!PackagingUtils.CheckEntryForPackaging (name)) { - continue; - } - var path = RootPath + name; - existingEntries.Remove (path); - if (apk.SkipExistingEntry (jarItem, path)) { - Log.LogDebugMessage ($"Skipping {path} as the archive file is up to date."); - continue; - } - // check for ignored items - bool exclude = false; - bool forceInclude = false; - foreach (var include in includePatterns) { - if (include.IsMatch (path)) { - forceInclude = true; - break; - } - } - if (!forceInclude) { - foreach (var pattern in excludePatterns) { - if (pattern.IsMatch (path)) { - Log.LogDebugMessage ($"Ignoring jar entry '{name}' from '{Path.GetFileName (jarFile)}'. Filename matched the exclude pattern '{pattern}'."); - exclude = true; - break; - } - } - } - if (exclude) - continue; - if (string.Compare (Path.GetFileName (name), "AndroidManifest.xml", StringComparison.OrdinalIgnoreCase) == 0) { - Log.LogDebugMessage ("Ignoring jar entry {0} from {1}: the same file already exists in the apk", name, Path.GetFileName (jarFile)); - continue; - } - - apk.AddJavaEntryAndFlush (jarFile, jarItem.FullName, path); - } - } - } - FixupArchive (apk); - - OutputApkFiles = apk.ApkFiles.ToArray (); - - } - } - - public override bool RunTask () - { - Aot.TryGetSequencePointsMode (AndroidSequencePointsMode, out sequencePointsMode); - - var outputFiles = new List (); - uncompressedFileExtensions = new HashSet (StringComparer.OrdinalIgnoreCase); - foreach (string? e in UncompressedFileExtensions?.Split (new char [] { ';', ',' }, StringSplitOptions.RemoveEmptyEntries) ?? Array.Empty ()) { - string? ext = e?.Trim (); - if (String.IsNullOrEmpty (ext)) { - continue; - } - - if (ext[0] != '.') { - ext = $".{ext}"; - } - uncompressedFileExtensions.Add (ext); - } - - existingEntries.Clear (); - - foreach (var pattern in ExcludeFiles ?? Array.Empty ()) { - excludePatterns.Add (FileGlobToRegEx (pattern, RegexOptions.IgnoreCase | RegexOptions.Compiled)); - } - foreach (var pattern in IncludeFiles ?? Array.Empty ()) { - includePatterns.Add (FileGlobToRegEx (pattern, RegexOptions.IgnoreCase | RegexOptions.Compiled)); - } - - bool debug = _Debug; - bool compress = !debug && EnableCompression; - IDictionary> compressedAssembliesInfo = null; - - if (compress) { - string key = CompressedAssemblyInfo.GetKey (ProjectFullPath); - Log.LogDebugMessage ($"Retrieving assembly compression info with key '{key}'"); - compressedAssembliesInfo = BuildEngine4.UnregisterTaskObjectAssemblyLocal>> (key, RegisteredTaskObjectLifetime.Build); - if (compressedAssembliesInfo == null) - throw new InvalidOperationException ($"Assembly compression info not found for key '{key}'. Compression will not be performed."); - } - - DSOWrapperGenerator.Config dsoWrapperConfig = DSOWrapperGenerator.GetConfig (Log, AndroidBinUtilsDirectory, IntermediateOutputPath); - ExecuteWithAbi (dsoWrapperConfig, SupportedAbis, ApkInputPath, ApkOutputPath, debug, compress, compressedAssembliesInfo, assemblyStoreApkName: null); - outputFiles.Add (ApkOutputPath); - - OutputFiles = outputFiles.Select (a => new TaskItem (a)).ToArray (); - - Log.LogDebugTaskItems (" [Output] OutputFiles :", OutputFiles); - DSODirectoriesToDelete = DSOWrapperGenerator.GetDirectoriesToCleanUp (dsoWrapperConfig).Select (d => new TaskItem (d)).ToArray (); - - return !Log.HasLoggedErrors; - } - - static Regex FileGlobToRegEx (string fileGlob, RegexOptions options) - { - StringBuilder sb = new StringBuilder (); - foreach (char c in fileGlob) { - switch (c) { - case '*': sb.Append (".*"); - break; - case '?': sb.Append ("."); - break; - case '.': sb.Append (@"\."); - break; - default: sb.Append (c); - break; - } - } - return new Regex (sb.ToString (), options); - } - - void AddRuntimeConfigBlob (DSOWrapperGenerator.Config dsoWrapperConfig, ZipArchiveFileListBuilder apk) - { - // We will place rc.bin in the `lib` directory next to the blob, to make startup slightly faster, as we will find the config file right after we encounter - // our assembly store. Not only that, but also we'll be able to skip scanning the `base.apk` archive when split configs are enabled (which they are in 99% - // of cases these days, since AAB enforces that split). `base.apk` contains only ABI-agnostic file, while one of the split config files contains only - // ABI-specific data+code. - if (!String.IsNullOrEmpty (RuntimeConfigBinFilePath) && File.Exists (RuntimeConfigBinFilePath)) { - foreach (string abi in SupportedAbis) { - // Prefix it with `a` because bundletool sorts entries alphabetically, and this will place it right next to `assemblies.*.blob.so`, which is what we - // like since we can finish scanning the zip central directory earlier at startup. - string inArchivePath = MakeArchiveLibPath (abi, "libarc.bin.so"); - string wrappedSourcePath = DSOWrapperGenerator.WrapIt (Log, dsoWrapperConfig, MonoAndroidHelper.AbiToTargetArch (abi), RuntimeConfigBinFilePath, Path.GetFileName (inArchivePath)); - AddFileToArchiveIfNewer (apk, wrappedSourcePath, inArchivePath, compressionMethod: GetCompressionMethod (inArchivePath)); - } - } - } - - void AddAssemblies (DSOWrapperGenerator.Config dsoWrapperConfig, ZipArchiveFileListBuilder apk, bool debug, bool compress, IDictionary> compressedAssembliesInfo, string assemblyStoreApkName) - { - string sourcePath; - AssemblyCompression.AssemblyData compressedAssembly = null; - string compressedOutputDir = Path.GetFullPath (Path.Combine (Path.GetDirectoryName (ApkOutputPath), "..", "lz4")); - AssemblyStoreBuilder? storeBuilder = null; - - if (UseAssemblyStore) { - storeBuilder = new AssemblyStoreBuilder (Log); - } - - // Add user assemblies - AssemblyPackagingHelper.AddAssembliesFromCollection (Log, SupportedAbis, ResolvedUserAssemblies, DoAddAssembliesFromArchCollection); - - // Add framework assemblies - AssemblyPackagingHelper.AddAssembliesFromCollection (Log, SupportedAbis, ResolvedFrameworkAssemblies, DoAddAssembliesFromArchCollection); - - if (!UseAssemblyStore) { - return; - } - - Dictionary assemblyStorePaths = storeBuilder.Generate (AppSharedLibrariesDir); - - if (assemblyStorePaths.Count == 0) { - throw new InvalidOperationException ("Assembly store generator did not generate any stores"); - } - - if (assemblyStorePaths.Count != SupportedAbis.Length) { - throw new InvalidOperationException ("Internal error: assembly store did not generate store for each supported ABI"); - } - - string inArchivePath; - foreach (var kvp in assemblyStorePaths) { - string abi = MonoAndroidHelper.ArchToAbi (kvp.Key); - inArchivePath = MakeArchiveLibPath (abi, "lib" + Path.GetFileName (kvp.Value)); - string wrappedSourcePath = DSOWrapperGenerator.WrapIt (Log, dsoWrapperConfig, kvp.Key, kvp.Value, Path.GetFileName (inArchivePath)); - AddFileToArchiveIfNewer (apk, wrappedSourcePath, inArchivePath, GetCompressionMethod (inArchivePath)); - } - - void DoAddAssembliesFromArchCollection (TaskLoggingHelper log, AndroidTargetArch arch, ITaskItem assembly) - { - // In the "all assemblies are per-RID" world, assemblies, pdb and config are disguised as shared libraries (that is, - // their names end with the .so extension) so that Android allows us to put them in the `lib/{ARCH}` directory. - // For this reason, they have to be treated just like other .so files, as far as compression rules are concerned. - // Thus, we no longer just store them in the apk but we call the `GetCompressionMethod` method to find out whether - // or not we're supposed to compress .so files. - sourcePath = CompressAssembly (assembly); - if (UseAssemblyStore) { - storeBuilder.AddAssembly (sourcePath, assembly, includeDebugSymbols: debug); - return; - } - - // Add assembly - (string assemblyPath, string assemblyDirectory) = GetInArchiveAssemblyPath (assembly); - string wrappedSourcePath = DSOWrapperGenerator.WrapIt (Log, dsoWrapperConfig, arch, sourcePath, Path.GetFileName (assemblyPath)); - AddFileToArchiveIfNewer (apk, wrappedSourcePath, assemblyPath, compressionMethod: GetCompressionMethod (assemblyPath)); - - // Try to add config if exists - var config = Path.ChangeExtension (assembly.ItemSpec, "dll.config"); - AddAssemblyConfigEntry (dsoWrapperConfig, apk, arch, assemblyDirectory, config); - - // Try to add symbols if Debug - if (!debug) { - return; - } - - string symbols = Path.ChangeExtension (assembly.ItemSpec, "pdb"); - if (!File.Exists (symbols)) { - return; - } - - string archiveSymbolsPath = assemblyDirectory + MonoAndroidHelper.MakeDiscreteAssembliesEntryName (Path.GetFileName (symbols)); - string wrappedSymbolsPath = DSOWrapperGenerator.WrapIt (Log, dsoWrapperConfig, arch, symbols, Path.GetFileName (archiveSymbolsPath)); - AddFileToArchiveIfNewer ( - apk, - wrappedSymbolsPath, - archiveSymbolsPath, - compressionMethod: GetCompressionMethod (archiveSymbolsPath) - ); - } - - void EnsureCompressedAssemblyData (string sourcePath, uint descriptorIndex) - { - if (compressedAssembly == null) - compressedAssembly = new AssemblyCompression.AssemblyData (sourcePath, descriptorIndex); - else - compressedAssembly.SetData (sourcePath, descriptorIndex); - } - - string CompressAssembly (ITaskItem assembly) - { - if (!compress) { - return assembly.ItemSpec; - } - - return AssemblyCompression.Compress (Log, assembly, compressedAssembliesInfo, compressedOutputDir); - } - } - - bool AddFileToArchiveIfNewer (ZipArchiveFileListBuilder apk, string file, string inArchivePath, CompressionMethod compressionMethod = CompressionMethod.Default) - { - existingEntries.Remove (inArchivePath.Replace (Path.DirectorySeparatorChar, '/')); - if (apk.SkipExistingFile (file, inArchivePath, compressionMethod)) { - Log.LogDebugMessage ($"Skipping {file} as the archive file is up to date."); - return false; - } - Log.LogDebugMessage ($"Adding {file} as the archive file is out of date."); - apk.AddFileAndFlush (file, inArchivePath, compressionMethod: compressionMethod); - return true; - } - - void AddAssemblyConfigEntry (DSOWrapperGenerator.Config dsoWrapperConfig, ZipArchiveFileListBuilder apk, AndroidTargetArch arch, string assemblyPath, string configFile) - { - string inArchivePath = MonoAndroidHelper.MakeDiscreteAssembliesEntryName (assemblyPath + Path.GetFileName (configFile)); - existingEntries.Remove (inArchivePath); - - if (!File.Exists (configFile)) { - return; - } - - CompressionMethod compressionMethod = GetCompressionMethod (inArchivePath); - if (apk.SkipExistingFile (configFile, inArchivePath, compressionMethod)) { - Log.LogDebugMessage ($"Skipping {configFile} as the archive file is up to date."); - return; - } - - Log.LogDebugMessage ($"Adding {configFile} as the archive file is out of date."); - string wrappedConfigFile = DSOWrapperGenerator.WrapIt (Log, dsoWrapperConfig, arch, configFile, Path.GetFileName (inArchivePath)); - apk.AddFileAndFlush (wrappedConfigFile, inArchivePath, compressionMethod); - } - - /// - /// Returns the in-archive path for an assembly - /// - (string assemblyFilePath, string assemblyDirectoryPath) GetInArchiveAssemblyPath (ITaskItem assembly) - { - var parts = new List (); - - // The PrepareSatelliteAssemblies task takes care of properly setting `DestinationSubDirectory`, so we can just use it here. - string? subDirectory = assembly.GetMetadata ("DestinationSubDirectory")?.Replace ('\\', '/'); - if (string.IsNullOrEmpty (subDirectory)) { - throw new InvalidOperationException ($"Internal error: assembly '{assembly}' lacks the required `DestinationSubDirectory` metadata"); - } - - string assemblyName = Path.GetFileName (assembly.ItemSpec); - // For discrete assembly entries we need to treat assemblies specially. - // All of the assemblies have their names mangled so that the possibility to clash with "real" shared - // library names is minimized. All of the assembly entries will start with a special character: - // - // `_` - for regular assemblies (e.g. `_Mono.Android.dll.so`) - // `-` - for satellite assemblies (e.g. `-es-Mono.Android.dll.so`) - // - // Second of all, we need to treat satellite assemblies with even more care. - // If we encounter one of them, we will return the culture as part of the path transformed - // so that it forms a `-culture-` assembly file name prefix, not a `culture/` subdirectory. - // This is necessary because Android doesn't allow subdirectories in `lib/{ABI}/` - // - string[] subdirParts = subDirectory.TrimEnd ('/').Split ('/'); - if (subdirParts.Length == 1) { - // Not a satellite assembly - parts.Add (subDirectory); - parts.Add (MonoAndroidHelper.MakeDiscreteAssembliesEntryName (assemblyName)); - } else if (subdirParts.Length == 2) { - parts.Add (subdirParts[0]); - parts.Add (MonoAndroidHelper.MakeDiscreteAssembliesEntryName (assemblyName, subdirParts[1])); - } else { - throw new InvalidOperationException ($"Internal error: '{assembly}' `DestinationSubDirectory` metadata has too many components ({parts.Count} instead of 1 or 2)"); - } - - string assemblyFilePath = MonoAndroidHelper.MakeZipArchivePath (ArchiveAssembliesPath, parts); - return (assemblyFilePath, Path.GetDirectoryName (assemblyFilePath) + "/"); - } - - sealed class LibInfo - { - public string Path; - public string Link; - public string Abi; - public string ArchiveFileName; - public ITaskItem Item; - } - - CompressionMethod GetCompressionMethod (string fileName) - { - return uncompressedFileExtensions.Contains (Path.GetExtension (fileName)) ? UncompressedMethod : CompressionMethod.Default; - } - - void AddNativeLibraryToArchive (ZipArchiveFileListBuilder apk, string abi, string filesystemPath, string inArchiveFileName, ITaskItem taskItem) - { - string archivePath = MakeArchiveLibPath (abi, inArchiveFileName); - existingEntries.Remove (archivePath); - CompressionMethod compressionMethod = GetCompressionMethod (archivePath); - if (apk.SkipExistingFile (filesystemPath, archivePath, compressionMethod)) { - Log.LogDebugMessage ($"Skipping {filesystemPath} (APK path: {archivePath}) as it is up to date."); - return; - } - Log.LogDebugMessage ($"Adding native library: {filesystemPath} (APK path: {archivePath})"); - ELFHelper.AssertValidLibraryAlignment (Log, ZipAlignmentPages, filesystemPath, taskItem); - apk.AddFileAndFlush (filesystemPath, archivePath, compressionMethod); - } - - void AddRuntimeLibraries (ZipArchiveFileListBuilder apk, string [] supportedAbis) - { - foreach (var abi in supportedAbis) { - foreach (ITaskItem item in ApplicationSharedLibraries) { - if (String.Compare (abi, item.GetMetadata ("abi"), StringComparison.Ordinal) != 0) - continue; - AddNativeLibraryToArchive (apk, abi, item.ItemSpec, Path.GetFileName (item.ItemSpec), item); - } - } - } - - bool IsWrapperScript (string path, string link) - { - if (Path.DirectorySeparatorChar == '/') { - path = path.Replace ('\\', '/'); - } - - if (String.Compare (Path.GetFileName (path), "wrap.sh", StringComparison.Ordinal) == 0) { - return true; - } - - if (String.IsNullOrEmpty (link)) { - return false; - } - - if (Path.DirectorySeparatorChar == '/') { - link = link.Replace ('\\', '/'); - } - - return String.Compare (Path.GetFileName (link), "wrap.sh", StringComparison.Ordinal) == 0; - } - - bool IncludeNativeLibrary (ITaskItem item) - { - if (IncludeWrapSh) - return true; - - return !IsWrapperScript (item.ItemSpec, item.GetMetadata ("Link")); - } - - string GetArchiveFileName (ITaskItem item) - { - string archiveFileName = item.GetMetadata ("ArchiveFileName"); - if (!String.IsNullOrEmpty (archiveFileName)) - return archiveFileName; - - if (!IsWrapperScript (item.ItemSpec, item.GetMetadata ("Link"))) { - return null; - } - - return "wrap.sh"; - } - - private void AddNativeLibraries (ArchiveFileList files, string [] supportedAbis) - { - var frameworkLibs = FrameworkNativeLibraries.Select (v => new LibInfo { - Path = v.ItemSpec, - Link = v.GetMetadata ("Link"), - Abi = GetNativeLibraryAbi (v), - ArchiveFileName = GetArchiveFileName (v), - Item = v, - }); - - AddNativeLibraries (files, supportedAbis, frameworkLibs); - - var libs = NativeLibraries.Concat (BundleNativeLibraries ?? Enumerable.Empty ()) - .Where (v => IncludeNativeLibrary (v)) - .Select (v => new LibInfo { - Path = v.ItemSpec, - Link = v.GetMetadata ("Link"), - Abi = GetNativeLibraryAbi (v), - ArchiveFileName = GetArchiveFileName (v), - Item = v, - } - ); - - AddNativeLibraries (files, supportedAbis, libs); - - if (String.IsNullOrWhiteSpace (CheckedBuild)) - return; - - string mode = CheckedBuild; - string sanitizerName; - if (String.Compare ("asan", mode, StringComparison.Ordinal) == 0) { - sanitizerName = "asan"; - } else if (String.Compare ("ubsan", mode, StringComparison.Ordinal) == 0) { - sanitizerName = "ubsan_standalone"; - } else { - LogSanitizerWarning ($"Unknown checked build mode '{CheckedBuild}'"); - return; - } - - if (!IncludeWrapSh) { - LogSanitizerError ("Checked builds require the wrapper script to be packaged. Please set the `$(AndroidIncludeWrapSh)` MSBuild property to `true` in your project."); - return; - } - - if (!libs.Any (info => IsWrapperScript (info.Path, info.Link))) { - LogSanitizerError ($"Checked builds require the wrapper script to be packaged. Please add `wrap.sh` appropriate for the {CheckedBuild} checker to your project."); - return; - } - - NdkTools ndk = NdkTools.Create (AndroidNdkDirectory, logErrors: false, log: Log); - if (Log.HasLoggedErrors) { - return; // NdkTools.Create will log appropriate error - } - - string clangDir = ndk.GetClangDeviceLibraryPath (); - if (String.IsNullOrEmpty (clangDir)) { - LogSanitizerError ($"Unable to find the clang compiler directory. Is NDK installed?"); - return; - } - - foreach (string abi in supportedAbis) { - string clangAbi = MonoAndroidHelper.MapAndroidAbiToClang (abi); - if (String.IsNullOrEmpty (clangAbi)) { - LogSanitizerError ($"Unable to map Android ABI {abi} to clang ABI"); - return; - } - - string sanitizerLib = $"libclang_rt.{sanitizerName}-{clangAbi}-android.so"; - string sanitizerLibPath = Path.Combine (clangDir, sanitizerLib); - if (!File.Exists (sanitizerLibPath)) { - LogSanitizerError ($"Unable to find sanitizer runtime for the {CheckedBuild} checker at {sanitizerLibPath}"); - return; - } - - AddNativeLibrary (files, sanitizerLibPath, abi, sanitizerLib); - } - } - - string GetNativeLibraryAbi (ITaskItem lib) - { - // If Abi is explicitly specified, simply return it. - var lib_abi = AndroidRidAbiHelper.GetNativeLibraryAbi (lib); - - if (string.IsNullOrWhiteSpace (lib_abi)) { - Log.LogCodedError ("XA4301", lib.ItemSpec, 0, Properties.Resources.XA4301_ABI, lib.ItemSpec); - return null; - } - - return lib_abi; - } - - void AddNativeLibraries (ArchiveFileList files, string [] supportedAbis, System.Collections.Generic.IEnumerable libs) - { - if (libs.Any (lib => lib.Abi == null)) - Log.LogCodedWarning ( - "XA4301", - Properties.Resources.XA4301_ABI_Ignoring, - string.Join (", ", libs.Where (lib => lib.Abi == null).Select (lib => lib.Path))); - libs = libs.Where (lib => lib.Abi != null); - libs = libs.Where (lib => supportedAbis.Contains (lib.Abi)); - foreach (var info in libs) { - AddNativeLibrary (files, info.Path, info.Abi, info.ArchiveFileName, info.Item); - } - } - - private void AddAdditionalNativeLibraries (ArchiveFileList files, string [] supportedAbis) - { - if (AdditionalNativeLibraryReferences == null || !AdditionalNativeLibraryReferences.Any ()) - return; - - var libs = AdditionalNativeLibraryReferences - .Select (l => new LibInfo { - Path = l.ItemSpec, - Abi = AndroidRidAbiHelper.GetNativeLibraryAbi (l), - ArchiveFileName = l.GetMetadata ("ArchiveFileName"), - Item = l, - }); - - AddNativeLibraries (files, supportedAbis, libs); - } - - void AddNativeLibrary (ArchiveFileList files, string path, string abi, string archiveFileName, ITaskItem? taskItem = null) - { - string fileName = string.IsNullOrEmpty (archiveFileName) ? Path.GetFileName (path) : archiveFileName; - var item = (filePath: path, archivePath: MakeArchiveLibPath (abi, fileName)); - if (files.Any (x => x.archivePath == item.archivePath)) { - Log.LogCodedWarning ("XA4301", path, 0, Properties.Resources.XA4301, item.archivePath); - return; - } - - ELFHelper.AssertValidLibraryAlignment (Log, ZipAlignmentPages, path, taskItem); - if (!ELFHelper.IsEmptyAOTLibrary (Log, item.filePath)) { - files.Add (item); - } else { - Log.LogDebugMessage ($"{item.filePath} is an empty (no executable code) AOT assembly, not including it in the archive"); - } - } - - // This method is used only for internal warnings which will never be shown to the end user, therefore there's - // no need to use coded warnings. - void LogSanitizerWarning (string message) - { - Log.LogWarning (message); - } - - void LogSanitizerError (string message) - { - Log.LogError (message); - } - - static string MakeArchiveLibPath (string abi, string fileName) => MonoAndroidHelper.MakeZipArchivePath (ArchiveLibPath, abi, fileName); - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildArchive.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildArchive.cs index 7e23d4eeffe..db57c4eb989 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildArchive.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildArchive.cs @@ -12,7 +12,9 @@ namespace Xamarin.Android.Tasks; /// /// Takes a list of files and adds them to an APK archive. If the APK archive already -/// exists, files are only added if they were changed. +/// exists, files are only added if they were changed. Note *ALL* files to be in the final +/// APK must be passed in via @(FilesToAddToArchive). This task will determine any unchanged files +/// and skip them, as well as remove any existing files in the APK that are no longer required. /// public class BuildArchive : AndroidTask { @@ -34,34 +36,17 @@ public class BuildArchive : AndroidTask public string? ZipFlushSizeLimit { get; set; } - readonly HashSet uncompressedFileExtensions; - readonly CompressionMethod uncompressedMethod = CompressionMethod.Store; + HashSet? uncompressedFileExtensions; + HashSet UncompressedFileExtensionsSet => uncompressedFileExtensions ??= ParseUncompressedFileExtensions (); - public BuildArchive () - { - uncompressedFileExtensions = new HashSet (StringComparer.OrdinalIgnoreCase); - - foreach (var extension in UncompressedFileExtensions?.Split ([';', ','], StringSplitOptions.RemoveEmptyEntries) ?? []) { - var ext = extension.Trim (); - - if (string.IsNullOrEmpty (ext)) { - continue; - } - - if (ext [0] != '.') { - ext = $".{ext}"; - } - - uncompressedFileExtensions.Add (ext); - } + CompressionMethod uncompressedMethod = CompressionMethod.Store; + public override bool RunTask () + { // Nothing needs to be compressed with app bundles. BundleConfig.json specifies the final compression mode. if (string.Compare (AndroidPackageFormat, "aab", true) == 0) uncompressedMethod = CompressionMethod.Default; - } - public override bool RunTask () - { var refresh = true; // If we have an input apk but no output apk, copy it to the output @@ -251,6 +236,27 @@ CompressionMethod GetCompressionMethod (ITaskItem item) return result; } - return uncompressedFileExtensions.Contains (Path.GetExtension (item.ItemSpec)) ? uncompressedMethod : CompressionMethod.Default; + return UncompressedFileExtensionsSet.Contains (Path.GetExtension (item.ItemSpec)) ? uncompressedMethod : CompressionMethod.Default; + } + + HashSet ParseUncompressedFileExtensions () + { + var uncompressedFileExtensions = new HashSet (StringComparer.OrdinalIgnoreCase); + + foreach (var extension in UncompressedFileExtensions?.Split ([';', ','], StringSplitOptions.RemoveEmptyEntries) ?? []) { + var ext = extension.Trim (); + + if (string.IsNullOrEmpty (ext)) { + continue; + } + + if (ext [0] != '.') { + ext = $".{ext}"; + } + + uncompressedFileExtensions.Add (ext); + } + + return uncompressedFileExtensions; } } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/CollectAssemblyFilesForArchive.cs b/src/Xamarin.Android.Build.Tasks/Tasks/CollectAssemblyFilesForArchive.cs new file mode 100644 index 00000000000..20f3e7e15cb --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Tasks/CollectAssemblyFilesForArchive.cs @@ -0,0 +1,227 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.IO; +using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +/// +/// Collects managed assemblies to be added to the final archive. +/// +public class CollectAssemblyFilesForArchive : AndroidTask +{ + const string ArchiveAssembliesPath = "lib"; + const string ArchiveLibPath = "lib"; + + public override string TaskPrefix => "CAF"; + + [Required] + public string AndroidBinUtilsDirectory { get; set; } = ""; + + [Required] + public string ApkOutputPath { get; set; } = ""; + + [Required] + public string AppSharedLibrariesDir { get; set; } = ""; + + public bool EmbedAssemblies { get; set; } + + [Required] + public bool EnableCompression { get; set; } + + public bool IncludeDebugSymbols { get; set; } + + [Required] + public string IntermediateOutputPath { get; set; } = ""; + + [Required] + public string ProjectFullPath { get; set; } = ""; + + [Required] + public ITaskItem [] ResolvedFrameworkAssemblies { get; set; } = []; + + [Required] + public ITaskItem [] ResolvedUserAssemblies { get; set; } = []; + + [Required] + public string [] SupportedAbis { get; set; } = []; + + public bool UseAssemblyStore { get; set; } + + [Output] + public ITaskItem [] FilesToAddToArchive { get; set; } = []; + + public override bool RunTask () + { + // If we aren't embedding assemblies, we don't need to do anything + if (!EmbedAssemblies) + return !Log.HasLoggedErrors; + + var files = new PackageFileListBuilder (); + + DSOWrapperGenerator.Config dsoWrapperConfig = DSOWrapperGenerator.GetConfig (Log, AndroidBinUtilsDirectory, IntermediateOutputPath); + bool compress = !IncludeDebugSymbols && EnableCompression; + IDictionary>? compressedAssembliesInfo = null; + + if (compress) { + string key = CompressedAssemblyInfo.GetKey (ProjectFullPath); + Log.LogDebugMessage ($"Retrieving assembly compression info with key '{key}'"); + compressedAssembliesInfo = BuildEngine4.UnregisterTaskObjectAssemblyLocal>> (key, RegisteredTaskObjectLifetime.Build); + if (compressedAssembliesInfo == null) + throw new InvalidOperationException ($"Assembly compression info not found for key '{key}'. Compression will not be performed."); + } + + AddAssemblies (dsoWrapperConfig, files, IncludeDebugSymbols, compress, compressedAssembliesInfo, assemblyStoreApkName: null); + + FilesToAddToArchive = files.ToArray (); + + return !Log.HasLoggedErrors; + } + + void AddAssemblies (DSOWrapperGenerator.Config dsoWrapperConfig, PackageFileListBuilder files, bool debug, bool compress, IDictionary>? compressedAssembliesInfo, string? assemblyStoreApkName) + { + string compressedOutputDir = Path.GetFullPath (Path.Combine (Path.GetDirectoryName (ApkOutputPath), "..", "lz4")); + AssemblyStoreBuilder? storeBuilder = null; + + if (UseAssemblyStore) { + storeBuilder = new AssemblyStoreBuilder (Log); + } + + // Add user assemblies + AssemblyPackagingHelper.AddAssembliesFromCollection (Log, SupportedAbis, ResolvedUserAssemblies, (TaskLoggingHelper log, AndroidTargetArch arch, ITaskItem assembly) => DoAddAssembliesFromArchCollection (log, arch, assembly, dsoWrapperConfig, files, debug, compress, compressedAssembliesInfo, compressedOutputDir, storeBuilder)); + + // Add framework assemblies + AssemblyPackagingHelper.AddAssembliesFromCollection (Log, SupportedAbis, ResolvedFrameworkAssemblies, (TaskLoggingHelper log, AndroidTargetArch arch, ITaskItem assembly) => DoAddAssembliesFromArchCollection (log, arch, assembly, dsoWrapperConfig, files, debug, compress, compressedAssembliesInfo, compressedOutputDir, storeBuilder)); + + if (!UseAssemblyStore) { + return; + } + + Dictionary assemblyStorePaths = storeBuilder!.Generate (AppSharedLibrariesDir); + + if (assemblyStorePaths.Count == 0) { + throw new InvalidOperationException ("Assembly store generator did not generate any stores"); + } + + if (assemblyStorePaths.Count != SupportedAbis.Length) { + throw new InvalidOperationException ("Internal error: assembly store did not generate store for each supported ABI"); + } + + string inArchivePath; + foreach (var kvp in assemblyStorePaths) { + string abi = MonoAndroidHelper.ArchToAbi (kvp.Key); + inArchivePath = MakeArchiveLibPath (abi, "lib" + Path.GetFileName (kvp.Value)); + string wrappedSourcePath = DSOWrapperGenerator.WrapIt (Log, dsoWrapperConfig, kvp.Key, kvp.Value, Path.GetFileName (inArchivePath)); + files.AddItem (wrappedSourcePath, inArchivePath); + } + } + + void DoAddAssembliesFromArchCollection (TaskLoggingHelper log, AndroidTargetArch arch, ITaskItem assembly, DSOWrapperGenerator.Config dsoWrapperConfig, PackageFileListBuilder files, bool debug, bool compress, IDictionary>? compressedAssembliesInfo, string compressedOutputDir, AssemblyStoreBuilder? storeBuilder) + { + // In the "all assemblies are per-RID" world, assemblies, pdb and config are disguised as shared libraries (that is, + // their names end with the .so extension) so that Android allows us to put them in the `lib/{ARCH}` directory. + // For this reason, they have to be treated just like other .so files, as far as compression rules are concerned. + // Thus, we no longer just store them in the apk but we call the `GetCompressionMethod` method to find out whether + // or not we're supposed to compress .so files. + var sourcePath = CompressAssembly (assembly, compress, compressedAssembliesInfo, compressedOutputDir); + if (UseAssemblyStore) { + storeBuilder!.AddAssembly (sourcePath, assembly, includeDebugSymbols: debug); + return; + } + + // Add assembly + (string assemblyPath, string assemblyDirectory) = GetInArchiveAssemblyPath (assembly); + string wrappedSourcePath = DSOWrapperGenerator.WrapIt (Log, dsoWrapperConfig, arch, sourcePath, Path.GetFileName (assemblyPath)); + files.AddItem (wrappedSourcePath, assemblyPath); + + // Try to add config if exists + var config = Path.ChangeExtension (assembly.ItemSpec, "dll.config"); + AddAssemblyConfigEntry (dsoWrapperConfig, files, arch, assemblyDirectory, config); + + // Try to add symbols if Debug + if (!debug) { + return; + } + + string symbols = Path.ChangeExtension (assembly.ItemSpec, "pdb"); + if (!File.Exists (symbols)) { + return; + } + + string archiveSymbolsPath = assemblyDirectory + MonoAndroidHelper.MakeDiscreteAssembliesEntryName (Path.GetFileName (symbols)); + string wrappedSymbolsPath = DSOWrapperGenerator.WrapIt (Log, dsoWrapperConfig, arch, symbols, Path.GetFileName (archiveSymbolsPath)); + files.AddItem (wrappedSymbolsPath, archiveSymbolsPath); + } + + /// + /// Returns the in-archive path for an assembly + /// + (string assemblyFilePath, string assemblyDirectoryPath) GetInArchiveAssemblyPath (ITaskItem assembly) + { + var parts = new List (); + + // The PrepareSatelliteAssemblies task takes care of properly setting `DestinationSubDirectory`, so we can just use it here. + string? subDirectory = assembly.GetMetadata ("DestinationSubDirectory")?.Replace ('\\', '/'); + if (string.IsNullOrEmpty (subDirectory)) { + throw new InvalidOperationException ($"Internal error: assembly '{assembly}' lacks the required `DestinationSubDirectory` metadata"); + } + + string assemblyName = Path.GetFileName (assembly.ItemSpec); + // For discrete assembly entries we need to treat assemblies specially. + // All of the assemblies have their names mangled so that the possibility to clash with "real" shared + // library names is minimized. All of the assembly entries will start with a special character: + // + // `_` - for regular assemblies (e.g. `_Mono.Android.dll.so`) + // `-` - for satellite assemblies (e.g. `-es-Mono.Android.dll.so`) + // + // Second of all, we need to treat satellite assemblies with even more care. + // If we encounter one of them, we will return the culture as part of the path transformed + // so that it forms a `-culture-` assembly file name prefix, not a `culture/` subdirectory. + // This is necessary because Android doesn't allow subdirectories in `lib/{ABI}/` + // + string [] subdirParts = subDirectory!.TrimEnd ('/').Split ('/'); + if (subdirParts.Length == 1) { + // Not a satellite assembly + parts.Add (subDirectory); + parts.Add (MonoAndroidHelper.MakeDiscreteAssembliesEntryName (assemblyName)); + } else if (subdirParts.Length == 2) { + parts.Add (subdirParts [0]); + parts.Add (MonoAndroidHelper.MakeDiscreteAssembliesEntryName (assemblyName, subdirParts [1])); + } else { + throw new InvalidOperationException ($"Internal error: '{assembly}' `DestinationSubDirectory` metadata has too many components ({parts.Count} instead of 1 or 2)"); + } + + string assemblyFilePath = MonoAndroidHelper.MakeZipArchivePath (ArchiveAssembliesPath, parts); + return (assemblyFilePath, Path.GetDirectoryName (assemblyFilePath) + "/"); + } + + void AddAssemblyConfigEntry (DSOWrapperGenerator.Config dsoWrapperConfig, PackageFileListBuilder files, AndroidTargetArch arch, string assemblyPath, string configFile) + { + string inArchivePath = MonoAndroidHelper.MakeDiscreteAssembliesEntryName (assemblyPath + Path.GetFileName (configFile)); + + if (!File.Exists (configFile)) { + return; + } + + string wrappedConfigFile = DSOWrapperGenerator.WrapIt (Log, dsoWrapperConfig, arch, configFile, Path.GetFileName (inArchivePath)); + + files.AddItem (wrappedConfigFile, inArchivePath); + } + + string CompressAssembly (ITaskItem assembly, bool compress, IDictionary>? compressedAssembliesInfo, string compressedOutputDir) + { + if (!compress) { + return assembly.ItemSpec; + } + + // NRT: compressedAssembliesInfo is guaranteed to be non-null if compress is true + return AssemblyCompression.Compress (Log, assembly, compressedAssembliesInfo!, compressedOutputDir); + } + + static string MakeArchiveLibPath (string abi, string fileName) => MonoAndroidHelper.MakeZipArchivePath (ArchiveLibPath, abi, fileName); +} diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/CollectDalvikFilesForArchive.cs b/src/Xamarin.Android.Build.Tasks/Tasks/CollectDalvikFilesForArchive.cs new file mode 100644 index 00000000000..3ad49ddc407 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Tasks/CollectDalvikFilesForArchive.cs @@ -0,0 +1,41 @@ +#nullable enable + +using System; +using System.IO; +using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Framework; + +namespace Xamarin.Android.Tasks; + +/// +/// Collects Dalvik classes to be added to the final archive. +/// +public class CollectDalvikFilesForArchive : AndroidTask +{ + public override string TaskPrefix => "CDF"; + + public string AndroidPackageFormat { get; set; } = ""; + + [Required] + public ITaskItem [] DalvikClasses { get; set; } = []; + + [Output] + public ITaskItem [] FilesToAddToArchive { get; set; } = []; + + public override bool RunTask () + { + var dalvikPath = AndroidPackageFormat.Equals ("aab", StringComparison.InvariantCultureIgnoreCase) ? "dex/" : ""; + var files = new PackageFileListBuilder (); + + foreach (var dex in DalvikClasses) { + var apkName = dex.GetMetadata ("ApkName"); + var dexPath = string.IsNullOrWhiteSpace (apkName) ? Path.GetFileName (dex.ItemSpec) : apkName; + + files.AddItem (dex.ItemSpec, dalvikPath + dexPath); + } + + FilesToAddToArchive = files.ToArray (); + + return !Log.HasLoggedErrors; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/CollectJarContentFilesForArchive.cs b/src/Xamarin.Android.Build.Tasks/Tasks/CollectJarContentFilesForArchive.cs new file mode 100644 index 00000000000..33ce886d810 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Tasks/CollectJarContentFilesForArchive.cs @@ -0,0 +1,146 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Framework; +using Xamarin.Tools.Zip; + +namespace Xamarin.Android.Tasks; + +/// +/// Collects extra files from inside Jar libraries to be added to the final archive. +/// +public class CollectJarContentFilesForArchive : AndroidTask +{ + public override string TaskPrefix => "CJC"; + + public string AndroidPackageFormat { get; set; } = ""; + + public string [] ExcludeFiles { get; set; } = []; + + public string [] IncludeFiles { get; set; } = []; + + public ITaskItem [] JavaSourceFiles { get; set; } = []; + + public ITaskItem [] JavaLibraries { get; set; } = []; + + public ITaskItem [] LibraryProjectJars { get; set; } = []; + + readonly List excludePatterns = new List (); + + readonly List includePatterns = new List (); + + [Output] + public ITaskItem [] FilesToAddToArchive { get; set; } = []; + + public override bool RunTask () + { + var rootPath = AndroidPackageFormat.Equals ("aab", StringComparison.InvariantCultureIgnoreCase) ? "root/" : ""; + + foreach (var pattern in ExcludeFiles) { + excludePatterns.Add (FileGlobToRegEx (pattern, RegexOptions.IgnoreCase | RegexOptions.Compiled)); + } + + foreach (var pattern in IncludeFiles) { + includePatterns.Add (FileGlobToRegEx (pattern, RegexOptions.IgnoreCase | RegexOptions.Compiled)); + } + + // Grab distinct .jar files from: + // - JavaSourceFiles + // - JavaLibraries + // - LibraryProjectJars + var jarFiles = (JavaSourceFiles != null) ? JavaSourceFiles.Where (f => f.ItemSpec.EndsWith (".jar", StringComparison.OrdinalIgnoreCase)) : null; + + if (jarFiles != null && JavaLibraries != null) + jarFiles = jarFiles.Concat (JavaLibraries); + else if (JavaLibraries != null) + jarFiles = JavaLibraries; + + var libraryProjectJars = MonoAndroidHelper.ExpandFiles (LibraryProjectJars) + .Where (jar => !MonoAndroidHelper.IsEmbeddedReferenceJar (jar)); + + var jarFilePaths = libraryProjectJars.Concat (jarFiles != null ? jarFiles.Select (j => j.ItemSpec) : Enumerable.Empty ()); + jarFilePaths = MonoAndroidHelper.DistinctFilesByContent (jarFilePaths); + + // Find files in the .jar files that match our include patterns to be added to the archive + var files = new PackageFileListBuilder (); + + foreach (var jarFile in jarFilePaths) { + using (var stream = File.OpenRead (jarFile)) + using (var jar = ZipArchive.Open (stream)) { + foreach (var jarItem in jar) { + if (jarItem.IsDirectory) + continue; + var name = jarItem.FullName; + if (!PackagingUtils.CheckEntryForPackaging (name)) { + continue; + } + + var path = rootPath + name; + + // check for ignored items + bool exclude = false; + bool forceInclude = false; + foreach (var include in includePatterns) { + if (include.IsMatch (path)) { + forceInclude = true; + break; + } + } + + if (!forceInclude) { + foreach (var pattern in excludePatterns) { + if (pattern.IsMatch (path)) { + Log.LogDebugMessage ($"Ignoring jar entry '{name}' from '{Path.GetFileName (jarFile)}'. Filename matched the exclude pattern '{pattern}'."); + exclude = true; + break; + } + } + } + + if (exclude) + continue; + + if (string.Compare (Path.GetFileName (name), "AndroidManifest.xml", StringComparison.OrdinalIgnoreCase) == 0) { + Log.LogDebugMessage ("Ignoring jar entry {0} from {1}: the same file already exists in the apk", name, Path.GetFileName (jarFile)); + continue; + } + + // An item's ItemSpec should be unique so use both the jar file name and the entry name + files.AddItem ($"{jarFile}#{jarItem.FullName}", path, jarItem.FullName); + } + } + } + + FilesToAddToArchive = files.ToArray (); + + return !Log.HasLoggedErrors; + } + + static Regex FileGlobToRegEx (string fileGlob, RegexOptions options) + { + StringBuilder sb = new StringBuilder (); + foreach (char c in fileGlob) { + switch (c) { + case '*': + sb.Append (".*"); + break; + case '?': + sb.Append ("."); + break; + case '.': + sb.Append (@"\."); + break; + default: + sb.Append (c); + break; + } + } + return new Regex (sb.ToString (), options); + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/CollectNativeFilesForArchive.cs b/src/Xamarin.Android.Build.Tasks/Tasks/CollectNativeFilesForArchive.cs new file mode 100644 index 00000000000..78b64397882 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Tasks/CollectNativeFilesForArchive.cs @@ -0,0 +1,328 @@ +#nullable enable +// Copyright (C) 2011 Xamarin, Inc. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.IO; + +using Microsoft.Build.Utilities; +using Microsoft.Build.Framework; + +using ArchiveFileList = System.Collections.Generic.List<(string filePath, string archivePath)>; +using Microsoft.Android.Build.Tasks; + +namespace Xamarin.Android.Tasks; + +/// +/// Collects native libraries to be added to the final archive. +/// +public class CollectNativeFilesForArchive : AndroidTask +{ + const string ArchiveLibPath = "lib"; + + public override string TaskPrefix => "CNF"; + + public string AndroidNdkDirectory { get; set; } = ""; + + [Required] + public string ApkOutputPath { get; set; } = ""; + + public ITaskItem[] AdditionalNativeLibraryReferences { get; set; } = []; + + [Required] + public ITaskItem[] FrameworkNativeLibraries { get; set; } = []; + + [Required] + public ITaskItem[] NativeLibraries { get; set; } = []; + + [Required] + public ITaskItem[] ApplicationSharedLibraries { get; set; } = []; + + public ITaskItem[] BundleNativeLibraries { get; set; } = []; + + [Required] + public string [] SupportedAbis { get; set; } = []; + + public bool IncludeWrapSh { get; set; } + + public string CheckedBuild { get; set; } = ""; + + public int ZipAlignmentPages { get; set; } = AndroidZipAlign.DefaultZipAlignment64Bit; + + [Required] + public string AndroidBinUtilsDirectory { get; set; } = ""; + + [Required] + public string IntermediateOutputPath { get; set; } = ""; + + [Output] + public ITaskItem[] OutputFiles { get; set; } = []; + + [Output] + public ITaskItem[] FilesToAddToArchive { get; set; } = []; + + [Output] + public ITaskItem [] DSODirectoriesToDelete { get; set; } = []; + + public override bool RunTask () + { + var apk = new PackageFileListBuilder (); + var dsoWrapperConfig = DSOWrapperGenerator.GetConfig (Log, AndroidBinUtilsDirectory, IntermediateOutputPath); + + var outputFiles = new List { + ApkOutputPath + }; + + var files = new ArchiveFileList (); + + AddRuntimeLibraries (apk, SupportedAbis); + AddNativeLibraries (files, SupportedAbis); + AddAdditionalNativeLibraries (files, SupportedAbis); + + foreach (var file in files) { + var item = Path.Combine (file.archivePath.Replace (Path.DirectorySeparatorChar, '/')); + Log.LogDebugMessage ("\tAdding {0}", file.filePath); + apk.AddItem (file.filePath, item); + } + + // Task output parameters + FilesToAddToArchive = apk.ToArray (); + OutputFiles = outputFiles.Select (a => new TaskItem (a)).ToArray (); + DSODirectoriesToDelete = DSOWrapperGenerator.GetDirectoriesToCleanUp (dsoWrapperConfig).Select (d => new TaskItem (d)).ToArray (); + + return !Log.HasLoggedErrors; + } + + void AddNativeLibraryToArchive (PackageFileListBuilder apk, string abi, string filesystemPath, string inArchiveFileName, ITaskItem taskItem) + { + string archivePath = MakeArchiveLibPath (abi, inArchiveFileName); + Log.LogDebugMessage ($"Adding native library: {filesystemPath} (APK path: {archivePath})"); + ELFHelper.AssertValidLibraryAlignment (Log, ZipAlignmentPages, filesystemPath, taskItem); + apk.AddItem (filesystemPath, archivePath); + } + + void AddRuntimeLibraries (PackageFileListBuilder apk, string [] supportedAbis) + { + foreach (var abi in supportedAbis) { + foreach (ITaskItem item in ApplicationSharedLibraries) { + if (string.Compare (abi, item.GetMetadata ("abi"), StringComparison.Ordinal) != 0) + continue; + AddNativeLibraryToArchive (apk, abi, item.ItemSpec, Path.GetFileName (item.ItemSpec), item); + } + } + } + + bool IsWrapperScript (string path, string? link) + { + if (Path.DirectorySeparatorChar == '/') { + path = path.Replace ('\\', '/'); + } + + if (string.Compare (Path.GetFileName (path), "wrap.sh", StringComparison.Ordinal) == 0) { + return true; + } + + if (string.IsNullOrEmpty (link)) { + return false; + } + + if (Path.DirectorySeparatorChar == '/') { + link = link!.Replace ('\\', '/'); + } + + return string.Compare (Path.GetFileName (link), "wrap.sh", StringComparison.Ordinal) == 0; + } + + bool IncludeNativeLibrary (ITaskItem item) + { + if (IncludeWrapSh) + return true; + + return !IsWrapperScript (item.ItemSpec, item.GetMetadata ("Link")); + } + + string? GetArchiveFileName (ITaskItem item) + { + string archiveFileName = item.GetMetadata ("ArchiveFileName"); + if (!string.IsNullOrEmpty (archiveFileName)) + return archiveFileName; + + if (!IsWrapperScript (item.ItemSpec, item.GetMetadata ("Link"))) { + return null; + } + + return "wrap.sh"; + } + + void AddNativeLibraries (ArchiveFileList files, string [] supportedAbis) + { + var frameworkLibs = FrameworkNativeLibraries.Select (v => new LibInfo ( + path: v.ItemSpec, + link: v.GetMetadata ("Link"), + abi: GetNativeLibraryAbi (v), + archiveFileName: GetArchiveFileName (v), + item: v + )); + + AddNativeLibraries (files, supportedAbis, frameworkLibs); + + var libs = NativeLibraries.Concat (BundleNativeLibraries ?? Enumerable.Empty ()) + .Where (v => IncludeNativeLibrary (v)) + .Select (v => new LibInfo ( + path: v.ItemSpec, + link: v.GetMetadata ("Link"), + abi: GetNativeLibraryAbi (v), + archiveFileName: GetArchiveFileName (v), + item: v + ) + ); + + AddNativeLibraries (files, supportedAbis, libs); + + if (string.IsNullOrWhiteSpace (CheckedBuild)) + return; + + string mode = CheckedBuild; + string sanitizerName; + if (string.Compare ("asan", mode, StringComparison.Ordinal) == 0) { + sanitizerName = "asan"; + } else if (string.Compare ("ubsan", mode, StringComparison.Ordinal) == 0) { + sanitizerName = "ubsan_standalone"; + } else { + LogSanitizerWarning ($"Unknown checked build mode '{CheckedBuild}'"); + return; + } + + if (!IncludeWrapSh) { + LogSanitizerError ("Checked builds require the wrapper script to be packaged. Please set the `$(AndroidIncludeWrapSh)` MSBuild property to `true` in your project."); + return; + } + + if (!libs.Any (info => IsWrapperScript (info.Path, info.Link))) { + LogSanitizerError ($"Checked builds require the wrapper script to be packaged. Please add `wrap.sh` appropriate for the {CheckedBuild} checker to your project."); + return; + } + + NdkTools ndk = NdkTools.Create (AndroidNdkDirectory, logErrors: false, log: Log); + if (Log.HasLoggedErrors) { + return; // NdkTools.Create will log appropriate error + } + + string clangDir = ndk.GetClangDeviceLibraryPath (); + if (string.IsNullOrEmpty (clangDir)) { + LogSanitizerError ($"Unable to find the clang compiler directory. Is NDK installed?"); + return; + } + + foreach (string abi in supportedAbis) { + string clangAbi = MonoAndroidHelper.MapAndroidAbiToClang (abi); + if (string.IsNullOrEmpty (clangAbi)) { + LogSanitizerError ($"Unable to map Android ABI {abi} to clang ABI"); + return; + } + + string sanitizerLib = $"libclang_rt.{sanitizerName}-{clangAbi}-android.so"; + string sanitizerLibPath = Path.Combine (clangDir, sanitizerLib); + if (!File.Exists (sanitizerLibPath)) { + LogSanitizerError ($"Unable to find sanitizer runtime for the {CheckedBuild} checker at {sanitizerLibPath}"); + return; + } + + AddNativeLibrary (files, sanitizerLibPath, abi, sanitizerLib); + } + } + + string? GetNativeLibraryAbi (ITaskItem lib) + { + // If Abi is explicitly specified, simply return it. + var lib_abi = AndroidRidAbiHelper.GetNativeLibraryAbi (lib); + + if (string.IsNullOrWhiteSpace (lib_abi)) { + Log.LogCodedError ("XA4301", lib.ItemSpec, 0, Properties.Resources.XA4301_ABI, lib.ItemSpec); + return null; + } + + return lib_abi; + } + + void AddNativeLibraries (ArchiveFileList files, string [] supportedAbis, System.Collections.Generic.IEnumerable libs) + { + if (libs.Any (lib => lib.Abi == null)) + Log.LogCodedWarning ( + "XA4301", + Properties.Resources.XA4301_ABI_Ignoring, + string.Join (", ", libs.Where (lib => lib.Abi == null).Select (lib => lib.Path))); + libs = libs.Where (lib => lib.Abi != null); + libs = libs.Where (lib => supportedAbis.Contains (lib.Abi)); + foreach (var info in libs) { + AddNativeLibrary (files, info.Path, info.Abi!, info.ArchiveFileName, info.Item); + } + } + + void AddAdditionalNativeLibraries (ArchiveFileList files, string [] supportedAbis) + { + if (AdditionalNativeLibraryReferences == null || !AdditionalNativeLibraryReferences.Any ()) + return; + + var libs = AdditionalNativeLibraryReferences + .Select (l => new LibInfo ( + path: l.ItemSpec, + link: null, + abi: AndroidRidAbiHelper.GetNativeLibraryAbi (l), + archiveFileName: l.GetMetadata ("ArchiveFileName"), + item: l + )); + + AddNativeLibraries (files, supportedAbis, libs); + } + + void AddNativeLibrary (ArchiveFileList files, string path, string abi, string? archiveFileName, ITaskItem? taskItem = null) + { + string fileName = string.IsNullOrEmpty (archiveFileName) ? Path.GetFileName (path) : archiveFileName!; + var item = (filePath: path, archivePath: MakeArchiveLibPath (abi, fileName)); + if (files.Any (x => x.archivePath == item.archivePath)) { + Log.LogCodedWarning ("XA4301", path, 0, Properties.Resources.XA4301, item.archivePath); + return; + } + + ELFHelper.AssertValidLibraryAlignment (Log, ZipAlignmentPages, path, taskItem); + if (!ELFHelper.IsEmptyAOTLibrary (Log, item.filePath)) { + files.Add (item); + } else { + Log.LogDebugMessage ($"{item.filePath} is an empty (no executable code) AOT assembly, not including it in the archive"); + } + } + + // This method is used only for internal warnings which will never be shown to the end user, therefore there's + // no need to use coded warnings. (They are only used when the internal property $(_AndroidCheckedBuild) is set.) + void LogSanitizerWarning (string message) + { + Log.LogWarning (message); + } + + void LogSanitizerError (string message) + { + Log.LogError (message); + } + + static string MakeArchiveLibPath (string abi, string fileName) => MonoAndroidHelper.MakeZipArchivePath (ArchiveLibPath, abi, fileName); + + sealed class LibInfo + { + public string Path; + public string? Link; + public string? Abi; + public string? ArchiveFileName; + public ITaskItem Item; + + public LibInfo (string path, string? link, string? abi, string? archiveFileName, ITaskItem item) + { + Path = path; + Link = link; + Abi = abi; + ArchiveFileName = archiveFileName; + Item = item; + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/CollectRuntimeConfigFilesForArchive.cs b/src/Xamarin.Android.Build.Tasks/Tasks/CollectRuntimeConfigFilesForArchive.cs new file mode 100644 index 00000000000..1aa86a4c907 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Tasks/CollectRuntimeConfigFilesForArchive.cs @@ -0,0 +1,58 @@ +#nullable enable + +using System; +using System.IO; +using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Framework; + +namespace Xamarin.Android.Tasks; + +/// +/// Collects rc.bin to be added to the final archive. +/// +public class CollectRuntimeConfigFilesForArchive : AndroidTask +{ + const string ArchiveLibPath = "lib"; + + public override string TaskPrefix => "CRF"; + + [Required] + public string AndroidBinUtilsDirectory { get; set; } = ""; + + [Required] + public string IntermediateOutputPath { get; set; } = ""; + + public string RuntimeConfigBinFilePath { get; set; } = ""; + + [Required] + public string [] SupportedAbis { get; set; } = []; + + [Output] + public ITaskItem [] FilesToAddToArchive { get; set; } = []; + + public override bool RunTask () + { + var files = new PackageFileListBuilder (); + var dsoWrapperConfig = DSOWrapperGenerator.GetConfig (Log, AndroidBinUtilsDirectory, IntermediateOutputPath); + + // We will place rc.bin in the `lib` directory next to the blob, to make startup slightly faster, as we will find the config file right after we encounter + // our assembly store. Not only that, but also we'll be able to skip scanning the `base.apk` archive when split configs are enabled (which they are in 99% + // of cases these days, since AAB enforces that split). `base.apk` contains only ABI-agnostic file, while one of the split config files contains only + // ABI-specific data+code. + if (!string.IsNullOrEmpty (RuntimeConfigBinFilePath) && File.Exists (RuntimeConfigBinFilePath)) { + foreach (var abi in SupportedAbis) { + // Prefix it with `a` because bundletool sorts entries alphabetically, and this will place it right next to `assemblies.*.blob.so`, which is what we + // like since we can finish scanning the zip central directory earlier at startup. + var inArchivePath = MakeArchiveLibPath (abi, "libarc.bin.so"); + var wrappedSourcePath = DSOWrapperGenerator.WrapIt (Log, dsoWrapperConfig, MonoAndroidHelper.AbiToTargetArch (abi), RuntimeConfigBinFilePath, Path.GetFileName (inArchivePath)); + files.AddItem (wrappedSourcePath, inArchivePath); + } + } + + FilesToAddToArchive = files.ToArray (); + + return !Log.HasLoggedErrors; + } + + static string MakeArchiveLibPath (string abi, string fileName) => MonoAndroidHelper.MakeZipArchivePath (ArchiveLibPath, abi, fileName); +} diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/CollectTypeMapFilesForArchive.cs b/src/Xamarin.Android.Build.Tasks/Tasks/CollectTypeMapFilesForArchive.cs new file mode 100644 index 00000000000..4dc18a8b545 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Tasks/CollectTypeMapFilesForArchive.cs @@ -0,0 +1,39 @@ +#nullable enable + +using System; +using System.IO; +using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Framework; + +namespace Xamarin.Android.Tasks; + +/// +/// Collects TypeMap to be added to the final archive. +/// +public class CollectTypeMapFilesForArchive : AndroidTask +{ + public override string TaskPrefix => "CTM"; + + public string AndroidPackageFormat { get; set; } = ""; + + public ITaskItem [] TypeMappings { get; set; } = []; + + [Output] + public ITaskItem [] FilesToAddToArchive { get; set; } = []; + + public override bool RunTask () + { + if (TypeMappings.Length == 0) + return true; + + var rootPath = AndroidPackageFormat.Equals ("aab", StringComparison.InvariantCultureIgnoreCase) ? "root/" : ""; + var files = new PackageFileListBuilder (); + + foreach (var tm in TypeMappings) + files.AddItem (tm.ItemSpec, rootPath + Path.GetFileName (tm.ItemSpec)); + + FilesToAddToArchive = files.ToArray (); + + return !Log.HasLoggedErrors; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/PackageFileListBuilder.cs b/src/Xamarin.Android.Build.Tasks/Utilities/PackageFileListBuilder.cs new file mode 100644 index 00000000000..a10b2facf82 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/PackageFileListBuilder.cs @@ -0,0 +1,26 @@ +#nullable enable + +using System; +using System.Collections.Generic; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Xamarin.Android.Tasks; + +class PackageFileListBuilder +{ + public List Items { get; } = []; + + public void AddItem (string filepath, string archivePath, string? javaEntryName = null) + { + var item = new TaskItem (filepath); + item.SetMetadata ("ArchivePath", archivePath); + + if (javaEntryName is not null) + item.SetMetadata ("JavaArchiveEntry", javaEntryName); + + Items.Add (item); + } + + public ITaskItem [] ToArray () => Items.ToArray (); +} diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index a42b16360d4..af28875fe64 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -30,7 +30,6 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. - @@ -41,8 +40,14 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. - + + + + + + + @@ -2076,15 +2081,11 @@ because xbuild doesn't support framework reference assemblies. Condition="'$(EmbedAssembliesIntoApk)' == 'True'"> - <_AndroidCreatePackagePerAbi>$(AndroidCreatePackagePerAbi) <_ApkOutputPath>$(ApkFileIntermediate) - <_AndroidCreatePackagePerAbi>False <_ApkOutputPath>$(_BaseZipIntermediate) - <_RootPath>root/ - <_DalvikPath>dex/ @@ -2093,48 +2094,64 @@ because xbuild doesn't support framework reference assemblies. also need to have the args added to Xamarin.Android.Common.Debugging.targets in monodroid. --> - + + + + + + + + + + + + + + + + + IntermediateOutputPath="$(IntermediateOutputPath)"> - + - + - <_AndroidCreatePackagePerAbi>$(AndroidCreatePackagePerAbi) <_ApkOutputPath>$(ApkFileIntermediate) @@ -2196,11 +2212,7 @@ because xbuild doesn't support framework reference assemblies. - <_AndroidCreatePackagePerAbi>False <_ApkOutputPath>$(_BaseZipIntermediate) - <_RootPath>root/ - <_DalvikPath>dex/ - <_BundleAssemblies>$(BundleAssemblies) <_BundleNativeLibraries>$(_BundleResultNativeLibraries) @@ -2210,48 +2222,69 @@ because xbuild doesn't support framework reference assemblies. - + + + + + + + + + + + + + + + + + + + + + - - + FrameworkNativeLibraries="@(FrameworkNativeLibrary)" + NativeLibraries="@(_NativeLibraries)" + ApplicationSharedLibraries="@(_ApplicationSharedLibrary)" + AdditionalNativeLibraryReferences="@(_AdditionalNativeLibraryReferences)" + SupportedAbis="@(_BuildTargetAbis)" + IncludeWrapSh="$(AndroidIncludeWrapSh)" + CheckedBuild="$(_AndroidCheckedBuild)" + AndroidBinUtilsDirectory="$(AndroidBinUtilsDirectory)" + IntermediateOutputPath="$(IntermediateOutputPath)"> + + - +