LLVM IR code generator refactoring and updates

src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs

@@ -74,16 +74,22 @@ void GenerateCompressedAssemblySources ()
 
 			void Generate (IDictionary<string, CompressedAssemblyInfo> dict)
 			{
-				var llvmAsmgen = new CompressedAssembliesNativeAssemblyGenerator (dict);
-				llvmAsmgen.Init ();
+				var composer = new CompressedAssembliesNativeAssemblyGenerator (dict);
+				LLVMIR.LlvmIrModule compressedAssemblies = composer.Construct ();
 
 				foreach (string abi in SupportedAbis) {
 					string baseAsmFilePath = Path.Combine (EnvironmentOutputDirectory, $"compressed_assemblies.{abi.ToLowerInvariant ()}");
 					string llvmIrFilePath = $"{baseAsmFilePath}.ll";
 
 					using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) {
-						llvmAsmgen.Write (GeneratePackageManagerJava.GetAndroidTargetArchForAbi (abi), sw, llvmIrFilePath);
-						sw.Flush ();
+						try {
+							composer.Generate (compressedAssemblies, GeneratePackageManagerJava.GetAndroidTargetArchForAbi (abi), sw, llvmIrFilePath);
+						} catch {
+							throw;
+						} finally {
+							sw.Flush ();
+						}
+
 						if (Files.CopyIfStreamChanged (sw.BaseStream, llvmIrFilePath)) {
 							Log.LogDebugMessage ($"File {llvmIrFilePath} was regenerated");
 						}

src/Xamarin.Android.Build.Tasks/Tasks/GenerateJniRemappingNativeCode.cs

@@ -77,24 +77,24 @@ void Generate ()
 			Generate (new JniRemappingAssemblyGenerator (typeReplacements, methodReplacements), typeReplacements.Count);
 		}
 
-		void Generate (JniRemappingAssemblyGenerator jniRemappingGenerator, int typeReplacementsCount)
+		void Generate (JniRemappingAssemblyGenerator jniRemappingComposer, int typeReplacementsCount)
 		{
-			jniRemappingGenerator.Init ();
+			LLVMIR.LlvmIrModule module =  jniRemappingComposer.Construct ();
 
 			foreach (string abi in SupportedAbis) {
 				string baseAsmFilePath = Path.Combine (OutputDirectory, $"jni_remap.{abi.ToLowerInvariant ()}");
 				string llFilePath  = $"{baseAsmFilePath}.ll";
 
 				using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) {
-					jniRemappingGenerator.Write (GeneratePackageManagerJava.GetAndroidTargetArchForAbi (abi), sw, llFilePath);
+					jniRemappingComposer.Generate (module, GeneratePackageManagerJava.GetAndroidTargetArchForAbi (abi), sw, llFilePath);
 					sw.Flush ();
 					Files.CopyIfStreamChanged (sw.BaseStream, llFilePath);
 				}
 			}
 
 			BuildEngine4.RegisterTaskObjectAssemblyLocal (
 				ProjectSpecificTaskObjectKey (JniRemappingNativeCodeInfoKey),
-				new JniRemappingNativeCodeInfo (typeReplacementsCount, jniRemappingGenerator.ReplacementMethodIndexEntryCount),
+				new JniRemappingNativeCodeInfo (typeReplacementsCount, jniRemappingComposer.ReplacementMethodIndexEntryCount),
 				RegisteredTaskObjectLifetime.Build
 			);
 		}

src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs

@@ -390,7 +390,7 @@ void AddEnvironment ()
 				// and up to 4 other for arch-specific assemblies. Only **one** arch-specific store is ever loaded on the app
 				// runtime, thus the number 2 here. All architecture specific stores contain assemblies with the same names
 				// and in the same order.
-				MonoComponents = monoComponents,
+				MonoComponents = (MonoComponent)monoComponents,
 				NativeLibraries = uniqueNativeLibraries,
 				HaveAssemblyStore = UseAssemblyStore,
 				AndroidRuntimeJNIEnvToken = android_runtime_jnienv_class_token,
@@ -400,7 +400,7 @@ void AddEnvironment ()
 				JniRemappingReplacementMethodIndexEntryCount = jniRemappingNativeCodeInfo == null ? 0 : jniRemappingNativeCodeInfo.ReplacementMethodIndexEntryCount,
 				MarshalMethodsEnabled = EnableMarshalMethods,
 			};
-			appConfigAsmGen.Init ();
+			LLVMIR.LlvmIrModule appConfigModule = appConfigAsmGen.Construct ();
 
 			var marshalMethodsState = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal<MarshalMethodsState> (ProjectSpecificTaskObjectKey (GenerateJavaStubs.MarshalMethodsRegisterTaskKey), RegisteredTaskObjectLifetime.Build);
 			MarshalMethodsNativeAssemblyGenerator marshalMethodsAsmGen;
@@ -415,26 +415,36 @@ void AddEnvironment ()
 			} else {
 				marshalMethodsAsmGen = new MarshalMethodsNativeAssemblyGenerator (assemblyCount, uniqueAssemblyNames);
 			}
-			marshalMethodsAsmGen.Init ();
+			LLVMIR.LlvmIrModule marshalMethodsModule = marshalMethodsAsmGen.Construct ();
 
 			foreach (string abi in SupportedAbis) {
 				string targetAbi = abi.ToLowerInvariant ();
 				string environmentBaseAsmFilePath = Path.Combine (EnvironmentOutputDirectory, $"environment.{targetAbi}");
 				string marshalMethodsBaseAsmFilePath = Path.Combine (EnvironmentOutputDirectory, $"marshal_methods.{targetAbi}");
 				string environmentLlFilePath  = $"{environmentBaseAsmFilePath}.ll";
 				string marshalMethodsLlFilePath = $"{marshalMethodsBaseAsmFilePath}.ll";
-
 				AndroidTargetArch targetArch = GetAndroidTargetArchForAbi (abi);
+
 				using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) {
-					appConfigAsmGen.Write (targetArch, sw, environmentLlFilePath);
-					sw.Flush ();
-					Files.CopyIfStreamChanged (sw.BaseStream, environmentLlFilePath);
+					try {
+						appConfigAsmGen.Generate (appConfigModule, targetArch, sw, environmentLlFilePath);
+					} catch {
+						throw;
+					} finally {
+						sw.Flush ();
+						Files.CopyIfStreamChanged (sw.BaseStream, environmentLlFilePath);
+					}
 				}
 
 				using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) {
-					marshalMethodsAsmGen.Write (targetArch, sw, marshalMethodsLlFilePath);
-					sw.Flush ();
-					Files.CopyIfStreamChanged (sw.BaseStream, marshalMethodsLlFilePath);
+					try {
+						marshalMethodsAsmGen.Generate (marshalMethodsModule, targetArch, sw, marshalMethodsLlFilePath);
+					} catch {
+						throw;
+					} finally {
+						sw.Flush ();
+						Files.CopyIfStreamChanged (sw.BaseStream, marshalMethodsLlFilePath);
+					}
 				}
 			}
 

src/Xamarin.Android.Build.Tasks/Tasks/GetAotArguments.cs

@@ -293,14 +293,8 @@ string GetLdFlags (NdkTools ndk, AndroidTargetArch arch, int level, string toolP
 				libs.Add (Path.Combine (androidLibPath, "libc.so"));
 				libs.Add (Path.Combine (androidLibPath, "libm.so"));
 			} else if (!UseAndroidNdk && EnableLLVM) {
-				// We need to link against libc and libm, but since NDK is not in use, the linker won't be able to find the actual Android libraries.
-				// Therefore, we will use their stubs to satisfy the linker. At runtime they will, of course, use the actual Android libraries.
-				string relPath = Path.Combine ("..", "..");
-				if (!OS.IsWindows) {
-					// the `binutils` directory is one level down (${OS}/binutils) than the Windows one
-					relPath = Path.Combine (relPath, "..");
-				}
-				string libstubsPath = Path.GetFullPath (Path.Combine (AndroidBinUtilsDirectory, relPath, "libstubs", ArchToRid (arch)));
+				string libstubsPath = MonoAndroidHelper.GetLibstubsArchDirectoryPath (AndroidBinUtilsDirectory, arch);
+
 				libs.Add (Path.Combine (libstubsPath, "libc.so"));
 				libs.Add (Path.Combine (libstubsPath, "libm.so"));
 			}
@@ -332,26 +326,6 @@ string GetLdFlags (NdkTools ndk, AndroidTargetArch arch, int level, string toolP
 			}
 
 			return ldFlags.ToString ();
-
-			string ArchToRid (AndroidTargetArch arch)
-			{
-				switch (arch) {
-					case AndroidTargetArch.Arm64:
-						return "android-arm64";
-
-					case AndroidTargetArch.Arm:
-						return "android-arm";
-
-					case AndroidTargetArch.X86:
-						return "android-x86";
-
-					case AndroidTargetArch.X86_64:
-						return "android-x64";
-
-					default:
-						throw new InvalidOperationException ($"Internal error: unsupported ABI '{arch}'");
-				}
-			}
 		}
 
 		static string GetNdkToolchainLibraryDir (NdkTools ndk, string binDir, string archDir = null)

src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs

@@ -29,6 +29,7 @@ sealed class InputFiles
 		{
 			public List<string> ObjectFiles;
 			public string OutputSharedLibrary;
+			public List<string> ExtraLibraries;
 		}
 
 		[Required]
@@ -112,22 +113,23 @@ void RunLinker (Config config)
 
 		IEnumerable<Config> GetLinkerConfigs ()
 		{
+			string runtimeNativeLibsDir = MonoAndroidHelper.GetNativeLibsRootDirectoryPath (AndroidBinUtilsDirectory);
+			string runtimeNativeLibStubsDir = MonoAndroidHelper.GetLibstubsRootDirectoryPath (AndroidBinUtilsDirectory);
 			var abis = new Dictionary <string, InputFiles> (StringComparer.Ordinal);
 			ITaskItem[] dsos = ApplicationSharedLibraries;
 			foreach (ITaskItem item in dsos) {
 				string abi = item.GetMetadata ("abi");
-				abis [abi] = GatherFilesForABI(item.ItemSpec, abi, ObjectFiles);
+				abis [abi] = GatherFilesForABI (item.ItemSpec, abi, ObjectFiles, runtimeNativeLibsDir, runtimeNativeLibStubsDir);
 			}
 
 			const string commonLinkerArgs =
-				"--unresolved-symbols=ignore-in-shared-libs " +
+				"--shared " +
+				"--allow-shlib-undefined " +
 				"--export-dynamic " +
 				"-soname libxamarin-app.so " +
 				"-z relro " +
 				"-z noexecstack " +
 				"--enable-new-dtags " +
-				"--eh-frame-hdr " +
-				"-shared " +
 				"--build-id " +
 				"--warn-shared-textrel " +
 				"--fatal-warnings";
@@ -177,6 +179,12 @@ IEnumerable<Config> GetLinkerConfigs ()
 				targetLinkerArgs.Add ("-o");
 				targetLinkerArgs.Add (QuoteFileName (inputs.OutputSharedLibrary));
 
+				if (inputs.ExtraLibraries != null) {
+					foreach (string lib in inputs.ExtraLibraries) {
+						targetLinkerArgs.Add (lib);
+					}
+				}
+
 				string targetArgs = String.Join (" ", targetLinkerArgs);
 				yield return new Config {
 					LinkerPath = ld,
@@ -186,11 +194,24 @@ IEnumerable<Config> GetLinkerConfigs ()
 			}
 		}
 
-		InputFiles GatherFilesForABI (string runtimeSharedLibrary, string abi, ITaskItem[] objectFiles)
+		InputFiles GatherFilesForABI (string runtimeSharedLibrary, string abi, ITaskItem[] objectFiles, string runtimeNativeLibsDir, string runtimeNativeLibStubsDir)
 		{
+			List<string> extraLibraries = null;
+			string RID = MonoAndroidHelper.AbiToRid (abi);
+			AndroidTargetArch targetArch = MonoAndroidHelper.AbiToTargetArch (abi);
+			string libStubsPath = Path.Combine (runtimeNativeLibStubsDir, RID);
+			string runtimeLibsDir = Path.Combine (runtimeNativeLibsDir, RID);
+
+			extraLibraries = new List<string> {
+				$"-L \"{runtimeLibsDir}\"",
+				$"-L \"{libStubsPath}\"",
+				"-lc",
+			};
+
 			return new InputFiles {
 				OutputSharedLibrary = runtimeSharedLibrary,
 				ObjectFiles = GetItemsForABI (abi, objectFiles),
+				ExtraLibraries = extraLibraries,
 			};
 		}
 

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs

@@ -402,7 +402,7 @@ static Dictionary<string, string> ReadEnvironmentVariables (EnvironmentFile envF
 		static string[] GetField (string llvmAssemblerFile, string nativeAssemblerFile, string line, ulong lineNumber)
 		{
 			string[] ret = line?.Trim ()?.Split ('\t');
-			Assert.IsTrue (ret.Length >= 2, $"Invalid assembler field format in file '{nativeAssemblerFile}:{lineNumber}': '{line}'. File generated from '{llvmAssemblerFile}'");
+			Assert.IsTrue (ret != null && ret.Length >= 2, $"Invalid assembler field format in file '{nativeAssemblerFile}:{lineNumber}': '{line}'. File generated from '{llvmAssemblerFile}'");
 
 			return ret;
 		}

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/NativeAssemblyParser.cs

@@ -192,7 +192,7 @@ public SymbolMetadata (SymbolMetadataKind kind, string value = null)
 
 		static readonly char[] splitOnWhitespace = new char[] { ' ', '\t' };
 		static readonly char[] splitOnComma = new char[] { ',' };
-		static readonly Regex assemblerLabelRegex = new Regex ("^[_.a-zA-Z0-9]+:", RegexOptions.Compiled);
+		static readonly Regex assemblerLabelRegex = new Regex ("^[_.$a-zA-Z0-9]+:", RegexOptions.Compiled);
 
 		Dictionary<string, AssemblerSymbol> symbols = new Dictionary<string, AssemblerSymbol> (StringComparer.Ordinal);
 		Dictionary<string, List<SymbolMetadata>> symbolMetadata = new Dictionary<string, List<SymbolMetadata>> (StringComparer.Ordinal);
@@ -238,8 +238,10 @@ void Load (string sourceFilePath)
 			AssemblerSection currentSection = null;
 			AssemblerSymbol currentSymbol = null;
 
-			string symbolName;
+			string symbolName = null;
 			ulong lineNumber = 0;
+			bool addedNewSymbol = false;
+
 			foreach (string l in File.ReadLines (sourceFilePath, Encoding.UTF8)) {
 				lineNumber++;
 
@@ -253,6 +255,15 @@ void Load (string sourceFilePath)
 					continue;
 				}
 
+				if (addedNewSymbol) {
+					addedNewSymbol = false;
+					// Some forms of LLVM IR can generate two labels for a single symbol, depending on symbol visibility, attributes and llc parameters.
+					// The exported symbol name 'symbol:' may be followed by another one '.Lsymbol$local:', we need to detect this and ignore the new symbol.
+					if (assemblerLabelRegex.IsMatch (line) && String.Compare (line.Trim (), $".L{symbolName}$local:", StringComparison.Ordinal) == 0) {
+						continue;
+					}
+				}
+
 				if (StartsNewSection (parts, ref currentSection)) {
 					currentSymbol = null; // Symbols cannot cross sections
 					continue;
@@ -265,6 +276,7 @@ void Load (string sourceFilePath)
 				if (assemblerLabelRegex.IsMatch (line)) {
 					symbolName = GetSymbolName (line);
 					currentSymbol = AddNewSymbol (symbolName);
+					addedNewSymbol = true;
 					continue;
 				}