[build] Reimplement NdkUtils

Context: #5996
Context: #5964
Context: #5964 (comment)

The `NdkUtils` class used by Xamarin.Andrid.Build.Tasks to find tooling
shipped with the Android NDK, has grown increasingly complicated over
the years due to a number of incompatibilities between various versions
of the NDK. The code became hard to follow and untidy. This commit
attempts to address the issue by replacing the single static `NdkUtils`
class with a hierarchy of dynamically instantiated classes rooted in a
new base class, `NdkTools`.

`NdkUtils` had to be initialized for each thread that needed to access
its methods, which led to various issues with concurrency and lack of
proper initialization since the initialization had to be done wherever
`NdkUtils` was first accessed, meaning that any task using it had to do
it.

`NdkTools` doesn't require such initialization, instead it provides a
factory method called `Create` which takes path to the NDK as its
parameter and returns an instance of `NdkTools` child class (or `null`
if an error occurs) which the can be safely used by the caller.  Callers
need not concern themselves with what is the actual type of the returned
instance, they access only methods and properties defined in the
`NdkTools` base abstract class.

The hierarchy of `NdkTools` derivatives is structured and named after
the breaking changes in the NDK.  For instance, NDK versions before 16
used the GNU compilers, while release 16 and above use the clang
compilers - this is reflected in existence of two classes derived from
`NdkTools`, `NoClang` for NDKs older than r16 and `WithClang` for the
newer ones. The other breaking changes are the addition of unified
headers in r19, removal of the `platforms` directory in r22 and removal
of GNU Binutils in r23.

NDK r23 is recognized in this commit but it is NOT supported. Support
for r23 is being worked on in PR #6073 which will be merged once r23 is
out of beta.

Author: grendello

src/Xamarin.Android.Build.Tasks/Tasks/Aot.cs

@@ -84,15 +84,6 @@ public class Aot : AndroidAsyncTask
 		AotMode AotMode;
 		SequencePointsMode sequencePointsMode;
 
-		public override bool RunTask ()
-		{
-			// NdkUtil must always be initialized - once per thread
-			if (!NdkUtil.Init (LogCodedError, AndroidNdkDirectory))
-				return false;
-
-			return base.RunTask ();
-		}
-
 		public static bool GetAndroidAotMode(string androidAotMode, out AotMode aotMode)
 		{
 			aotMode = AotMode.Normal;
@@ -138,7 +129,7 @@ public static bool TryGetSequencePointsMode (string value, out SequencePointsMod
 			return false;
 		}
 
-		static string GetNdkToolchainLibraryDir(string binDir, string archDir = null)
+		static string GetNdkToolchainLibraryDir (NdkTools ndk, string binDir, string archDir = null)
 		{
 			var baseDir = Path.GetFullPath(Path.Combine(binDir, ".."));
 
@@ -154,7 +145,7 @@ static string GetNdkToolchainLibraryDir(string binDir, string archDir = null)
 				goto no_toolchain_error;
 			}
 
-			if (NdkUtil.UsingClangNDK)
+			if (ndk.UsesClang)
 				return libPath;
 
 			gccLibDir = Directory.EnumerateDirectories(libPath).ToList();
@@ -171,9 +162,9 @@ static string GetNdkToolchainLibraryDir(string binDir, string archDir = null)
 			throw new Exception("Could not find a valid NDK compiler toolchain library path");
 		}
 
-		static string GetNdkToolchainLibraryDir (string binDir, AndroidTargetArch arch)
+		static string GetNdkToolchainLibraryDir (NdkTools ndk, string binDir, AndroidTargetArch arch)
 		{
-			return GetNdkToolchainLibraryDir (binDir, NdkUtil.GetArchDirName (arch));
+			return GetNdkToolchainLibraryDir (ndk, binDir, ndk.GetArchDirName (arch));
 		}
 
 		static string QuoteFileName(string fileName)
@@ -183,7 +174,7 @@ static string QuoteFileName(string fileName)
 			return builder.ToString();
 		}
 
-		int GetNdkApiLevel(string androidNdkPath, string androidApiLevel, AndroidTargetArch arch)
+		int GetNdkApiLevel (NdkTools ndk, string androidApiLevel, AndroidTargetArch arch)
 		{
 			var manifest    = AndroidAppManifest.Load (ManifestFile.ItemSpec, MonoAndroidHelper.SupportedVersions);
 
@@ -209,15 +200,15 @@ int GetNdkApiLevel(string androidNdkPath, string androidApiLevel, AndroidTargetA
 			else if (level == 23) level = 21;
 
 			// API levels below level 21 do not provide support for 64-bit architectures.
-			if (NdkUtil.IsNdk64BitArch(arch) && level < 21) {
+			if (ndk.IsNdk64BitArch (arch) && level < 21) {
 				level = 21;
 			}
 
 			// We perform a downwards API level lookup search since we might not have hardcoded the correct API
 			// mapping above and we do not want to crash needlessly.
 			for (; level >= 5; level--) {
 				try {
-					NdkUtil.GetNdkPlatformLibPath (androidNdkPath, arch, level);
+					ndk.GetDirectoryPath (NdkToolchainDir.PlatformLib, arch, level);
 					break;
 				} catch (InvalidOperationException ex) {
 					// Path not found, continue searching...
@@ -230,10 +221,9 @@ int GetNdkApiLevel(string androidNdkPath, string androidApiLevel, AndroidTargetA
 
 		public async override System.Threading.Tasks.Task RunTaskAsync ()
 		{
-			// NdkUtil must always be initialized - once per thread
-			if (!NdkUtil.Init (LogCodedError, AndroidNdkDirectory)) {
-				LogDebugMessage ("Failed to initialize NdkUtil");
-				return;
+			NdkTools? ndk = NdkTools.Create (AndroidNdkDirectory, Log);
+			if (ndk == null) {
+				return; // NdkTools.Create will log appropriate error
 			}
 
 			bool hasValidAotMode = GetAndroidAotMode (AndroidAotMode, out AotMode);
@@ -251,7 +241,7 @@ public async override System.Threading.Tasks.Task RunTaskAsync ()
 
 			var nativeLibs = new List<string> ();
 
-			await this.WhenAllWithLock (GetAotConfigs (),
+			await this.WhenAllWithLock (GetAotConfigs (ndk),
 				(config, lockObject) => {
 					if (!config.Valid) {
 						Cancel ();
@@ -277,7 +267,7 @@ await this.WhenAllWithLock (GetAotConfigs (),
 			LogDebugTaskItems ("  NativeLibrariesReferences: ", NativeLibrariesReferences);
 		}
 
-		IEnumerable<Config> GetAotConfigs ()
+		IEnumerable<Config> GetAotConfigs (NdkTools ndk)
 		{
 			if (!Directory.Exists (AotOutputDirectory))
 				Directory.CreateDirectory (AotOutputDirectory);
@@ -325,7 +315,7 @@ IEnumerable<Config> GetAotConfigs ()
 					throw new Exception ("Unsupported Android target architecture ABI: " + abi);
 				}
 
-				if (EnableLLVM && !NdkUtil.ValidateNdkPlatform (LogMessage, LogCodedError, AndroidNdkDirectory, arch, enableLLVM:EnableLLVM)) {
+				if (EnableLLVM && !ndk.ValidateNdkPlatform (LogMessage, LogCodedError, arch, enableLLVM:EnableLLVM)) {
 					yield return Config.Invalid;
 					yield break;
 				}
@@ -341,8 +331,8 @@ IEnumerable<Config> GetAotConfigs ()
 
 				int level = 0;
 				string toolPrefix = EnableLLVM
-					? NdkUtil.GetNdkToolPrefix (AndroidNdkDirectory, arch, level = GetNdkApiLevel (AndroidNdkDirectory, AndroidApiLevel, arch))
-					: Path.Combine (AndroidBinUtilsDirectory, $"{NdkUtil.GetArchDirName (arch)}-");
+					? ndk.GetNdkToolPrefixForAOT (arch, level = GetNdkApiLevel (ndk, AndroidApiLevel, arch))
+					: Path.Combine (AndroidBinUtilsDirectory, $"{ndk.GetArchDirName (arch)}-");
 				var toolchainPath = toolPrefix.Substring(0, toolPrefix.LastIndexOf(Path.DirectorySeparatorChar));
 				var ldFlags = string.Empty;
 				if (EnableLLVM) {
@@ -353,25 +343,25 @@ IEnumerable<Config> GetAotConfigs ()
 
 					string androidLibPath = string.Empty;
 					try {
-						androidLibPath = NdkUtil.GetNdkPlatformLibPath(AndroidNdkDirectory, arch, level);
+						androidLibPath = ndk.GetDirectoryPath (NdkToolchainDir.PlatformLib, arch, level);
 					} catch (InvalidOperationException ex) {
 						Diagnostic.Error (5101, ex.Message);
 					}
 
 					string toolchainLibDir;
-					if (NdkUtil.UsingClangNDK)
-						toolchainLibDir = GetNdkToolchainLibraryDir (toolchainPath, arch);
+					if (ndk.UsesClang)
+						toolchainLibDir = GetNdkToolchainLibraryDir (ndk, toolchainPath, arch);
 					else
-						toolchainLibDir = GetNdkToolchainLibraryDir (toolchainPath);
+						toolchainLibDir = GetNdkToolchainLibraryDir (ndk, toolchainPath);
 
 					var libs = new List<string>();
-					if (NdkUtil.UsingClangNDK) {
+					if (ndk.UsesClang) {
 						libs.Add ($"-L{toolchainLibDir.TrimEnd ('\\')}");
 						libs.Add ($"-L{androidLibPath.TrimEnd ('\\')}");
 
 						if (arch == AndroidTargetArch.Arm) {
 							// Needed for -lunwind to work
-							string compilerLibDir = Path.Combine (toolchainPath, "..", "sysroot", "usr", "lib", NdkUtil.GetArchDirName (arch));
+							string compilerLibDir = Path.Combine (toolchainPath, "..", "sysroot", "usr", "lib", ndk.GetArchDirName (arch));
 							libs.Add ($"-L{compilerLibDir.TrimEnd ('\\')}");
 						}
 					}
@@ -385,7 +375,7 @@ IEnumerable<Config> GetAotConfigs ()
 
 				string ldName = String.Empty;
 				if (EnableLLVM) {
-					ldName = NdkUtil.GetNdkTool (AndroidNdkDirectory, arch, "ld", level);
+					ldName = ndk.GetToolPath (NdkToolKind.Linker, arch, level);
 					if (!String.IsNullOrEmpty (ldName)) {
 						ldName = Path.GetFileName (ldName);
 						if (ldName.IndexOf ('-') >= 0) {

src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs

@@ -635,8 +635,12 @@ private void AddNativeLibraries (ArchiveFileList files, string [] supportedAbis)
 				return;
 			}
 
-			NdkUtil.Init (AndroidNdkDirectory);
-			string clangDir = NdkUtil.GetClangDeviceLibraryPath (AndroidNdkDirectory);
+			NdkTools? ndk = NdkTools.Create (AndroidNdkDirectory, Log);
+			if (ndk == null) {
+				return; // NdkTools.Create will log appropriate error
+			}
+
+			string clangDir = ndk.GetClangDeviceLibraryPath ();
 			if (String.IsNullOrEmpty (clangDir)) {
 				LogSanitizerError ($"Unable to find the clang compiler directory. Is NDK installed?");
 				return;

src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs

@@ -168,7 +168,7 @@ IEnumerable<Config> GetLinkerConfigs ()
 					linkerArgs.Add (QuoteFileName (file));
 				}
 
-				string ld = MonoAndroidHelper.GetExecutablePath (AndroidBinUtilsDirectory, $"{NdkUtil.GetNdkToolchainPrefix (arch, false)}ld");
+				string ld = MonoAndroidHelper.GetExecutablePath (AndroidBinUtilsDirectory, $"{NdkTools.GetBinutilsToolchainPrefix (arch)}ld");
 				yield return new Config {
 					LinkerPath = Path.Combine (AndroidBinUtilsDirectory, ld),
 					LinkerOptions = String.Join (" ", linkerArgs),

src/Xamarin.Android.Build.Tasks/Tasks/MakeBundleNativeCodeExternal.cs

@@ -28,11 +28,11 @@ public class MakeBundleNativeCodeExternal : AndroidTask
 
 		[Required]
 		public ITaskItem[] Assemblies { get; set; }
-		
+
 		// Which ABIs to include native libs for
 		[Required]
 		public string [] SupportedAbis { get; set; }
-		
+
 		[Required]
 		public string TempOutputPath { get; set; }
 
@@ -57,11 +57,13 @@ public MakeBundleNativeCodeExternal ()
 
 		public override bool RunTask ()
 		{
-			if (!NdkUtil.Init (Log, AndroidNdkDirectory))
-				return false;
+			NdkTools? ndk = NdkTools.Create (AndroidNdkDirectory, Log);
+			if (ndk == null) {
+				return false; // NdkTools.Create will log appropriate error
+			}
 
 			try {
-				return DoExecute ();
+				return DoExecute (ndk);
 			} catch (XamarinAndroidException e) {
 				Log.LogCodedError (string.Format ("XA{0:0000}", e.Code), e.MessageWithoutCode);
 				if (MonoAndroidHelper.LogInternalExceptions)
@@ -72,7 +74,7 @@ public override bool RunTask ()
 			return !Log.HasLoggedErrors;
 		}
 
-		bool DoExecute ()
+		bool DoExecute (NdkTools ndk)
 		{
 			var results = new List<ITaskItem> ();
 			string bundlepath = Path.Combine (TempOutputPath, "bundles");
@@ -102,11 +104,11 @@ bool DoExecute ()
 					break;
 				}
 
-				if (!NdkUtil.ValidateNdkPlatform (Log, AndroidNdkDirectory, arch, enableLLVM: false)) {
+				if (!ndk.ValidateNdkPlatform (arch, enableLLVM: false)) {
 					return false;
 				}
 
-				int level = NdkUtil.GetMinimumApiLevelFor (arch, AndroidNdkDirectory);
+				int level = ndk.GetMinimumApiLevelFor (arch);
 				var outpath = Path.Combine (bundlepath, abi);
 				if (!Directory.Exists (outpath))
 					Directory.CreateDirectory (outpath);
@@ -140,10 +142,10 @@ bool DoExecute ()
 					CreateNoWindow = true,
 					WindowStyle = ProcessWindowStyle.Hidden,
 				};
-				string windowsCompilerSwitches = NdkUtil.GetCompilerTargetParameters (AndroidNdkDirectory, arch, level);
-				var compilerNoQuotes = NdkUtil.GetNdkTool (AndroidNdkDirectory, arch, "gcc", level);
+				string windowsCompilerSwitches = ndk.GetCompilerTargetParameters (arch, level);
+				var compilerNoQuotes = ndk.GetToolPath (NdkToolKind.CompilerC, arch, level);
 				var compiler = $"\"{compilerNoQuotes}\" {windowsCompilerSwitches}".Trim ();
-				var gas = '"' + NdkUtil.GetNdkTool (AndroidNdkDirectory, arch, "as", level) + '"';
+				var gas = '"' + ndk.GetToolPath (NdkToolKind.Assembler, arch, level) + '"';
 				psi.EnvironmentVariables ["CC"] = compiler;
 				psi.EnvironmentVariables ["AS"] = gas;
 				Log.LogDebugMessage ("CC=" + compiler);
@@ -167,7 +169,7 @@ bool DoExecute ()
 
 				clb = new CommandLineBuilder ();
 
-				// See NdkUtils.GetNdkTool for reasons why
+				// See NdkToolsWithClangWithPlatforms.ctor for reasons why
 				if (!String.IsNullOrEmpty (windowsCompilerSwitches))
 					clb.AppendTextUnquoted (windowsCompilerSwitches);
 
@@ -188,14 +190,14 @@ bool DoExecute ()
 					clb.AppendFileNameIfNotNull (IncludePath);
 				}
 
-				string asmIncludePath = NdkUtil.GetNdkAsmIncludePath (AndroidNdkDirectory, arch, level);
+				string asmIncludePath = ndk.GetDirectoryPath (NdkToolchainDir.AsmInclude, arch, level);
 				if (!String.IsNullOrEmpty (asmIncludePath)) {
 					clb.AppendSwitch ("-I");
 					clb.AppendFileNameIfNotNull (asmIncludePath);
 				}
 
 				clb.AppendSwitch ("-I");
-				clb.AppendFileNameIfNotNull (NdkUtil.GetNdkPlatformIncludePath (AndroidNdkDirectory, arch, level));
+				clb.AppendFileNameIfNotNull (ndk.GetDirectoryPath (NdkToolchainDir.PlatformInclude, arch, level));
 				clb.AppendFileNameIfNotNull (Path.Combine (outpath, "temp.c"));
 				Log.LogDebugMessage ("[CC] " + compiler + " " + clb);
 				if (MonoAndroidHelper.RunProcess (compilerNoQuotes, clb.ToString (), OnCcOutputData,  OnCcErrorData) != 0) {
@@ -216,13 +218,13 @@ bool DoExecute ()
 				clb.AppendSwitch ("-o");
 				clb.AppendFileNameIfNotNull (Path.Combine (outpath, BundleSharedLibraryName));
 				clb.AppendSwitch ("-L");
-				clb.AppendFileNameIfNotNull (NdkUtil.GetNdkPlatformLibPath (AndroidNdkDirectory, arch, level));
+				clb.AppendFileNameIfNotNull (ndk.GetDirectoryPath (NdkToolchainDir.PlatformLib, arch, level));
 				clb.AppendSwitch ("-lc");
 				clb.AppendSwitch ("-lm");
 				clb.AppendSwitch ("-ldl");
 				clb.AppendSwitch ("-llog");
 				clb.AppendSwitch ("-lz"); // Compress
-				string ld = NdkUtil.GetNdkTool (AndroidNdkDirectory, arch, "ld", level);
+				string ld = ndk.GetToolPath (NdkToolKind.Linker, arch, level);
 				Log.LogMessage (MessageImportance.Normal, "[LD] " + ld + " " + clb);
 				if (MonoAndroidHelper.RunProcess (ld, clb.ToString (), OnLdOutputData,  OnLdErrorData) != 0) {
 					Log.LogCodedError ("XA5201", Properties.Resources.XA5201, proc.ExitCode);