[Xamarin.Android.Build.Tasks] Hitting "System.IO.IOException: Too many open files" when building a large app with AOT and all supported architectures (#493)

Fixes: https://bugzilla.xamarin.com/show_bug.cgi?id=53122

There is a problem with libzip in that is does not have an
API for Flushing the current items in memory to disk.
This was why ZipArchiveEx was introduced in commit 9166e03.
This commit reworks the BuildApk task to make sure of this
new functinality and to periodically flush the zip to disk.
This will reduce the chances of running out of memory.

It also adds a unit test which generates and builds an app
which has a large number of refernces and assets. This app
also gets AOT'd so it should be pushing the system to the limit.

Also added code to make sure the cross tools and llc have
execute permissions.

Added code to make sure we dispose of each process in the Aapt and
Aot tasks. This is to ensure any files or pipes these processed have
are removed as soon as possible.

Updated the unix-distribution-setup.targets to ensure that the cross-arm
tooling and llc have the correct execute permissions on MacOS and Linux.

Author: dellis1972

build-tools/unix-distribution-setup/unix-distribution-setup.targets

@@ -9,6 +9,14 @@
         Condition=" '$(HostOS)' != 'Windows' "
         Command="chmod +x $(OutputPath)bin\generator"
     />
+    <Exec
+        Condition=" '$(HostOS)' != 'Windows' "
+        Command="chmod +x $(OutputPath)bin\cross*"
+    />
+    <Exec
+        Condition=" '$(HostOS)' != 'Windows' "
+        Command="chmod +x $(OutputPath)bin\llc"
+    />
   </Target>
 
   <Target Name="Clean">

src/Xamarin.Android.Build.Tasks/Tasks/Aapt.cs

@@ -99,36 +99,37 @@ bool RunAapt (string commandLine)
 				WindowStyle = ProcessWindowStyle.Hidden,
 			};
 
-			var proc = new Process ();
-			proc.OutputDataReceived += (sender, e) => {
-				if (e.Data != null)
-					LogEventsFromTextOutput (e.Data, MessageImportance.Normal);
-				else
-					stdout_completed.Set ();
-			};
-			proc.ErrorDataReceived += (sender, e) => {
-				if (e.Data != null)
-					LogEventsFromTextOutput (e.Data, MessageImportance.Normal);
-				else
-					stderr_completed.Set ();
-			};
-			proc.StartInfo = psi;
-			proc.Start ();
-			proc.BeginOutputReadLine ();
-			proc.BeginErrorReadLine ();
-			Token.Register (() => {
-				try {
-					proc.Kill ();
-				} catch (Exception) {
-				}
-			});
-			LogDebugMessage ("Executing {0}", commandLine);
-			proc.WaitForExit ();
-			if (psi.RedirectStandardError)
-				stderr_completed.WaitOne (TimeSpan.FromSeconds (30));
-			if (psi.RedirectStandardOutput)
-				stdout_completed.WaitOne (TimeSpan.FromSeconds (30));
-			return proc.ExitCode == 0;
+			using (var proc = new Process ()) {
+				proc.OutputDataReceived += (sender, e) => {
+					if (e.Data != null)
+						LogEventsFromTextOutput (e.Data, MessageImportance.Normal);
+					else
+						stdout_completed.Set ();
+				};
+				proc.ErrorDataReceived += (sender, e) => {
+					if (e.Data != null)
+						LogEventsFromTextOutput (e.Data, MessageImportance.Normal);
+					else
+						stderr_completed.Set ();
+				};
+				proc.StartInfo = psi;
+				proc.Start ();
+				proc.BeginOutputReadLine ();
+				proc.BeginErrorReadLine ();
+				Token.Register (() => {
+					try {
+						proc.Kill ();
+					} catch (Exception) {
+					}
+				});
+				LogDebugMessage ("Executing {0}", commandLine);
+				proc.WaitForExit ();
+				if (psi.RedirectStandardError)
+					stderr_completed.WaitOne (TimeSpan.FromSeconds (30));
+				if (psi.RedirectStandardOutput)
+					stdout_completed.WaitOne (TimeSpan.FromSeconds (30));
+				return proc.ExitCode == 0;
+			}
 		}
 
 		bool ExecuteForAbi (string cmd, string currentResourceOutputFile)

src/Xamarin.Android.Build.Tasks/Tasks/Aot.cs

@@ -434,6 +434,8 @@ IEnumerable<Config> GetAotConfigs ()
 
 		bool RunAotCompiler (string assembliesPath, string aotCompiler, string aotOptions, string assembly, CancellationToken token)
 		{
+			var stdout_completed = new ManualResetEvent (false);
+			var stderr_completed = new ManualResetEvent (false);
 			var psi = new ProcessStartInfo () {
 				FileName = aotCompiler,
 				Arguments = aotOptions + " \"" + assembly + "\"",
@@ -453,16 +455,32 @@ bool RunAotCompiler (string assembliesPath, string aotCompiler, string aotOption
 			LogDebugMessage ("[AOT] MONO_PATH=\"{0}\" MONO_ENV_OPTIONS=\"{1}\" {2} {3}",
 				psi.EnvironmentVariables ["MONO_PATH"], psi.EnvironmentVariables ["MONO_ENV_OPTIONS"], psi.FileName, psi.Arguments);
 
-			var proc = new Process ();
-			proc.OutputDataReceived += OnAotOutputData;
-			proc.ErrorDataReceived += OnAotErrorData;
-			proc.StartInfo = psi;
-			proc.Start ();
-			proc.BeginOutputReadLine ();
-			proc.BeginErrorReadLine ();
-			token.Register (() => { try { proc.Kill (); } catch (Exception) {} });
-			proc.WaitForExit ();
-			return proc.ExitCode == 0;
+			using (var proc = new Process ()) {
+				proc.OutputDataReceived += (s, e) => {
+					if (e.Data != null)
+						OnAotOutputData (s, e);
+					else
+						stdout_completed.Set ();
+				};
+				proc.ErrorDataReceived += (s, e) => {
+					if (e.Data != null)
+						OnAotErrorData (s, e);
+					else
+						stderr_completed.Set ();
+				};
+				proc.StartInfo = psi;
+				proc.Start ();
+				proc.BeginOutputReadLine ();
+				proc.BeginErrorReadLine ();
+				token.Register (() => { try { proc.Kill (); } catch (Exception) { } });
+				proc.WaitForExit ();
+				if (psi.RedirectStandardError)
+					stderr_completed.WaitOne (TimeSpan.FromSeconds (30));
+				if (psi.RedirectStandardOutput)
+					stdout_completed.WaitOne (TimeSpan.FromSeconds (30));
+				return proc.ExitCode == 0;
+			}
+			GC.Collect ();
 		}
 
 		void OnAotOutputData (object sender, DataReceivedEventArgs e)