[CI] Use dotnet test slicer in nightly tests

Updates the `TimeZoneInfo` and `Localization` test stages to use the
`dotnet-test-slicer` tool used by other MSBuild test runs.

These test jobs have a pretty high frequency of failing and timing out,
and a few changes have been made to try to reduce the points of failure.

SetUp and TearDown functions have been trimmed down and made standard
across the two fixtures.  OneTimeSetup will prepare the device and
install the project, and we will only attempt to attach log files during
TearDown.

The tests will launch the app activity directly with ADB, rather than
using `dotnet build -t:Run` for every iteration.  The tests will attempt
to reinstall the app if it is not present as a fallback.

Author: pjcollins

build-tools/automation/azure-pipelines-nightly.yaml

@@ -152,7 +152,7 @@ static CodeBehindTests ()
 		{
 			TestProjectRootDirectory = Path.GetFullPath (Path.Combine (XABuildPaths.TopDirectory, "tests", "CodeBehind", "BuildTests"));
 			CommonSampleLibraryRootDirectory = Path.GetFullPath (Path.Combine (XABuildPaths.TopDirectory, "tests", "CodeBehind", CommonSampleLibraryName));
-			TestOutputDir = Path.Combine (SetUp.TestDirectoryRoot, "temp", "CodeBehind");
+			TestOutputDir = Path.Combine (XABuildPaths.TestOutputDirectory, "temp", "CodeBehind");
 			if (Builder.UseDotNet) {
 				ProjectName += ".NET";
 			}

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/CodeBehindTests.cs

@@ -57,7 +57,7 @@ public bool Build (string projectOrSolution, string target, string [] parameters
 		static EmbeddedDSOTests ()
 		{
 			TestProjectRootDirectory = Path.GetFullPath (Path.Combine (XABuildPaths.TopDirectory, "tests", "EmbeddedDSOs", "EmbeddedDSO"));
-			TestOutputDir = Path.Combine (SetUp.TestDirectoryRoot, "temp", "EmbeddedDSO");
+			TestOutputDir = Path.Combine (XABuildPaths.TestOutputDirectory, "temp", "EmbeddedDSO");
 
 			produced_binaries = new List <string> {
 				$"{ProjectAssemblyName}.dll",

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/EmbeddedDSOTests.cs

@@ -23,58 +23,9 @@ public class BaseTest
 		public static ConcurrentDictionary<string, string> TestOutputDirectories = new ConcurrentDictionary<string, string> ();
 		public static ConcurrentDictionary<string, string> TestPackageNames = new ConcurrentDictionary<string, string> ();
 
-		[SetUpFixture]
-		public class SetUp
-		{
-			public static string TestDirectoryRoot {
-				get;
-				private set;
-			}
-
-			[OneTimeSetUp]
-			public void BeforeAllTests ()
-			{
-				TestDirectoryRoot = XABuildPaths.TestOutputDirectory;
-			}
-
-			[OneTimeTearDown]
-			public void AfterAllTests ()
-			{
-				if (System.Diagnostics.Debugger.IsAttached)
-					return;
-
-				//NOTE: adb.exe can cause a couple issues on Windows
-				//	1) it holds a lock on ~/android-toolchain, so a future build that needs to delete/recreate would fail
-				//	2) the MSBuild <Exec /> task *can* hang until adb.exe exits
-
-				try {
-					RunAdbCommand ("kill-server", true);
-				} catch (Exception ex) {
-					Console.Error.WriteLine ("Failed to run adb kill-server: " + ex);
-				}
-
-				//NOTE: in case `adb kill-server` fails, kill the process as a last resort
-				foreach (var p in Process.GetProcessesByName ("adb.exe"))
-					p.Kill ();
-			}
-
-		}
-
 		protected bool IsWindows => TestEnvironment.IsWindows;
 
-		protected bool IsMacOS => TestEnvironment.IsMacOS;
-
-		protected bool IsLinux => TestEnvironment.IsLinux;
-
-		public string StagingPath {
-			get { return Environment.GetFolderPath (Environment.SpecialFolder.MyDocuments); }
-		}
-
-		public string Root {
-			get {
-				return Path.GetFullPath (SetUp.TestDirectoryRoot);
-			}
-		}
+		public string Root => Path.GetFullPath (XABuildPaths.TestOutputDirectory);
 
 		public static bool CommercialBuildAvailable => TestEnvironment.CommercialBuildAvailable;
 
@@ -569,6 +520,28 @@ public void TestSetup ()
 			TestContext.Out.Flush ();
 		}
 
+		[OneTimeTearDown]
+		protected virtual void AfterAllTests ()
+		{
+			if (System.Diagnostics.Debugger.IsAttached)
+				return;
+
+			//NOTE: adb.exe can cause a couple issues on Windows
+			//	1) it holds a lock on ~/android-toolchain, so a future build that needs to delete/recreate would fail
+			//	2) the MSBuild <Exec /> task *can* hang until adb.exe exits
+			if (IsWindows) {
+				try {
+					RunAdbCommand ("kill-server", true);
+				} catch (Exception ex) {
+					Console.Error.WriteLine ("Failed to run adb kill-server: " + ex);
+				}
+
+				//NOTE: in case `adb kill-server` fails, kill the process as a last resort
+				foreach (var p in Process.GetProcessesByName ("adb.exe"))
+					p.Kill ();
+			}
+		}
+
 		[TearDown]
 		protected virtual void CleanupTest ()
 		{

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/BaseTest.cs

@@ -73,7 +73,7 @@ public void DeviceSetup ()
 		}
 
 		[OneTimeTearDown]
-		public void DeviceTearDown ()
+		protected virtual void DeviceTearDown ()
 		{
 			if (IsDeviceAttached ()) {
 				// make sure we are not on a guest user anymore.
@@ -199,11 +199,16 @@ protected static void RunProjectAndAssert (XamarinAndroidApplicationProject proj
 				builder.BuildLogFile = logName;
 				Assert.True (builder.RunTarget (proj, "_Run", doNotCleanupOnUpdate: doNotCleanupOnUpdate, parameters: parameters), "Project should have run.");
 			} else {
-				var result = AdbStartActivity ($"{proj.PackageName}/{proj.JavaPackageName}.MainActivity");
-				Assert.IsTrue (result.Contains ("Starting: Intent { cmp="), $"Attempt to start activity failed with:\n{result}");
+				StartActivityAndAssert (proj);
 			}
 		}
 
+		protected static void StartActivityAndAssert (XamarinAndroidApplicationProject proj)
+		{
+			var result = AdbStartActivity ($"{proj.PackageName}/{proj.JavaPackageName}.MainActivity");
+			Assert.IsTrue (result.Contains ("Starting: Intent { cmp="), $"Attempt to start activity failed with:\n{result}");
+		}
+
 		protected TimeSpan ProfileFor (Func<bool> func, TimeSpan? timeout = null)
 		{
 			var stopwatch = new Stopwatch ();

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/DeviceTest.cs

@@ -3,6 +3,7 @@
 using System.Globalization;
 using System.IO;
 using System.Linq;
+using System.Threading;
 using Mono.Unix.Native;
 using NUnit.Framework;
 using NUnit.Framework.Interfaces;
@@ -16,8 +17,8 @@ namespace Xamarin.Android.Build.Tests
 	[NonParallelizable]
 	public class LocalizationTests : DeviceTest
 	{
-		static ProjectBuilder builder;
-		static XamarinAndroidApplicationProject proj;
+		ProjectBuilder builder;
+		XamarinAndroidApplicationProject proj;
 		string localeFileSuffix;
 
 		[OneTimeSetUp]
@@ -60,19 +61,6 @@ public override void SetupTest ()
 			if (!string.IsNullOrEmpty (localeParam)) {
 				localeFileSuffix = localeParam.Replace ("/", "-");
 			}
-
-			if (!IsDeviceAttached (refreshCachedValue: true)) {
-				RestartDevice ();
-				AssertHasDevices ();
-			}
-
-			// Attempt to reinstall the app that was installed during fixture setup if it is missing
-			var packageOutput = RunAdbCommand ($"shell pm list packages {proj.PackageName}").Trim ();
-			var expectedPackageOutput = $"package:{proj.PackageName}";
-			if (packageOutput != expectedPackageOutput) {
-				builder.BuildLogFile = $"setup-install-{localeFileSuffix}.log";
-				Assert.IsTrue (builder.Install (proj), "Install should have succeeded.");
-			}
 		}
 
 		/// <summary>
@@ -93,27 +81,22 @@ protected override void CleanupTest ()
 			}
 		}
 
+		protected override void DeviceTearDown ()
+		{
+		}
+
 		[OneTimeTearDown]
-		public void AfterAllTests ()
+		protected override void AfterAllTests ()
 		{
-			string output = Path.Combine (Root, builder?.ProjectDirectory);
-			if (TestContext.CurrentContext.Result.FailCount == 0 && Directory.Exists (output)) {
-				try {
-					Directory.Delete (output, recursive: true);
-				} catch (IOException ex) {
-					// This happens on CI occasionally, let's not fail the test
-					TestContext.Out.WriteLine ($"Failed to delete '{output}': {ex}");
-				}
-			}
 		}
 
 
 		const int LOCALIZATION_NODE_COUNT = 15;
 		const int LOCALIZATION_RETRY_COUNT = 3;
 
-		static object [] GetLocalizationTestCases (int node)
+		static List<string> GetLocalizationTestInfo ()
 		{
-			List<object> tests = new List<object> ();
+			var tests = new List<string> ();
 			var ignore = new string [] {
 				"he-IL", // maps to wi-IL on Android.
 				"id-ID", // maps to in-ID on Android
@@ -127,11 +110,14 @@ static object [] GetLocalizationTestCases (int node)
 					TestContext.WriteLine ($"Ignoring {ci.Name} Localization Test");
 					continue;
 				}
-				tests.Add (new object [] {
-					ci.Name,
-				});
+				tests.Add (ci.Name);
 			}
+			return tests;
+		}
 
+		static object [] GetLocalizationTestCases (int node)
+		{
+			var tests = GetLocalizationTestInfo ();
 			return tests.Where (p => tests.IndexOf (p) % LOCALIZATION_NODE_COUNT == node).ToArray ();
 		}
 
@@ -210,41 +196,56 @@ static object [] GetLocalizationTestCases (int node)
 		[TestCaseSource (nameof (GetLocalizationTestCases), new object [] { 14 })]
 		public void CheckLocalizationIsCorrectNode15 (string locale) => CheckLocalizationIsCorrect (locale);
 
+		[Test]
+		[Retry (LOCALIZATION_RETRY_COUNT)]
+		[TestCaseSource (nameof (GetLocalizationTestInfo))]
+		public void CheckLocalizationIsCorrectWithSlicer (string locale) => CheckLocalizationIsCorrect (locale);
+
 
 		public void CheckLocalizationIsCorrect (string locale)
 		{
+			AssertHasDevices ();
+
+			// Attempt to reinstall the app that was installed during fixture setup if it is missing
+			var packageOutput = RunAdbCommand ($"shell pm list packages {proj.PackageName}").Trim ();
+			var expectedPackageOutput = $"package:{proj.PackageName}";
+			if (packageOutput != expectedPackageOutput) {
+				builder.BuildLogFile = $"setup-install-{localeFileSuffix}.log";
+				Assert.IsTrue (builder.Install (proj), "Install should have succeeded.");
+			}
+
+			RunAdbCommand ($"shell am force-stop --user all {proj.PackageName}");
+			RunAdbCommand ($"shell am kill --user all {proj.PackageName}");
+
 			var appStartupLogcatFile = Path.Combine (Root, builder.ProjectDirectory, $"startup-logcat-{locale.Replace ("/", "-")}.log");
 			string deviceLocale = RunAdbCommand ("shell getprop persist.sys.locale")?.Trim ();
 			TestContext.Out.WriteLine ($"test value:{locale}, prop value:{deviceLocale}");
 
-			if (deviceLocale != locale) {
-				for (int attempt = 0; attempt < 5; attempt++) {
-					TestContext.Out.WriteLine ($"{nameof(CheckLocalizationIsCorrect)}: Setting Locale to {locale}, attempt {attempt}...");
-					ClearAdbLogcat ();
-					var rebootLogcatFile = Path.Combine (Root, builder.ProjectDirectory, $"reboot{attempt}-logcat-{locale.Replace ("/", "-")}.log");
-
-					// https://developer.android.com/guide/topics/resources/localization#changing-the-emulator-locale-from-the-adb-shell
-					RunAdbCommand ($"shell \"su root setprop persist.sys.locale {locale};su root stop;sleep 5;su root start;\"");
-
-					if (!MonitorAdbLogcat ((l) => {
-						if (l.Contains ("ActivityManager: Finished processing BOOT_COMPLETED"))
-							return true;
-						return false;
-					}, rebootLogcatFile, timeout: 60)) {
-						TestContext.Out.WriteLine ($"{nameof(CheckLocalizationIsCorrect)}: wating for boot to complete failed or timed out.");
-					}
-					deviceLocale = RunAdbCommand ("shell getprop persist.sys.locale")?.Trim ();
-					if (deviceLocale == locale) {
-						break;
-					}
+			for (int attempt = 0; attempt < 5; attempt++) {
+				TestContext.Out.WriteLine ($"{nameof(CheckLocalizationIsCorrect)}: Setting Locale to {locale}, attempt {attempt}...");
+				ClearAdbLogcat ();
+				var rebootLogcatFile = Path.Combine (Root, builder.ProjectDirectory, $"reboot{attempt}-logcat-{locale.Replace ("/", "-")}.log");
+
+				// https://developer.android.com/guide/topics/resources/localization#changing-the-emulator-locale-from-the-adb-shell
+				RunAdbCommand ($"shell \"su root setprop persist.sys.locale {locale};su root stop;sleep 5;su root start;\"");
+
+				if (!MonitorAdbLogcat ((l) => {
+					if (l.Contains ("ActivityManager: Finished processing BOOT_COMPLETED"))
+						return true;
+					return false;
+				}, rebootLogcatFile, timeout: 60)) {
+					TestContext.Out.WriteLine ($"{nameof(CheckLocalizationIsCorrect)}: wating for boot to complete failed or timed out.");
+				}
+				deviceLocale = RunAdbCommand ("shell getprop persist.sys.locale")?.Trim ();
+				if (deviceLocale == locale) {
+					break;
 				}
 			}
 
 			Assert.AreEqual (locale, deviceLocale, $"The command to set the device locale to {locale} failed. Current device locale is {deviceLocale}");
 			ClearAdbLogcat ();
-			RunAdbCommand ($"shell am force-stop --user all {proj.PackageName}");
-			RunAdbCommand ($"shell am kill --user all {proj.PackageName}");
-			RunProjectAndAssert (proj, builder, logName: $"run-{locale.Replace ("/", "-")}.log");
+			Thread.Sleep (1000);
+			StartActivityAndAssert (proj);
 
 			string logcatSearchString = "Strings.SomeString=";
 			string expectedLogcatOutput = $"{logcatSearchString}{locale}";
@@ -258,7 +259,6 @@ public void CheckLocalizationIsCorrect (string locale)
 				return false;
 			}, appStartupLogcatFile, 45), $"App output did not contain '{logcatSearchString}'");
 			Assert.IsTrue (logLine.Contains (expectedLogcatOutput), $"Line '{logLine}' did not contain '{expectedLogcatOutput}'");
-
 			string humanizerLogCatFile = Path.Combine (Root, builder.ProjectDirectory, $"humanizer-logcat-{locale.Replace ("/", "-")}.log");
 			var culture = new CultureInfo (locale);
 			expectedLogcatOutput = DateTime.UtcNow.AddHours(-30).Humanize(culture: culture);
@@ -271,7 +271,6 @@ public void CheckLocalizationIsCorrect (string locale)
 				return false;
 			}, humanizerLogCatFile, timeout:45), $"App output did not contain '{logcatSearchString}'");
 			Assert.IsTrue (logLine.Contains (expectedLogcatOutput), $"Line '{logLine}' did not contain '{expectedLogcatOutput}'");
-
 		}
 	}
 }