diff --git a/build-tools/automation/azure-pipelines-nightly.yaml b/build-tools/automation/azure-pipelines-nightly.yaml index 029a3992395..be6442f7253 100644 --- a/build-tools/automation/azure-pipelines-nightly.yaml +++ b/build-tools/automation/azure-pipelines-nightly.yaml @@ -170,155 +170,44 @@ stages: - group: Xamarin-Secrets - group: xamops-azdev-secrets jobs: - - template: yaml-templates/run-emulator-tests.yaml - parameters: - jobName: TimeZoneInfoTests1 - emulatorMSBuildArgs: -p:TestAvdShowWindow=true - testSteps: - - template: run-nunit-tests.yaml - parameters: - testRunTitle: CheckTimeZoneInfoIsCorrectNode1 On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name=CheckTimeZoneInfoIsCorrectNode1" - testResultsFile: TestResult-CheckTimeZoneInfoIsCorrectNode1-$(XA.Build.Configuration).xml - timeoutInMinutes: 75 - retryCountOnTaskFailure: 2 - - template: run-nunit-tests.yaml - parameters: - testRunTitle: CheckTimeZoneInfoIsCorrectNode2 On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name=CheckTimeZoneInfoIsCorrectNode2" - testResultsFile: TestResult-CheckTimeZoneInfoIsCorrectNode2-$(XA.Build.Configuration).xml - timeoutInMinutes: 75 - retryCountOnTaskFailure: 2 - - template: run-nunit-tests.yaml - parameters: - testRunTitle: CheckTimeZoneInfoIsCorrectNode3 On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name=CheckTimeZoneInfoIsCorrectNode3" - testResultsFile: TestResult-CheckTimeZoneInfoIsCorrectNode3-$(XA.Build.Configuration).xml - timeoutInMinutes: 75 - retryCountOnTaskFailure: 2 + - job: mac_timezoneinfo_tests + displayName: TimeZoneInfoTests Emulator Tests + strategy: + parallel: 4 + pool: + vmImage: $(HostedMacImage) + timeoutInMinutes: 120 + workspace: + clean: all + steps: + - template: yaml-templates/setup-test-environment.yaml + parameters: + installTestSlicer: true + installLegacyDotNet: false + restoreNUnitConsole: false + updateMono: false + xaprepareScenario: EmulatorTestDependencies - - template: yaml-templates/run-emulator-tests.yaml - parameters: - jobName: TimeZoneInfoTests2 - emulatorMSBuildArgs: -p:TestAvdShowWindow=true - testSteps: - - template: run-nunit-tests.yaml - parameters: - testRunTitle: CheckTimeZoneInfoIsCorrectNode4 On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name=CheckTimeZoneInfoIsCorrectNode4" - testResultsFile: TestResult-CheckTimeZoneInfoIsCorrectNode4-$(XA.Build.Configuration).xml - timeoutInMinutes: 75 - retryCountOnTaskFailure: 2 - - template: run-nunit-tests.yaml - parameters: - testRunTitle: CheckTimeZoneInfoIsCorrectNode5 On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name=CheckTimeZoneInfoIsCorrectNode5" - testResultsFile: TestResult-CheckTimeZoneInfoIsCorrectNode5-$(XA.Build.Configuration).xml - timeoutInMinutes: 75 - retryCountOnTaskFailure: 2 - - template: run-nunit-tests.yaml - parameters: - testRunTitle: CheckTimeZoneInfoIsCorrectNode6 On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name=CheckTimeZoneInfoIsCorrectNode6" - testResultsFile: TestResult-CheckTimeZoneInfoIsCorrectNode6-$(XA.Build.Configuration).xml - timeoutInMinutes: 75 - retryCountOnTaskFailure: 2 + - task: DownloadPipelineArtifact@2 + inputs: + artifactName: $(TestAssembliesArtifactName) + downloadPath: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration) - - template: yaml-templates/run-emulator-tests.yaml - parameters: - jobName: TimeZoneInfoTests3 - emulatorMSBuildArgs: -p:TestAvdShowWindow=true - testSteps: - - template: run-nunit-tests.yaml - parameters: - testRunTitle: CheckTimeZoneInfoIsCorrectNode7 On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name=CheckTimeZoneInfoIsCorrectNode7" - testResultsFile: TestResult-CheckTimeZoneInfoIsCorrectNode7-$(XA.Build.Configuration).xml - timeoutInMinutes: 75 - retryCountOnTaskFailure: 2 - - template: run-nunit-tests.yaml - parameters: - testRunTitle: CheckTimeZoneInfoIsCorrectNode8 On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name=CheckTimeZoneInfoIsCorrectNode8" - testResultsFile: TestResult-CheckTimeZoneInfoIsCorrectNode8-$(XA.Build.Configuration).xml - timeoutInMinutes: 75 - retryCountOnTaskFailure: 2 - - template: run-nunit-tests.yaml - parameters: - testRunTitle: CheckTimeZoneInfoIsCorrectNode9 On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name=CheckTimeZoneInfoIsCorrectNode9" - testResultsFile: TestResult-CheckTimeZoneInfoIsCorrectNode9-$(XA.Build.Configuration).xml - timeoutInMinutes: 75 - retryCountOnTaskFailure: 2 + - template: yaml-templates/start-stop-emulator.yaml + parameters: + emulatorMSBuildArgs: -p:TestAvdShowWindow=true - - template: yaml-templates/run-emulator-tests.yaml - parameters: - jobName: TimeZoneInfoTests4 - emulatorMSBuildArgs: -p:TestAvdShowWindow=true - testSteps: - - template: run-nunit-tests.yaml - parameters: - testRunTitle: CheckTimeZoneInfoIsCorrectNode10 On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name=CheckTimeZoneInfoIsCorrectNode10" - testResultsFile: TestResult-CheckTimeZoneInfoIsCorrectNode10-$(XA.Build.Configuration).xml - timeoutInMinutes: 75 - retryCountOnTaskFailure: 2 - - template: run-nunit-tests.yaml - parameters: - testRunTitle: CheckTimeZoneInfoIsCorrectNode11 On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name=CheckTimeZoneInfoIsCorrectNode11" - testResultsFile: TestResult-CheckTimeZoneInfoIsCorrectNode11-$(XA.Build.Configuration).xml - timeoutInMinutes: 75 - retryCountOnTaskFailure: 2 - - template: run-nunit-tests.yaml - parameters: - testRunTitle: CheckTimeZoneInfoIsCorrectNode12 On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name=CheckTimeZoneInfoIsCorrectNode12" - testResultsFile: TestResult-CheckTimeZoneInfoIsCorrectNode12-$(XA.Build.Configuration).xml - timeoutInMinutes: 75 - retryCountOnTaskFailure: 2 + - template: yaml-templates/run-sliced-nunit-tests.yaml + parameters: + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + testFilter: method == CheckTimeZoneInfoIsCorrectWithSlicer + testRunTitle: CheckTimeZoneInfoIsCorrectNode On Device - macOS - - template: yaml-templates/run-emulator-tests.yaml - parameters: - jobName: TimeZoneInfoTests5 - emulatorMSBuildArgs: -p:TestAvdShowWindow=true - testSteps: - - template: run-nunit-tests.yaml - parameters: - testRunTitle: CheckTimeZoneInfoIsCorrectNode13 On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name=CheckTimeZoneInfoIsCorrectNode13" - testResultsFile: TestResult-CheckTimeZoneInfoIsCorrectNode13-$(XA.Build.Configuration).xml - timeoutInMinutes: 75 - retryCountOnTaskFailure: 2 - - template: run-nunit-tests.yaml - parameters: - testRunTitle: CheckTimeZoneInfoIsCorrectNode14 On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name=CheckTimeZoneInfoIsCorrectNode14" - testResultsFile: TestResult-CheckTimeZoneInfoIsCorrectNode14-$(XA.Build.Configuration).xml - timeoutInMinutes: 75 - retryCountOnTaskFailure: 2 - - template: run-nunit-tests.yaml - parameters: - testRunTitle: CheckTimeZoneInfoIsCorrectNode15 On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name=CheckTimeZoneInfoIsCorrectNode15" - testResultsFile: TestResult-CheckTimeZoneInfoIsCorrectNode15-$(XA.Build.Configuration).xml - timeoutInMinutes: 75 - retryCountOnTaskFailure: 2 + - template: yaml-templates/upload-results.yaml + parameters: + artifactName: Test Results - TimeZoneInfoTests With Emulator - macOS-$(System.JobPositionInPhase) + + - template: yaml-templates/fail-on-issue.yaml # Localization test jobs @@ -329,137 +218,41 @@ stages: - group: Xamarin-Secrets - group: xamops-azdev-secrets jobs: - - template: yaml-templates/run-emulator-tests.yaml - parameters: - jobName: LocalizationTests1 - emulatorMSBuildArgs: -p:TestAvdShowWindow=true - testSteps: - - template: run-nunit-tests.yaml - parameters: - testRunTitle: CheckLocalizationIsCorrectNode1 On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name=CheckLocalizationIsCorrectNode1" - testResultsFile: TestResult-CheckLocalizationIsCorrectNode1-$(XA.Build.Configuration).xml - retryCountOnTaskFailure: 2 - - template: run-nunit-tests.yaml - parameters: - testRunTitle: CheckLocalizationIsCorrectNode2 On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name=CheckLocalizationIsCorrectNode2" - testResultsFile: TestResult-CheckLocalizationIsCorrectNode2-$(XA.Build.Configuration).xml - retryCountOnTaskFailure: 2 - - template: run-nunit-tests.yaml - parameters: - testRunTitle: CheckLocalizationIsCorrectNode3 On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name=CheckLocalizationIsCorrectNode3" - testResultsFile: TestResult-CheckLocalizationIsCorrectNode3-$(XA.Build.Configuration).xml - retryCountOnTaskFailure: 2 + - job: mac_localization_tests + displayName: Localization Emulator Tests + strategy: + parallel: 10 + pool: + vmImage: $(HostedMacImage) + timeoutInMinutes: 150 + workspace: + clean: all + steps: + - template: yaml-templates/setup-test-environment.yaml + parameters: + installTestSlicer: true + installLegacyDotNet: false + restoreNUnitConsole: false + updateMono: false + xaprepareScenario: EmulatorTestDependencies - - template: yaml-templates/run-emulator-tests.yaml - parameters: - jobName: LocalizationTests2 - emulatorMSBuildArgs: -p:TestAvdShowWindow=true - testSteps: - - template: run-nunit-tests.yaml - parameters: - testRunTitle: CheckLocalizationIsCorrectNode4 On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name=CheckLocalizationIsCorrectNode4" - testResultsFile: TestResult-CheckLocalizationIsCorrectNode4-$(XA.Build.Configuration).xml - retryCountOnTaskFailure: 2 - - template: run-nunit-tests.yaml - parameters: - testRunTitle: CheckLocalizationIsCorrectNode5 On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name=CheckLocalizationIsCorrectNode5" - testResultsFile: TestResult-CheckLocalizationIsCorrectNode5-$(XA.Build.Configuration).xml - retryCountOnTaskFailure: 2 - - template: run-nunit-tests.yaml - parameters: - testRunTitle: CheckLocalizationIsCorrectNode6 On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name=CheckLocalizationIsCorrectNode6" - testResultsFile: TestResult-CheckLocalizationIsCorrectNode6-$(XA.Build.Configuration).xml - retryCountOnTaskFailure: 2 + - task: DownloadPipelineArtifact@2 + inputs: + artifactName: $(TestAssembliesArtifactName) + downloadPath: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration) - - template: yaml-templates/run-emulator-tests.yaml - parameters: - jobName: LocalizationTests3 - emulatorMSBuildArgs: -p:TestAvdShowWindow=true - testSteps: - - template: run-nunit-tests.yaml - parameters: - testRunTitle: CheckLocalizationIsCorrectNode7 On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name=CheckLocalizationIsCorrectNode7" - testResultsFile: TestResult-CheckLocalizationIsCorrectNode7-$(XA.Build.Configuration).xml - retryCountOnTaskFailure: 2 - - template: run-nunit-tests.yaml - parameters: - testRunTitle: CheckLocalizationIsCorrectNode8 On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name=CheckLocalizationIsCorrectNode8" - testResultsFile: TestResult-CheckLocalizationIsCorrectNode8-$(XA.Build.Configuration).xml - retryCountOnTaskFailure: 2 - - template: run-nunit-tests.yaml - parameters: - testRunTitle: CheckLocalizationIsCorrectNode9 On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name=CheckLocalizationIsCorrectNode9" - testResultsFile: TestResult-CheckLocalizationIsCorrectNode9-$(XA.Build.Configuration).xml - retryCountOnTaskFailure: 2 + - template: yaml-templates/start-stop-emulator.yaml + parameters: + emulatorMSBuildArgs: -p:TestAvdShowWindow=true - - template: yaml-templates/run-emulator-tests.yaml - parameters: - jobName: LocalizationTests4 - emulatorMSBuildArgs: -p:TestAvdShowWindow=true - testSteps: - - template: run-nunit-tests.yaml - parameters: - testRunTitle: CheckLocalizationIsCorrectNode10 On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name=CheckLocalizationIsCorrectNode10" - testResultsFile: TestResult-CheckLocalizationIsCorrectNode10-$(XA.Build.Configuration).xml - retryCountOnTaskFailure: 2 - - template: run-nunit-tests.yaml - parameters: - testRunTitle: CheckLocalizationIsCorrectNode11 On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name=CheckLocalizationIsCorrectNode11" - testResultsFile: TestResult-CheckLocalizationIsCorrectNode11-$(XA.Build.Configuration).xml - retryCountOnTaskFailure: 2 - - template: run-nunit-tests.yaml - parameters: - testRunTitle: CheckLocalizationIsCorrectNode12 On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name=CheckLocalizationIsCorrectNode12" - testResultsFile: TestResult-CheckLocalizationIsCorrectNode12-$(XA.Build.Configuration).xml - retryCountOnTaskFailure: 2 + - template: yaml-templates/run-sliced-nunit-tests.yaml + parameters: + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll + testFilter: method == CheckLocalizationIsCorrectWithSlicer + testRunTitle: CheckTimeZoneInfoIsCorrect On Device - macOS - - template: yaml-templates/run-emulator-tests.yaml - parameters: - jobName: LocalizationTests5 - emulatorMSBuildArgs: -p:TestAvdShowWindow=true - testSteps: - - template: run-nunit-tests.yaml - parameters: - testRunTitle: CheckLocalizationIsCorrectNode13 On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name=CheckLocalizationIsCorrectNode13" - testResultsFile: TestResult-CheckLocalizationIsCorrectNode13-$(XA.Build.Configuration).xml - retryCountOnTaskFailure: 2 - - template: run-nunit-tests.yaml - parameters: - testRunTitle: CheckLocalizationIsCorrectNode14 On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name=CheckLocalizationIsCorrectNode14" - testResultsFile: TestResult-CheckLocalizationIsCorrectNode14-$(XA.Build.Configuration).xml - retryCountOnTaskFailure: 2 - - template: run-nunit-tests.yaml - parameters: - testRunTitle: CheckLocalizationIsCorrectNode15 On Device - macOS - testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - dotNetTestExtraArgs: --filter "Name=CheckLocalizationIsCorrectNode15" - testResultsFile: TestResult-CheckLocalizationIsCorrectNode15-$(XA.Build.Configuration).xml - retryCountOnTaskFailure: 2 + - template: yaml-templates/upload-results.yaml + parameters: + artifactName: Test Results - Localization With Emulator - macOS-$(System.JobPositionInPhase) + + - template: yaml-templates/fail-on-issue.yaml diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/CodeBehindTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/CodeBehindTests.cs index c47c0a169ee..a64a11f7bce 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/CodeBehindTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/CodeBehindTests.cs @@ -152,7 +152,7 @@ static CodeBehindTests () { TestProjectRootDirectory = Path.GetFullPath (Path.Combine (XABuildPaths.TopDirectory, "tests", "CodeBehind", "BuildTests")); CommonSampleLibraryRootDirectory = Path.GetFullPath (Path.Combine (XABuildPaths.TopDirectory, "tests", "CodeBehind", CommonSampleLibraryName)); - TestOutputDir = Path.Combine (SetUp.TestDirectoryRoot, "temp", "CodeBehind"); + TestOutputDir = Path.Combine (XABuildPaths.TestOutputDirectory, "temp", "CodeBehind"); if (Builder.UseDotNet) { ProjectName += ".NET"; } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/EmbeddedDSOTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/EmbeddedDSOTests.cs index d11cc65c818..714f88a9449 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/EmbeddedDSOTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/EmbeddedDSOTests.cs @@ -57,7 +57,7 @@ public bool Build (string projectOrSolution, string target, string [] parameters static EmbeddedDSOTests () { TestProjectRootDirectory = Path.GetFullPath (Path.Combine (XABuildPaths.TopDirectory, "tests", "EmbeddedDSOs", "EmbeddedDSO")); - TestOutputDir = Path.Combine (SetUp.TestDirectoryRoot, "temp", "EmbeddedDSO"); + TestOutputDir = Path.Combine (XABuildPaths.TestOutputDirectory, "temp", "EmbeddedDSO"); produced_binaries = new List { $"{ProjectAssemblyName}.dll", diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/BaseTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/BaseTest.cs index ee3ad36fa0a..e753a25b41b 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/BaseTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/BaseTest.cs @@ -23,58 +23,9 @@ public class BaseTest public static ConcurrentDictionary TestOutputDirectories = new ConcurrentDictionary (); public static ConcurrentDictionary TestPackageNames = new ConcurrentDictionary (); - [SetUpFixture] - public class SetUp - { - public static string TestDirectoryRoot { - get; - private set; - } - - [OneTimeSetUp] - public void BeforeAllTests () - { - TestDirectoryRoot = XABuildPaths.TestOutputDirectory; - } - - [OneTimeTearDown] - public void AfterAllTests () - { - if (System.Diagnostics.Debugger.IsAttached) - return; - - //NOTE: adb.exe can cause a couple issues on Windows - // 1) it holds a lock on ~/android-toolchain, so a future build that needs to delete/recreate would fail - // 2) the MSBuild task *can* hang until adb.exe exits - - try { - RunAdbCommand ("kill-server", true); - } catch (Exception ex) { - Console.Error.WriteLine ("Failed to run adb kill-server: " + ex); - } - - //NOTE: in case `adb kill-server` fails, kill the process as a last resort - foreach (var p in Process.GetProcessesByName ("adb.exe")) - p.Kill (); - } - - } - protected bool IsWindows => TestEnvironment.IsWindows; - protected bool IsMacOS => TestEnvironment.IsMacOS; - - protected bool IsLinux => TestEnvironment.IsLinux; - - public string StagingPath { - get { return Environment.GetFolderPath (Environment.SpecialFolder.MyDocuments); } - } - - public string Root { - get { - return Path.GetFullPath (SetUp.TestDirectoryRoot); - } - } + public string Root => Path.GetFullPath (XABuildPaths.TestOutputDirectory); public static bool CommercialBuildAvailable => TestEnvironment.CommercialBuildAvailable; @@ -569,6 +520,28 @@ public void TestSetup () TestContext.Out.Flush (); } + [OneTimeTearDown] + protected virtual void AfterAllTests () + { + if (System.Diagnostics.Debugger.IsAttached) + return; + + //NOTE: adb.exe can cause a couple issues on Windows + // 1) it holds a lock on ~/android-toolchain, so a future build that needs to delete/recreate would fail + // 2) the MSBuild task *can* hang until adb.exe exits + if (IsWindows) { + try { + RunAdbCommand ("kill-server", true); + } catch (Exception ex) { + Console.Error.WriteLine ("Failed to run adb kill-server: " + ex); + } + + //NOTE: in case `adb kill-server` fails, kill the process as a last resort + foreach (var p in Process.GetProcessesByName ("adb.exe")) + p.Kill (); + } + } + [TearDown] protected virtual void CleanupTest () { diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/DeviceTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/DeviceTest.cs index 0e5d0ed3909..0184ef1d425 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/DeviceTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/DeviceTest.cs @@ -73,7 +73,7 @@ public void DeviceSetup () } [OneTimeTearDown] - public void DeviceTearDown () + protected virtual void DeviceTearDown () { if (IsDeviceAttached ()) { // make sure we are not on a guest user anymore. @@ -199,11 +199,16 @@ protected static void RunProjectAndAssert (XamarinAndroidApplicationProject proj builder.BuildLogFile = logName; Assert.True (builder.RunTarget (proj, "_Run", doNotCleanupOnUpdate: doNotCleanupOnUpdate, parameters: parameters), "Project should have run."); } else { - var result = AdbStartActivity ($"{proj.PackageName}/{proj.JavaPackageName}.MainActivity"); - Assert.IsTrue (result.Contains ("Starting: Intent { cmp="), $"Attempt to start activity failed with:\n{result}"); + StartActivityAndAssert (proj); } } + protected static void StartActivityAndAssert (XamarinAndroidApplicationProject proj) + { + var result = AdbStartActivity ($"{proj.PackageName}/{proj.JavaPackageName}.MainActivity"); + Assert.IsTrue (result.Contains ("Starting: Intent { cmp="), $"Attempt to start activity failed with:\n{result}"); + } + protected TimeSpan ProfileFor (Func func, TimeSpan? timeout = null) { var stopwatch = new Stopwatch (); diff --git a/tests/MSBuildDeviceIntegration/Tests/LocalizationTests.cs b/tests/MSBuildDeviceIntegration/Tests/LocalizationTests.cs index 0f58f12ba19..647bfee8024 100644 --- a/tests/MSBuildDeviceIntegration/Tests/LocalizationTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/LocalizationTests.cs @@ -3,6 +3,7 @@ using System.Globalization; using System.IO; using System.Linq; +using System.Threading; using Mono.Unix.Native; using NUnit.Framework; using NUnit.Framework.Interfaces; @@ -16,8 +17,8 @@ namespace Xamarin.Android.Build.Tests [NonParallelizable] public class LocalizationTests : DeviceTest { - static ProjectBuilder builder; - static XamarinAndroidApplicationProject proj; + ProjectBuilder builder; + XamarinAndroidApplicationProject proj; string localeFileSuffix; [OneTimeSetUp] @@ -60,19 +61,6 @@ public override void SetupTest () if (!string.IsNullOrEmpty (localeParam)) { localeFileSuffix = localeParam.Replace ("/", "-"); } - - if (!IsDeviceAttached (refreshCachedValue: true)) { - RestartDevice (); - AssertHasDevices (); - } - - // Attempt to reinstall the app that was installed during fixture setup if it is missing - var packageOutput = RunAdbCommand ($"shell pm list packages {proj.PackageName}").Trim (); - var expectedPackageOutput = $"package:{proj.PackageName}"; - if (packageOutput != expectedPackageOutput) { - builder.BuildLogFile = $"setup-install-{localeFileSuffix}.log"; - Assert.IsTrue (builder.Install (proj), "Install should have succeeded."); - } } /// @@ -93,27 +81,22 @@ protected override void CleanupTest () } } + protected override void DeviceTearDown () + { + } + [OneTimeTearDown] - public void AfterAllTests () + protected override void AfterAllTests () { - string output = Path.Combine (Root, builder?.ProjectDirectory); - if (TestContext.CurrentContext.Result.FailCount == 0 && Directory.Exists (output)) { - try { - Directory.Delete (output, recursive: true); - } catch (IOException ex) { - // This happens on CI occasionally, let's not fail the test - TestContext.Out.WriteLine ($"Failed to delete '{output}': {ex}"); - } - } } const int LOCALIZATION_NODE_COUNT = 15; const int LOCALIZATION_RETRY_COUNT = 3; - static object [] GetLocalizationTestCases (int node) + static List GetLocalizationTestInfo () { - List tests = new List (); + var tests = new List (); var ignore = new string [] { "he-IL", // maps to wi-IL on Android. "id-ID", // maps to in-ID on Android @@ -127,11 +110,14 @@ static object [] GetLocalizationTestCases (int node) TestContext.WriteLine ($"Ignoring {ci.Name} Localization Test"); continue; } - tests.Add (new object [] { - ci.Name, - }); + tests.Add (ci.Name); } + return tests; + } + static object [] GetLocalizationTestCases (int node) + { + var tests = GetLocalizationTestInfo (); return tests.Where (p => tests.IndexOf (p) % LOCALIZATION_NODE_COUNT == node).ToArray (); } @@ -210,41 +196,56 @@ static object [] GetLocalizationTestCases (int node) [TestCaseSource (nameof (GetLocalizationTestCases), new object [] { 14 })] public void CheckLocalizationIsCorrectNode15 (string locale) => CheckLocalizationIsCorrect (locale); + [Test] + [Retry (LOCALIZATION_RETRY_COUNT)] + [TestCaseSource (nameof (GetLocalizationTestInfo))] + public void CheckLocalizationIsCorrectWithSlicer (string locale) => CheckLocalizationIsCorrect (locale); + public void CheckLocalizationIsCorrect (string locale) { + AssertHasDevices (); + + // Attempt to reinstall the app that was installed during fixture setup if it is missing + var packageOutput = RunAdbCommand ($"shell pm list packages {proj.PackageName}").Trim (); + var expectedPackageOutput = $"package:{proj.PackageName}"; + if (packageOutput != expectedPackageOutput) { + builder.BuildLogFile = $"setup-install-{localeFileSuffix}.log"; + Assert.IsTrue (builder.Install (proj), "Install should have succeeded."); + } + + RunAdbCommand ($"shell am force-stop --user all {proj.PackageName}"); + RunAdbCommand ($"shell am kill --user all {proj.PackageName}"); + var appStartupLogcatFile = Path.Combine (Root, builder.ProjectDirectory, $"startup-logcat-{locale.Replace ("/", "-")}.log"); string deviceLocale = RunAdbCommand ("shell getprop persist.sys.locale")?.Trim (); TestContext.Out.WriteLine ($"test value:{locale}, prop value:{deviceLocale}"); - if (deviceLocale != locale) { - for (int attempt = 0; attempt < 5; attempt++) { - TestContext.Out.WriteLine ($"{nameof(CheckLocalizationIsCorrect)}: Setting Locale to {locale}, attempt {attempt}..."); - ClearAdbLogcat (); - var rebootLogcatFile = Path.Combine (Root, builder.ProjectDirectory, $"reboot{attempt}-logcat-{locale.Replace ("/", "-")}.log"); - - // https://developer.android.com/guide/topics/resources/localization#changing-the-emulator-locale-from-the-adb-shell - RunAdbCommand ($"shell \"su root setprop persist.sys.locale {locale};su root stop;sleep 5;su root start;\""); - - if (!MonitorAdbLogcat ((l) => { - if (l.Contains ("ActivityManager: Finished processing BOOT_COMPLETED")) - return true; - return false; - }, rebootLogcatFile, timeout: 60)) { - TestContext.Out.WriteLine ($"{nameof(CheckLocalizationIsCorrect)}: wating for boot to complete failed or timed out."); - } - deviceLocale = RunAdbCommand ("shell getprop persist.sys.locale")?.Trim (); - if (deviceLocale == locale) { - break; - } + for (int attempt = 0; attempt < 5; attempt++) { + TestContext.Out.WriteLine ($"{nameof(CheckLocalizationIsCorrect)}: Setting Locale to {locale}, attempt {attempt}..."); + ClearAdbLogcat (); + var rebootLogcatFile = Path.Combine (Root, builder.ProjectDirectory, $"reboot{attempt}-logcat-{locale.Replace ("/", "-")}.log"); + + // https://developer.android.com/guide/topics/resources/localization#changing-the-emulator-locale-from-the-adb-shell + RunAdbCommand ($"shell \"su root setprop persist.sys.locale {locale};su root stop;sleep 5;su root start;\""); + + if (!MonitorAdbLogcat ((l) => { + if (l.Contains ("ActivityManager: Finished processing BOOT_COMPLETED")) + return true; + return false; + }, rebootLogcatFile, timeout: 60)) { + TestContext.Out.WriteLine ($"{nameof(CheckLocalizationIsCorrect)}: wating for boot to complete failed or timed out."); + } + deviceLocale = RunAdbCommand ("shell getprop persist.sys.locale")?.Trim (); + if (deviceLocale == locale) { + break; } } Assert.AreEqual (locale, deviceLocale, $"The command to set the device locale to {locale} failed. Current device locale is {deviceLocale}"); ClearAdbLogcat (); - RunAdbCommand ($"shell am force-stop --user all {proj.PackageName}"); - RunAdbCommand ($"shell am kill --user all {proj.PackageName}"); - RunProjectAndAssert (proj, builder, logName: $"run-{locale.Replace ("/", "-")}.log"); + Thread.Sleep (1000); + StartActivityAndAssert (proj); string logcatSearchString = "Strings.SomeString="; string expectedLogcatOutput = $"{logcatSearchString}{locale}"; @@ -271,7 +272,6 @@ public void CheckLocalizationIsCorrect (string locale) return false; }, humanizerLogCatFile, timeout:45), $"App output did not contain '{logcatSearchString}'"); Assert.IsTrue (logLine.Contains (expectedLogcatOutput), $"Line '{logLine}' did not contain '{expectedLogcatOutput}'"); - } } } diff --git a/tests/MSBuildDeviceIntegration/Tests/TimeZoneInfoTests.cs b/tests/MSBuildDeviceIntegration/Tests/TimeZoneInfoTests.cs index cc25697cc69..d4e6033a063 100644 --- a/tests/MSBuildDeviceIntegration/Tests/TimeZoneInfoTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/TimeZoneInfoTests.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading; using NUnit.Framework; using NUnit.Framework.Interfaces; using Xamarin.ProjectTools; @@ -13,11 +14,15 @@ namespace Xamarin.Android.Build.Tests [NonParallelizable] public class TimeZoneInfoTests : DeviceTest { - static ProjectBuilder builder; + ProjectBuilder builder; + XamarinAndroidApplicationProject proj; + string tzFileSuffix; [OneTimeSetUp] public void BeforeAllTests () { + AssertHasDevices (); + string debuggable = RunAdbCommand ("shell getprop ro.debuggable"); if (debuggable != "1") { Assert.Fail ("TimeZoneInfoTests need to use `su root` and this device does not support that feature. Try using an emulator."); @@ -25,15 +30,23 @@ public void BeforeAllTests () // Disable auto timezone RunAdbCommand ("shell settings put global auto_time_zone 0"); + proj = new XamarinAndroidApplicationProject (packageName: "TimeZoneInfoTests"); + proj.MainActivity = proj.DefaultMainActivity.Replace ("//${AFTER_ONCREATE}", @"button.Text = $""TimeZoneInfo={TimeZoneInfo.Local.Id}""; + Console.WriteLine ($""TimeZoneInfoNative={Java.Util.TimeZone.Default.ID}""); + Console.WriteLine ($""TimeZoneInfoTests.TimeZoneInfo={TimeZoneInfo.Local.Id}""); +"); + builder = CreateApkBuilder (Path.Combine ("temp", "TimeZoneInfoTests")); + builder.BuildLogFile = "onetimesetup-install.log"; + Assert.IsTrue (builder.Install (proj), "Install should have succeeded."); } [SetUp] public override void SetupTest () { - if (!IsDeviceAttached (refreshCachedValue: true)) { - RestartDevice (); - AssertHasDevices (); + var tzParam = TestContext.CurrentContext.Test.Arguments[0] as string; + if (!string.IsNullOrEmpty (tzParam)) { + tzFileSuffix = tzParam.Replace ("/", "-"); } } @@ -44,40 +57,33 @@ public override void SetupTest () [TearDown] protected override void CleanupTest () { - var tzParam = TestContext.CurrentContext.Test.Arguments[0] as string; - if (!string.IsNullOrEmpty (tzParam)) { - tzParam = tzParam.Replace ("/", "-"); - } - string output = Path.Combine (Root, builder?.ProjectDirectory); if (TestContext.CurrentContext.Result.Outcome.Status == TestStatus.Failed && Directory.Exists (output)) { - foreach (var testFile in Directory.GetFiles (output, $"*{tzParam}*log", SearchOption.AllDirectories)) { + foreach (var setupFile in Directory.GetFiles (output, $"*onetimesetup*log", SearchOption.AllDirectories)) { + TestContext.AddTestAttachment (setupFile, Path.GetFileNameWithoutExtension (setupFile)); + } + foreach (var testFile in Directory.GetFiles (output, $"*{tzFileSuffix}*log", SearchOption.AllDirectories)) { TestContext.AddTestAttachment (testFile, Path.GetFileNameWithoutExtension (testFile)); } } } + protected override void DeviceTearDown () + { + } + [OneTimeTearDown] - public void AfterAllTests () + protected override void AfterAllTests () { - string output = Path.Combine (Root, builder?.ProjectDirectory); - if (TestContext.CurrentContext.Result.FailCount == 0 && Directory.Exists (output)) { - try { - Directory.Delete (output, recursive: true); - } catch (IOException ex) { - // This happens on CI occasionally, let's not fail the test - TestContext.Out.WriteLine ($"Failed to delete '{output}': {ex}"); - } - } } const int TIMEZONE_NODE_COUNT = 15; const int TIMEZONE_RETRY_COUNT = 3; - static object [] GetTimeZoneTestCases (int node) + static List GetTimeZoneTestInfo () { - List tests = new List (); + var tests = new List (); var ignore = new string [] { "Asia/Qostanay", "US/Pacific-New" @@ -88,10 +94,14 @@ static object [] GetTimeZoneTestCases (int node) TestContext.WriteLine ($"Ignoring {tz} TimeZone Test"); continue; } - tests.Add (new object [] { - tz, - }); + tests.Add (tz); } + return tests; + } + + static object [] GetTimeZoneTestCases (int node) + { + var tests = GetTimeZoneTestInfo (); return tests.Where (p => tests.IndexOf (p) % TIMEZONE_NODE_COUNT == node).ToArray (); } @@ -170,44 +180,46 @@ static object [] GetTimeZoneTestCases (int node) [TestCaseSource (nameof (GetTimeZoneTestCases), new object [] { 14 })] public void CheckTimeZoneInfoIsCorrectNode15 (string timeZone) => CheckTimeZoneInfoIsCorrect (timeZone); + [Test] + [Retry (TIMEZONE_RETRY_COUNT)] + [TestCaseSource (nameof (GetTimeZoneTestInfo))] + public void CheckTimeZoneInfoIsCorrectWithSlicer (string timeZone) => CheckTimeZoneInfoIsCorrect (timeZone); + public void CheckTimeZoneInfoIsCorrect (string timeZone) { - var proj = new XamarinAndroidApplicationProject (packageName: "TimeZoneInfoTests"); - if (Builder.UseDotNet) { - proj.MainActivity = proj.DefaultMainActivity.Replace ("//${AFTER_ONCREATE}", @"button.Text = $""TimeZoneInfo={TimeZoneInfo.Local.Id}""; - Console.WriteLine ($""TimeZoneInfoNative={Java.Util.TimeZone.Default.ID}""); - Console.WriteLine ($""TimeZoneInfoTests.TimeZoneInfo={TimeZoneInfo.Local.Id}""); -"); - } else { - proj.MainActivity = proj.DefaultMainActivity.Replace ("//${AFTER_ONCREATE}", @"button.Text = $""TimeZoneInfo={TimeZoneInfo.Local.DisplayName}""; - Console.WriteLine ($""TimeZoneInfoNative={Java.Util.TimeZone.Default.ID}""); - Console.WriteLine ($""TimeZoneInfoTests.TimeZoneInfo={TimeZoneInfo.Local.DisplayName}""); -"); + AssertHasDevices (); + + // Attempt to reinstall the app that was installed during fixture setup if it is missing + var packageOutput = RunAdbCommand ($"shell pm list packages {proj.PackageName}").Trim (); + var expectedPackageOutput = $"package:{proj.PackageName}"; + if (packageOutput != expectedPackageOutput) { + builder.BuildLogFile = $"setup-install-{tzFileSuffix}.log"; + Assert.IsTrue (builder.Install (proj), "Install should have succeeded."); } + RunAdbCommand ($"shell am force-stop --user all {proj.PackageName}"); + RunAdbCommand ($"shell am kill --user all {proj.PackageName}"); + var appStartupLogcatFile = Path.Combine (Root, builder.ProjectDirectory, $"startup-logcat-{timeZone.Replace ("/", "-")}.log"); + RunAdbCommand ($"shell su root setprop persist.sys.timezone \"America/New_York\""); string deviceTz = RunAdbCommand ("shell getprop persist.sys.timezone")?.Trim (); - - if (deviceTz != timeZone) { - for (int attempt = 0; attempt < 5; attempt++) { - TestContext.Out.WriteLine ($"{nameof (CheckTimeZoneInfoIsCorrect)}: Setting TimeZone to {timeZone}, attempt {attempt}..."); - ClearAdbLogcat (); - RunAdbCommand ($"shell su root setprop persist.sys.timezone \"{timeZone}\""); - deviceTz = RunAdbCommand ("shell getprop persist.sys.timezone")?.Trim (); - if (deviceTz == timeZone) { - break; - } + TestContext.Out.WriteLine ($"test value:{timeZone}, prop value:{deviceTz}"); + + for (int attempt = 0; attempt < 5; attempt++) { + TestContext.Out.WriteLine ($"{nameof (CheckTimeZoneInfoIsCorrect)}: Setting TimeZone to {timeZone}, attempt {attempt}..."); + ClearAdbLogcat (); + RunAdbCommand ($"shell su root setprop persist.sys.timezone \"{timeZone}\""); + deviceTz = RunAdbCommand ("shell getprop persist.sys.timezone")?.Trim (); + if (deviceTz == timeZone) { + break; } } Assert.AreEqual (timeZone, deviceTz, $"The command to set the device timezone to {timeZone} failed. Current device timezone is {deviceTz}"); - builder.BuildLogFile = $"install-{timeZone.Replace ("/", "-")}.log"; - Assert.IsTrue (builder.Install (proj), "Install should have succeeded."); ClearAdbLogcat (); - RunAdbCommand ($"shell am force-stop --user all {proj.PackageName}"); - RunAdbCommand ($"shell am kill --user all {proj.PackageName}"); - RunProjectAndAssert (proj, builder, logName: $"run-{timeZone.Replace ("/", "-")}.log"); + Thread.Sleep (1000); + StartActivityAndAssert (proj); string logcatSearchString = "TimeZoneInfoTests.TimeZoneInfo="; string expectedLogcatOutput = $"{logcatSearchString}{timeZone}";