From e3fe64ac58efd121b12f0dc289001de6d0bf9215 Mon Sep 17 00:00:00 2001 From: Peter Collins Date: Thu, 23 Feb 2023 19:23:39 -0500 Subject: [PATCH] [ci] Run nunit tests with stable .NET version Attempt to improve desktop test execution reliability by running our test assemblies with a stable version of .NET by default. The default value of the `useDotNet` yaml template parameters have been flipped as part of continued efforts to remove classic test runs. .NET 7.0 SDK provisioning has been updated to install the latest GA version now that it is stable. --- .../automation/azure-pipelines-nightly.yaml | 1 + build-tools/automation/azure-pipelines.yaml | 21 ++++++++------- .../yaml-templates/apk-instrumentation.yaml | 2 +- .../yaml-templates/build-windows.yaml | 2 +- .../run-localization-tests.yaml | 1 - .../yaml-templates/run-nunit-tests.yaml | 26 ++++++++++++++++--- .../run-timezoneinfo-tests.yaml | 1 + .../automation/yaml-templates/variables.yaml | 4 +-- 8 files changed, 40 insertions(+), 18 deletions(-) diff --git a/build-tools/automation/azure-pipelines-nightly.yaml b/build-tools/automation/azure-pipelines-nightly.yaml index 89b55d11a86..cc6168b0f35 100644 --- a/build-tools/automation/azure-pipelines-nightly.yaml +++ b/build-tools/automation/azure-pipelines-nightly.yaml @@ -142,6 +142,7 @@ stages: extraBuildArgs: /p:TestAvdApiLevel=$(avdApiLevel) /p:TestAvdAbi=$(avdAbi) /p:TestAvdType=$(avdType) artifactSource: bin/Test$(XA.Build.Configuration)/Mono.Android_Tests-Signed.apk artifactFolder: Default + useDotNet: false - task: MSBuild@1 displayName: shut down emulator diff --git a/build-tools/automation/azure-pipelines.yaml b/build-tools/automation/azure-pipelines.yaml index 2e060009042..ef8c0138fbe 100644 --- a/build-tools/automation/azure-pipelines.yaml +++ b/build-tools/automation/azure-pipelines.yaml @@ -132,6 +132,7 @@ stages: testResultsFiles: TestResult-Xamarin.Android.JcwGen_Tests-$(ApkTestConfiguration).xml artifactSource: bin/Test$(ApkTestConfiguration)/Xamarin.Android.JcwGen_Tests-Signed.apk artifactFolder: Default + useDotNet: false - template: yaml-templates/apk-instrumentation.yaml parameters: @@ -142,6 +143,7 @@ stages: artifactSource: bin/Test$(ApkTestConfiguration)/Xamarin.Android.JcwGen_Tests-Signed.apk artifactFolder: FastDev_Assemblies_Dexes extraBuildArgs: /p:AndroidFastDeploymentType=Assemblies:Dexes + useDotNet: false - template: yaml-templates/apk-instrumentation.yaml parameters: @@ -151,6 +153,7 @@ stages: testResultsFiles: TestResult-Xamarin.Android.Locale_Tests-$(ApkTestConfiguration).xml artifactSource: bin/Test$(ApkTestConfiguration)/Xamarin.Android.Locale_Tests-Signed.apk artifactFolder: Default + useDotNet: false - template: yaml-templates/apk-instrumentation.yaml parameters: @@ -161,6 +164,7 @@ stages: extraBuildArgs: /p:AotAssemblies=True artifactSource: bin/Test$(ApkTestConfiguration)/Xamarin.Android.Locale_Tests-Signed.apk artifactFolder: Aot + useDotNet: false - template: yaml-templates/apk-instrumentation.yaml parameters: @@ -171,6 +175,7 @@ stages: extraBuildArgs: /p:AotAssemblies=True /p:AndroidEnableProfiledAot=true artifactSource: bin/Test$(ApkTestConfiguration)/Xamarin.Android.Locale_Tests-Signed.apk artifactFolder: Profiled-Aot + useDotNet: false - template: yaml-templates/apk-instrumentation.yaml parameters: @@ -180,6 +185,7 @@ stages: testResultsFiles: TestResult-Xamarin.Android.EmbeddedDSO_Test.nunit-$(ApkTestConfiguration).xml artifactSource: bin/Test$(ApkTestConfiguration)/Xamarin.Android.EmbeddedDSO_Test-Signed.apk artifactFolder: Default + useDotNet: false - template: yaml-templates/apk-instrumentation.yaml parameters: @@ -189,6 +195,7 @@ stages: testResultsFiles: TestResult-apkdiff-Xamarin.Forms_Performance_Integration-Signed-$(ApkTestConfiguration).xml artifactSource: bin/Test$(ApkTestConfiguration)/Xamarin.Forms_Performance_Integration-Signed.apk artifactFolder: Default + useDotNet: false - template: yaml-templates/apk-instrumentation.yaml parameters: @@ -199,6 +206,7 @@ stages: extraBuildArgs: /p:AotAssemblies=true artifactSource: bin/Test$(ApkTestConfiguration)/Xamarin.Forms_Performance_Integration-Signed.apk artifactFolder: Aot + useDotNet: false - template: yaml-templates/apk-instrumentation.yaml parameters: @@ -209,6 +217,7 @@ stages: extraBuildArgs: /p:AotAssemblies=True /p:AndroidEnableProfiledAot=true artifactSource: bin/Test$(ApkTestConfiguration)/Xamarin.Forms_Performance_Integration-Signed.apk artifactFolder: Profiled-Aot + useDotNet: false - template: yaml-templates/apk-instrumentation.yaml parameters: @@ -219,6 +228,7 @@ stages: extraBuildArgs: /p:BundleAssemblies=true artifactSource: bin/Test$(ApkTestConfiguration)/Xamarin.Forms_Performance_Integration-Signed.apk artifactFolder: Bundle + useDotNet: false - template: yaml-templates/apk-instrumentation.yaml parameters: @@ -228,6 +238,7 @@ stages: testResultsFiles: TestResult-apkdiff-com.companyname.vsandroidapp-Signed-$(ApkTestConfiguration).xml artifactSource: bin/Test$(ApkTestConfiguration)/com.companyname.vsandroidapp-Signed.apk artifactFolder: Default + useDotNet: false - task: MSBuild@1 displayName: shut down emulator @@ -347,7 +358,6 @@ stages: testResultsFiles: TestResult-Mono.Android.NET_Tests-$(XA.Build.Configuration).xml artifactSource: bin/Test$(XA.Build.Configuration)/$(DotNetTargetFramework)-android/Mono.Android.NET_Tests-Signed.aab artifactFolder: $(DotNetTargetFramework)-$(XA.Build.Configuration) - useDotNet: true - template: yaml-templates/apk-instrumentation.yaml parameters: @@ -358,7 +368,6 @@ stages: testResultsFiles: TestResult-Mono.Android.NET_Tests-Debug.xml artifactSource: bin/Test$(XA.Build.Configuration)/$(DotNetTargetFramework)-android/Mono.Android.NET_Tests-Signed.apk artifactFolder: $(DotNetTargetFramework)-Debug - useDotNet: true - template: yaml-templates/apk-instrumentation.yaml parameters: @@ -369,7 +378,6 @@ stages: extraBuildArgs: -p:TestsFlavor=NoAab -p:AndroidPackageFormat=apk artifactSource: bin/Test$(XA.Build.Configuration)/$(DotNetTargetFramework)-android/Mono.Android.NET_Tests-Signed.apk artifactFolder: $(DotNetTargetFramework)-NoAab - useDotNet: true - template: yaml-templates/apk-instrumentation.yaml parameters: @@ -380,7 +388,6 @@ stages: extraBuildArgs: -p:TestsFlavor=Interpreter -p:UseInterpreter=True artifactSource: bin/Test$(XA.Build.Configuration)/$(DotNetTargetFramework)-android/Mono.Android.NET_Tests-Signed.aab artifactFolder: $(DotNetTargetFramework)-Interpreter - useDotNet: true - template: yaml-templates/apk-instrumentation.yaml parameters: @@ -391,7 +398,6 @@ stages: extraBuildArgs: -p:TestsFlavor=NoAot -p:RunAOTCompilation=false artifactSource: bin/Test$(XA.Build.Configuration)/$(DotNetTargetFramework)-android/Mono.Android.NET_Tests-Signed.aab artifactFolder: $(DotNetTargetFramework)-NoAot - useDotNet: true - template: yaml-templates/apk-instrumentation.yaml parameters: @@ -402,7 +408,6 @@ stages: extraBuildArgs: -p:TestsFlavor=AotLlvm -p:EnableLLVM=true -p:AndroidEnableProfiledAot=false artifactSource: bin/Test$(XA.Build.Configuration)/$(DotNetTargetFramework)-android/Mono.Android.NET_Tests-Signed.aab artifactFolder: $(DotNetTargetFramework)-AotLlvm - useDotNet: true - task: MSBuild@1 displayName: shut down emulator @@ -497,7 +502,6 @@ stages: - template: yaml-templates/run-nunit-tests.yaml parameters: - useDotNet: true testRunTitle: MSBuildDeviceIntegration Smoke - macOS testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll dotNetTestExtraArgs: --filter "TestCategory = SmokeTests $(DotNetNUnitCategories)" @@ -546,7 +550,6 @@ stages: - template: yaml-templates/run-nunit-tests.yaml parameters: - useDotNet: true testRunTitle: Xamarin.Android.Build.Tests - Linux .NET 6 Smoke Tests testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/$(DotNetStableTargetFramework)/Xamarin.Android.Build.Tests.dll dotNetTestExtraArgs: --filter "TestCategory = SmokeTests $(DotNetNUnitCategories)" @@ -739,7 +742,6 @@ stages: - template: yaml-templates/run-nunit-tests.yaml parameters: - useDotNet: true testRunTitle: WearOS On Device - macOS testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll dotNetTestExtraArgs: --filter "TestCategory = WearOS" @@ -805,6 +807,7 @@ stages: testResultsFiles: TestResult-Xamarin.Android.Bcl_Tests.nunit-$(XA.Build.Configuration).xml artifactSource: bin/Test$(XA.Build.Configuration)/Xamarin.Android.Bcl_Tests-Signed.apk artifactFolder: Default + useDotNet: false - task: PublishTestResults@2 displayName: publish Xamarin.Android.Bcl-Tests-XUnit results diff --git a/build-tools/automation/yaml-templates/apk-instrumentation.yaml b/build-tools/automation/yaml-templates/apk-instrumentation.yaml index 028df889d7b..354ca060758 100644 --- a/build-tools/automation/yaml-templates/apk-instrumentation.yaml +++ b/build-tools/automation/yaml-templates/apk-instrumentation.yaml @@ -9,7 +9,7 @@ parameters: testResultsFormat: NUnit artifactSource: "" artifactFolder: "" - useDotNet: false + useDotNet: true condition: succeeded() steps: diff --git a/build-tools/automation/yaml-templates/build-windows.yaml b/build-tools/automation/yaml-templates/build-windows.yaml index 4de0eb8b73c..626b75095de 100644 --- a/build-tools/automation/yaml-templates/build-windows.yaml +++ b/build-tools/automation/yaml-templates/build-windows.yaml @@ -116,6 +116,7 @@ stages: - template: run-nunit-tests.yaml parameters: + useDotNet: false testRunTitle: Smoke MSBuild Tests - Windows Build Tree testAssembly: $(System.DefaultWorkingDirectory)\bin\Test$(XA.Build.Configuration)\net472\Xamarin.Android.Build.Tests.dll testResultsFile: TestResult-SmokeMSBuildTests-WinBuildTree-$(XA.Build.Configuration).xml @@ -123,7 +124,6 @@ stages: - template: run-nunit-tests.yaml parameters: - useDotNet: true testRunTitle: Smoke MSBuild Tests - Windows Dotnet Build testAssembly: $(System.DefaultWorkingDirectory)\bin\Test$(XA.Build.Configuration)\$(DotNetStableTargetFramework)\Xamarin.Android.Build.Tests.dll testResultsFile: TestResult-SmokeMSBuildTests-WinDotnetBuild-$(XA.Build.Configuration).xml diff --git a/build-tools/automation/yaml-templates/run-localization-tests.yaml b/build-tools/automation/yaml-templates/run-localization-tests.yaml index 1ed205cf539..2376c0e3dfb 100644 --- a/build-tools/automation/yaml-templates/run-localization-tests.yaml +++ b/build-tools/automation/yaml-templates/run-localization-tests.yaml @@ -36,7 +36,6 @@ jobs: parameters: testRunTitle: LocalizationTests On Device - macOS testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll - useDotNet: true dotNetTestExtraArgs: --filter "Name~CheckLocalizationIsCorrectNode${{ parameters.node_id }}" testResultsFile: TestResult-LocalizationTests-Node${{ parameters.node_id }}-$(XA.Build.Configuration).xml diff --git a/build-tools/automation/yaml-templates/run-nunit-tests.yaml b/build-tools/automation/yaml-templates/run-nunit-tests.yaml index c3a7ed29c28..95d49d08506 100644 --- a/build-tools/automation/yaml-templates/run-nunit-tests.yaml +++ b/build-tools/automation/yaml-templates/run-nunit-tests.yaml @@ -6,12 +6,15 @@ parameters: testResultsFile: TestResult.xml nunitConsoleExtraArgs: '' dotNetTestExtraArgs: '' - useDotNet: false + useDotNet: true + useDotNetPreview: false workers: $(NUnit.NumberOfTestWorkers) condition: succeeded() + timeoutInMinutes: 0 + retryCountOnTaskFailure: 0 steps: -- ${{ if eq(parameters.useDotNet, false) }}: +- ${{ if and(eq(parameters.useDotNet, false), eq(parameters.useDotNetPreview, false)) }}: - powershell: | Write-Host '##vso[task.setvariable variable=TestResultsFormat]NUnit' if ([Environment]::OSVersion.Platform -eq "Unix") { @@ -28,7 +31,7 @@ steps: condition: ${{ parameters.condition }} continueOnError: true -- ${{ if eq(parameters.useDotNet, true) }}: +- ${{ if and(eq(parameters.useDotNet, true), eq(parameters.useDotNetPreview, true)) }}: - powershell: Write-Host '##vso[task.setvariable variable=TestResultsFormat]VSTest' - template: run-dotnet-preview.yaml parameters: @@ -43,6 +46,21 @@ steps: displayName: run ${{ parameters.testRunTitle }} condition: ${{ parameters.condition }} +- ${{ if and(eq(parameters.useDotNet, true), eq(parameters.useDotNetPreview, false)) }}: + - task: DotNetCoreCLI@2 + inputs: + command: test + projects: ${{ parameters.testAssembly }} + arguments: >- + ${{ parameters.dotNetTestExtraArgs }} -- NUnit.NumberOfTestWorkers=${{ parameters.workers }} + publishTestResults: true + testRunTitle: ${{ parameters.testRunTitle }} + displayName: run ${{ parameters.testRunTitle }} + condition: ${{ parameters.condition }} + continueOnError: true + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + retryCountOnTaskFailure: ${{ parameters.retryCountOnTaskFailure }} + - template: kill-processes.yaml - task: PublishTestResults@2 @@ -50,4 +68,4 @@ steps: testResultsFormat: $(TestResultsFormat) testResultsFiles: ${{ parameters.testResultsFile }} testRunTitle: ${{ parameters.testRunTitle }} - condition: ${{ parameters.condition }} + condition: and(${{ parameters.condition }}, or(ne('${{ parameters.useDotNet }}', 'true'), eq('${{ parameters.useDotNetPreview }}', 'true'))) diff --git a/build-tools/automation/yaml-templates/run-timezoneinfo-tests.yaml b/build-tools/automation/yaml-templates/run-timezoneinfo-tests.yaml index baa624c70cd..3f96affb8bf 100644 --- a/build-tools/automation/yaml-templates/run-timezoneinfo-tests.yaml +++ b/build-tools/automation/yaml-templates/run-timezoneinfo-tests.yaml @@ -34,6 +34,7 @@ jobs: - template: run-nunit-tests.yaml parameters: + useDotNet: false testRunTitle: TimeZoneInfoTests On Device - macOS testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/net472/MSBuildDeviceIntegration.dll nunitConsoleExtraArgs: --where "test == Xamarin.Android.Build.Tests.DeploymentTest.CheckTimeZoneInfoIsCorrectNode${{ parameters.node_id }}" diff --git a/build-tools/automation/yaml-templates/variables.yaml b/build-tools/automation/yaml-templates/variables.yaml index 1c5899f879c..5f49d1afd0f 100644 --- a/build-tools/automation/yaml-templates/variables.yaml +++ b/build-tools/automation/yaml-templates/variables.yaml @@ -18,9 +18,9 @@ variables: - name: NUnit.NumberOfTestWorkers value: 4 - name: DotNetSdkVersion - value: 7.0.1xx + value: 7.0 - name: DotNetSdkQuality - value: preview + value: GA - name: GitHub.Token value: $(github--pat--vs-mobiletools-engineering-service2) - name: HostedMacImage