[ci] Automatically retry failed emulator tests. (#7997)

[Previously](#7963), we have used `retryCountOnTaskFailure` to retry some of our flaky test suites.  However this is not a good solution for our MSBuild and emulator tests for 2 reasons:
- Running each suite takes 30-60 minutes, so it would take a long time to retry the entire suite.
- The suites are very flaky, and rerunning the entire suite would likely result in test(s) that were successful on the first run to fail on the second.

We need a solution that only retries the specific test(s) that failed.  Unfortunately nothing like that exists, so we'll once again have to roll our own solution, and we can reuse `dotnet-test-slicer` for this.

This involves some trickery to make everything show up correctly in the AzDO UI:
- Run the initial test suite
  - Run in a Powershell so we can ignore any test failures that would move the pipeline into `SucceededWithIssues` state
  - Do not automatically publish the test results, as they would contain test failures
- Run `dotnet-test-slicer retry`
  - This creates a new `.runsettings` file that only contains the failed tests so `dotnet test` can run only them
  - It also rewrites the test results file of the initial run and removes any failed tests from so it can be published to AzDO
- Publish the successful tests from the first run to AzDO
- Run the retried tests
  - This time we can use normal `dotnet`, and any test failures can be automatically reported to AzDO

Affected Test Suites:
- MSBuild Emulator Tests

Author: jpobst

build-tools/automation/yaml-templates/install-dotnet-test-slicer.yaml

@@ -1,5 +1,5 @@
 parameters:
-  version: '0.1.0-alpha2'
+  version: '0.1.0-alpha5'
   condition: succeeded()
   continueOnError: true
 

build-tools/automation/yaml-templates/run-msbuild-tests.yaml

@@ -48,7 +48,7 @@ jobs:
       testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/$(DotNetStableTargetFramework)/Xamarin.Android.Build.Tests.dll
       testFilter: ${{ parameters.testFilter }} $(ExcludedNUnitCategories)
       testRunTitle: Xamarin.Android.Build.Tests - ${{ parameters.testOS }}
-      testResultsTitle: TestResult-MSBuildTests-${{ parameters.jobName }}
+      retryFailedTests: false
 
   - template: upload-results.yaml
     parameters:

build-tools/automation/yaml-templates/run-sliced-nunit-tests.yaml

@@ -2,22 +2,55 @@ parameters:
   testAssembly:           # NUnit test assembly to run
   testFilter:             # Filter used to select tests (NUnit test selection language, not dotnet test filter language)
   testRunTitle:           # Title of the test run
-  testResultsTitle:       # Title used to construct test results file name
+  retryFailedTests: true  # Retry failed tests once
 
 steps:
 - pwsh: |
     dotnet-test-slicer `
+      slice `
       --test-assembly="${{ parameters.testAssembly }}" `
       --test-filter="${{ parameters.testFilter }}" `
       --slice-number=$(System.JobPositionInPhase) `
       --total-slices=$(System.TotalJobsInPhase) `
       --outfile="${{ parameters.testAssembly }}.runsettings"
   displayName: Slice unit tests
 
-- template: run-nunit-tests.yaml
-  parameters:
-    useDotNet: true
-    testRunTitle: ${{ parameters.testRunTitle }}-$(System.JobPositionInPhase)
-    testAssembly: ${{ parameters.testAssembly }}
-    dotNetTestExtraArgs: --settings "${{ parameters.testAssembly }}.runsettings"
-    testResultsFile: ${{ parameters.testResultsTitle }}-$(System.JobPositionInPhase)-$(XA.Build.Configuration).xml
+- ${{ if eq(parameters.retryFailedTests, 'false') }}:
+  # If we aren't using auto-retry logic, then this is just a simple template call
+  - template: run-nunit-tests.yaml
+    parameters:
+      testRunTitle: ${{ parameters.testRunTitle }}-$(System.JobPositionInPhase)
+      testAssembly: ${{ parameters.testAssembly }}
+      dotNetTestExtraArgs: --settings "${{ parameters.testAssembly }}.runsettings"
+
+- ${{ if eq(parameters.retryFailedTests, 'true') }}:
+  # We need a custom dotnet test invocation here that does not trigger a task failure on failed tests
+  - pwsh: |
+      dotnet `
+        test `
+        ${{ parameters.testAssembly }} `
+        --settings "${{ parameters.testAssembly }}.runsettings" `
+        --logger trx --results-directory $(Agent.TempDirectory) `
+        -- NUnit.NumberOfTestWorkers=$(NUnit.NumberOfTestWorkers)
+    displayName: Run tests
+    ignoreLASTEXITCODE: true
+      
+  - pwsh: |
+      dotnet-test-slicer `
+        retry `
+        --trx="$(Agent.TempDirectory)" `
+        --outfile="${{ parameters.testAssembly }}.runsettings"
+    displayName: Look for failed tests
+
+    # dotnet-test-slicer removed the failed tests from our results file, so it's safe to publish it now
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFormat: VSTest
+      testResultsFiles: $(Agent.TempDirectory)/*.trx
+      testRunTitle: ${{ parameters.testRunTitle }}-$(System.JobPositionInPhase)
+
+  - template: run-nunit-tests.yaml
+    parameters:
+      testRunTitle: ${{ parameters.testRunTitle }}-$(System.JobPositionInPhase) (Auto-Retry)
+      testAssembly: ${{ parameters.testAssembly }}
+      dotNetTestExtraArgs: --settings "${{ parameters.testAssembly }}.runsettings"

build-tools/automation/yaml-templates/stage-msbuild-emulator-tests.yaml

@@ -53,7 +53,6 @@ stages:
         testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll
         testFilter: cat != TimeZoneInfo & cat != Localization $(ExcludedNUnitCategories)
         testRunTitle: MSBuildDeviceIntegration On Device - macOS
-        testResultsTitle: TestResult-MSBuildDeviceIntegration-${{ parameters.job_name }}
 
     - template: start-stop-emulator.yaml
       parameters:

build-tools/automation/yaml-templates/start-stop-emulator.yaml

@@ -5,7 +5,7 @@ parameters:
   avdApiLevel:            # Device API level, like '30', required if 'specificImage' is 'true'
   avdAbi:                 # Device ABI, like 'x86', required if 'specificImage' is 'true'
   avdType:                # Device AVD, like 'android-wear', required if 'specificImage' is 'true'
-  launchTimeoutMin: 15    # Minutes to wait for the emulator to start
+  launchTimeoutMin: 20    # Minutes to wait for the emulator to start
   xaSourcePath: $(System.DefaultWorkingDirectory) # working directory
 
 steps: