[CI] Allow MSBuild test stages to run in the megapipeline

build-tools/automation/azure-pipelines.yaml

@@ -42,22 +42,6 @@ parameters:
 variables:
 - template: yaml-templates/variables.yaml
 - template: templates/common/vs-release-vars.yml@sdk-insertions
-  # Run all tests if:
-  # - User who queued the job requested it (They set XA.RunAllTests to true)
-  # - This is the master integration branch (Pipeline defaults XA.RunAllTests to true)
-  # - This is a non-fork branch with name containing "mono-" (for Mono bumps)
-- name: IsMonoBranch
-  value: $[and(ne(variables['System.PullRequest.IsFork'], 'True'), or(contains(variables['Build.SourceBranchName'], 'mono-'), contains(variables['System.PullRequest.SourceBranch'], 'mono-')))]
-- name: RunAllTests
-  value: $[or(eq(variables['XA.RunAllTests'], true), eq(variables['IsMonoBranch'], true))]
-- name: IsRelOrTargetingRel
-  value: $[or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['System.PullRequest.TargetBranch'], 'release/'))]
-- name: DotNetNUnitCategories
-  value: '& TestCategory != DotNetIgnore & TestCategory != HybridAOT & TestCategory != MkBundle & TestCategory != MonoSymbolicate & TestCategory != StaticProject & TestCategory != SystemApplication'
-- name: ExcludedNUnitCategories
-  value: '& cat != DotNetIgnore & cat != HybridAOT & cat != MkBundle & cat != MonoSymbolicate & cat != StaticProject & cat != SystemApplication'
-- name: DefaultTestSdkPlatforms  # Comma-separated SDK Platform(s) to install on test agents (no spaces)
-  value: 33,UpsideDownCake
 - ${{ if eq(variables['Build.DefinitionName'], 'Xamarin.Android-Private') }}:
   - group: AzureDevOps-Artifact-Feeds-Pats
   - group: DotNet-MSRC-Storage

build-tools/automation/yaml-templates/fail-on-issue.yaml

@@ -8,3 +8,4 @@ steps:
         Write-Host "##vso[task.complete result=Failed;]DONE"
     }
   displayName: fail if any issues occurred
+  condition: ${{ parameters.condition }}

build-tools/automation/yaml-templates/run-msbuild-tests.yaml

@@ -7,6 +7,7 @@ parameters:
   xaSourcePath: $(System.DefaultWorkingDirectory)
   repositoryAlias: 'self'
   commit: ''
+  shouldFailOnIssue: true
 
 jobs:
 - job: ${{ parameters.jobName }}
@@ -41,17 +42,21 @@ jobs:
   - task: DownloadPipelineArtifact@2
     inputs:
       artifactName: $(TestAssembliesArtifactName)
-      downloadPath: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)
+      downloadPath: ${{ parameters.xaSourcePath }}/bin/Test$(XA.Build.Configuration)
 
   - template: run-sliced-nunit-tests.yaml
     parameters:
-      testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/$(DotNetStableTargetFramework)/Xamarin.Android.Build.Tests.dll
+      testAssembly: ${{ parameters.xaSourcePath }}/bin/Test$(XA.Build.Configuration)/$(DotNetStableTargetFramework)/Xamarin.Android.Build.Tests.dll
       testFilter: ${{ parameters.testFilter }} $(ExcludedNUnitCategories)
       testRunTitle: Xamarin.Android.Build.Tests - ${{ parameters.testOS }}
       retryFailedTests: false
+      xaSourcePath: ${{ parameters.xaSourcePath }}
 
   - template: upload-results.yaml
     parameters:
       artifactName: Test Results - MSBuild - ${{ parameters.testOS }}-$(System.JobPositionInPhase)
+      xaSourcePath: ${{ parameters.xaSourcePath }}
 
   - template: fail-on-issue.yaml
+    parameters:
+      condition: ${{ parameters.shouldFailOnIssue }}

build-tools/automation/yaml-templates/run-sliced-nunit-tests.yaml

@@ -2,6 +2,7 @@ parameters:
   testAssembly:           # NUnit test assembly to run
   testFilter:             # Filter used to select tests (NUnit test selection language, not dotnet test filter language)
   testRunTitle:           # Title of the test run
+  xaSourcePath: $(System.DefaultWorkingDirectory)
   retryFailedTests: true  # Retry failed tests once
 
 steps:
@@ -22,6 +23,7 @@ steps:
       testRunTitle: ${{ parameters.testRunTitle }}-$(System.JobPositionInPhase)
       testAssembly: ${{ parameters.testAssembly }}
       dotNetTestExtraArgs: --settings "${{ parameters.testAssembly }}.runsettings"
+      xaSourcePath: ${{ parameters.xaSourcePath }}
 
 - ${{ if eq(parameters.retryFailedTests, 'true') }}:
   # We need a custom dotnet test invocation here that does not trigger a task failure on failed tests
@@ -34,13 +36,15 @@ steps:
         -- NUnit.NumberOfTestWorkers=$(NUnit.NumberOfTestWorkers)
     displayName: Run tests
     ignoreLASTEXITCODE: true
-
+    workingDirectory: ${{ parameters.xaSourcePath }}
+
   - pwsh: |
       dotnet-test-slicer `
         retry `
         --trx="$(Agent.TempDirectory)" `
         --outfile="${{ parameters.testAssembly }}.runsettings"
     displayName: Look for failed tests
+    workingDirectory: ${{ parameters.xaSourcePath }}
 
     # dotnet-test-slicer removed the failed tests from our results file, so it's safe to publish it now
   - task: PublishTestResults@2
@@ -61,3 +65,4 @@ steps:
       testRunTitle: ${{ parameters.testRunTitle }}-$(System.JobPositionInPhase) (Auto-Retry)
       testAssembly: ${{ parameters.testAssembly }}
       dotNetTestExtraArgs: --settings "${{ parameters.testAssembly }}.runsettings"
+      xaSourcePath: ${{ parameters.xaSourcePath }}

build-tools/automation/yaml-templates/setup-test-environment.yaml

@@ -22,8 +22,6 @@ steps:
     commit: ${{ parameters.commit }}
     clean: true
     submodules: recursive
-    ${{ if ne(parameters.xaSourcePath, variables['System.DefaultWorkingDirectory']) }}:
-      path: s/xamarin-android
 
 - ${{ if eq(parameters.updateVS, true) }}:
   - template: update-vs.yaml

build-tools/automation/yaml-templates/stage-msbuild-emulator-tests.yaml

@@ -11,6 +11,8 @@ parameters:
   repositoryAlias: 'self'
   commit: ''
   usesCleanImages: true
+  shouldFailOnIssue: true
+  emulatorStartContinueOnError: false
 
 stages:
 - stage: ${{ parameters.stageName }}
@@ -45,26 +47,33 @@ stages:
     - task: DownloadPipelineArtifact@2
       inputs:
         artifactName: $(TestAssembliesArtifactName)
-        downloadPath: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)
+        downloadPath: ${{ parameters.xaSourcePath }}/bin/Test$(XA.Build.Configuration)
 
     - template: start-stop-emulator.yaml
+      parameters:
+        xaSourcePath: ${{ parameters.xaSourcePath }}
+        startContinueOnError: ${{ parameters.emulatorStartContinueOnError }}
 
     - template: run-sliced-nunit-tests.yaml
       parameters:
-        testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll
+        testAssembly: ${{ parameters.xaSourcePath }}/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll
         testFilter: cat != TimeZoneInfo & cat != Localization $(ExcludedNUnitCategories)
         testRunTitle: MSBuildDeviceIntegration On Device - macOS
 
     - ${{ if ne(parameters.usesCleanImages, true) }}:
       - template: start-stop-emulator.yaml
         parameters:
           command: stop
+          xaSourcePath: ${{ parameters.xaSourcePath }}
 
     - template: upload-results.yaml
       parameters:
         artifactName: Test Results - MSBuild With Emulator - macOS-$(System.JobPositionInPhase)
+        xaSourcePath: ${{ parameters.xaSourcePath }}
 
     - template: fail-on-issue.yaml
+      parameters:
+        condition: ${{ parameters.shouldFailOnIssue }}
 
   - job: wear_tests
     displayName: macOS > Tests > WearOS 
@@ -99,7 +108,7 @@ stages:
     - task: DownloadPipelineArtifact@2
       inputs:
         artifactName: $(TestAssembliesArtifactName)
-        downloadPath: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)
+        downloadPath: ${{ parameters.xaSourcePath }}/bin/Test$(XA.Build.Configuration)
 
     - template: start-stop-emulator.yaml
       parameters:
@@ -108,12 +117,15 @@ stages:
         avdApiLevel: $(avdApiLevel)
         avdAbi: $(avdAbi)
         avdType: $(avdType)
+        xaSourcePath: ${{ parameters.xaSourcePath }}
+        startContinueOnError: ${{ parameters.emulatorStartContinueOnError }}
 
     - template: run-sliced-nunit-tests.yaml
       parameters:
         testRunTitle: WearOS On Device - macOS
-        testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll
+        testAssembly: ${{ parameters.xaSourcePath }}/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll
         testFilter: cat = WearOS
+        xaSourcePath: ${{ parameters.xaSourcePath }}
 
     - ${{ if ne(parameters.usesCleanImages, true) }}:
       - template: start-stop-emulator.yaml
@@ -124,10 +136,14 @@ stages:
           avdApiLevel: $(avdApiLevel)
           avdAbi: $(avdAbi)
           avdType: $(avdType)
+          xaSourcePath: ${{ parameters.xaSourcePath }}
 
     - template: upload-results.yaml
       parameters:
         configuration: $(XA.Build.Configuration)
         artifactName: Test Results - Emulator $(avdApiLevel)-$(avdAbi)-$(avdType) - macOS
+        xaSourcePath: ${{ parameters.xaSourcePath }}
 
     - template: fail-on-issue.yaml
+      parameters:
+        condition: ${{ parameters.shouldFailOnIssue }}

build-tools/automation/yaml-templates/stage-msbuild-tests.yaml

@@ -8,6 +8,7 @@ parameters:
   xaSourcePath: $(System.DefaultWorkingDirectory)
   repositoryAlias: 'self'
   commit: ''
+  shouldFailOnIssue: true
 
 stages:
 - stage: ${{ parameters.stageName }}
@@ -25,6 +26,7 @@ stages:
       xaSourcePath: ${{ parameters.xaSourcePath }}
       repositoryAlias: ${{ parameters.repositoryAlias }}
       commit: ${{ parameters.commit }}
+      shouldFailOnIssue: ${{ parameters.shouldFailOnIssue }}
 
   - template: run-msbuild-tests.yaml
     parameters:
@@ -36,3 +38,4 @@ stages:
       xaSourcePath: ${{ parameters.xaSourcePath }}
       repositoryAlias: ${{ parameters.repositoryAlias }}
       commit: ${{ parameters.commit }}
+      shouldFailOnIssue: ${{ parameters.shouldFailOnIssue }}

build-tools/automation/yaml-templates/start-stop-emulator.yaml

@@ -7,13 +7,14 @@ parameters:
   avdType:                # Device AVD, like 'android-wear', required if 'specificImage' is 'true'
   emulatorMSBuildArgs:    # Extra args to pass to the emulator configuration like '-p:TestAvdExtraBootArgs=-writable-system', optional
   launchTimeoutMin: 20    # Minutes to wait for the emulator to start
+  startContinueOnError: false    # If 'true', do not mark the step as failed if starting the emulator fails
   xaSourcePath: $(System.DefaultWorkingDirectory) # working directory
 
 steps:
 - ${{ if eq(parameters.command, 'start') }}:
   - task: DotNetCoreCLI@2
     displayName: Start emulator
-    continueOnError: false
+    continueOnError: ${{ parameters.startContinueOnError }}
     inputs:
       projects: ${{ parameters.xaSourcePath }}/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Emulator.csproj
       ${{ if eq(parameters.specificImage, true) }}:

build-tools/automation/yaml-templates/variables.yaml

@@ -43,3 +43,11 @@ variables:
 # Workaround: https://github.com/dotnet/linker/issues/3012
 - name: DOTNET_gcServer
   value: 0
+- name: IsRelOrTargetingRel
+  value: $[or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['System.PullRequest.TargetBranch'], 'release/'))]
+- name: DotNetNUnitCategories
+  value: '& TestCategory != DotNetIgnore & TestCategory != HybridAOT & TestCategory != MkBundle & TestCategory != MonoSymbolicate & TestCategory != StaticProject & TestCategory != SystemApplication'
+- name: DefaultTestSdkPlatforms  # Comma-separated SDK Platform(s) to install on test agents (no spaces)
+  value: 33,UpsideDownCake
+- name: ExcludedNUnitCategories
+  value: '& cat != DotNetIgnore & cat != HybridAOT & cat != MkBundle & cat != MonoSymbolicate & cat != StaticProject & cat != SystemApplication'