Move all quarantined tests to the aspnetcore-quarantined-tests pipeline (#22445)

Author: BrennanConroy

.azure/pipelines/ci.yml

@@ -503,16 +503,6 @@ stages:
       beforeBuild:
       - powershell: "& ./src/Servers/IIS/tools/UpdateIISExpressCertificate.ps1; & ./src/Servers/IIS/tools/update_schema.ps1"
         displayName: Setup IISExpress test certificates and schema
-      afterBuild:
-      - powershell: "& ./build.ps1 -CI -nobl -NoBuild -Test /p:RunQuarantinedTests=true"
-        displayName: Run Quarantined Tests
-        continueOnError: true
-      - task: PublishTestResults@2
-        displayName: Publish Quarantined Test Results
-        inputs:
-          testResultsFormat: 'xUnit'
-          testResultsFiles: '*.xml'
-          searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Quarantined'
       artifacts:
       - name: Windows_Test_Dumps
         path: artifacts/dumps/
@@ -566,20 +556,6 @@ stages:
       beforeBuild:
       - bash: "./eng/scripts/install-nginx-mac.sh"
         displayName: Installing Nginx
-      afterBuild:
-      - bash: ./build.sh --ci --nobl --pack --no-build --no-restore --no-build-deps
-        displayName: Pack Packages (for Template tests)
-      - bash: ./src/ProjectTemplates/build.sh --ci --nobl --pack --no-restore --no-build-deps
-        displayName: Pack Templates (for Template tests)
-      - bash: ./build.sh --no-build --ci --nobl --test -p:RunQuarantinedTests=true
-        displayName: Run Quarantined Tests
-        continueOnError: true
-      - task: PublishTestResults@2
-        displayName: Publish Quarantined Test Results
-        inputs:
-          testResultsFormat: 'xUnit'
-          testResultsFiles: '*.xml'
-          searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Quarantined'
       artifacts:
       - name: MacOS_Test_Logs
         path: artifacts/log/
@@ -604,20 +580,6 @@ stages:
         displayName: Installing Nginx
       - bash: "echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p"
         displayName: Increase inotify limit
-      afterBuild:
-      - bash: ./build.sh --ci --nobl --pack --no-build --no-restore --no-build-deps
-        displayName: Pack Packages (for Template tests)
-      - bash: ./src/ProjectTemplates/build.sh --ci --nobl --pack --no-restore --no-build-deps
-        displayName: Pack Templates (for Template tests)
-      - bash: ./build.sh --no-build --ci --nobl --test -p:RunQuarantinedTests=true
-        displayName: Run Quarantined Tests
-        continueOnError: true
-      - task: PublishTestResults@2
-        displayName: Publish Quarantined Test Results
-        inputs:
-          testResultsFormat: 'xUnit'
-          testResultsFiles: '*.xml'
-          searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Quarantined'
       artifacts:
       - name: Linux_Test_Logs
         path: artifacts/log/

.azure/pipelines/quarantined-tests.yml

@@ -25,10 +25,10 @@ variables:
 jobs:
 - template: jobs/default-build.yml
   parameters:
-    jobName: Helix_quarantine_x64
-    jobDisplayName: 'Tests: Helix Quarantine x64'
+    jobName: Helix_quarantined_x64
+    jobDisplayName: 'Tests: Helix'
     agentOs: Windows
-    timeoutInMinutes: 240
+    timeoutInMinutes: 180
     steps:
     # Build the shared framework
     - script: ./build.cmd -ci -nobl -all -pack -arch x64 /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log
@@ -46,3 +46,101 @@ jobs:
     - name: Helix_logs
       path: artifacts/log/
       publishOnError: true
+
+- template: jobs/default-build.yml
+  parameters:
+    jobName: Windows_Quarantined_x64
+    jobDisplayName: 'Tests: Windows x64'
+    agentOs: Windows
+    timeoutInMinutes: 180
+    isTestingJob: true
+    steps:
+    - powershell: "& ./build.ps1 -CI -nobl -all -pack -NoBuildJava"
+      displayName: Build
+    # The templates part can be removed when the Blazor Templates run on Helix
+    - script: ./src/ProjectTemplates/build.cmd -ci -nobl -pack -NoRestore -NoBuilddeps "/p:RunTemplateTests=true"
+      displayName: Pack Templates
+    - script: ./build.cmd -ci -nobl -test -NoRestore -NoBuild -NoBuilddeps "/p:RunTemplateTests=true /p:RunQuarantinedTests=true /p:SkipHelixReadyTests=true"
+      displayName: Run Quarantined Tests
+      condition: always()
+    - task: PublishTestResults@2
+      displayName: Publish Quarantined Test Results
+      inputs:
+        testResultsFormat: 'xUnit'
+        testResultsFiles: '*.xml'
+        searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Quarantined'
+      condition: always()
+    artifacts:
+    - name: Windows_Quarantined_Test_Logs
+      path: artifacts/log/
+      publishOnError: true
+      includeForks: true
+    - name: Windows_Quarantined_Test_Results
+      path: artifacts/TestResults/
+      publishOnError: true
+      includeForks: true
+
+- template: jobs/default-build.yml
+  parameters:
+    jobName: MacOS_Quarantined_Test
+    jobDisplayName: "Tests: macOS 10.14"
+    agentOs: macOS
+    timeoutInMinutes: 180
+    isTestingJob: true
+    steps:
+    - bash: ./build.sh --all --pack --ci --nobl --no-build-java
+      displayName: Build
+    # The templates part can be removed when the Blazor Templates run on Helix
+    - bash: ./src/ProjectTemplates/build.sh --ci --nobl --pack --no-restore --no-build-deps
+      displayName: Pack Templates (for Template tests)
+    - bash: ./build.sh --no-build --ci --nobl --test -p:RunTemplateTests=true -p:RunQuarantinedTests=true -p:SkipHelixReadyTests=true
+      displayName: Run Quarantined Tests
+    - task: PublishTestResults@2
+      displayName: Publish Quarantined Test Results
+      inputs:
+        testResultsFormat: 'xUnit'
+        testResultsFiles: '*.xml'
+        searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Quarantined'
+      condition: always()
+    artifacts:
+    - name: MacOS_Quarantined_Test_Logs
+      path: artifacts/log/
+      publishOnError: true
+      includeForks: true
+    - name: MacOS_Quarantined_Test_Results
+      path: artifacts/TestResults/
+      publishOnError: true
+      includeForks: true
+
+- template: jobs/default-build.yml
+  parameters:
+    jobName: Linux_Quarantined_Test
+    jobDisplayName: "Tests: Ubuntu 16.04 x64"
+    agentOs: Linux
+    timeoutInMinutes: 180
+    isTestingJob: true
+    useHostedUbuntu: false
+    steps:
+    - bash: ./build.sh --all --pack --ci --nobl --no-build-java
+      displayName: Build
+    # The templates part can be removed when the Blazor Templates run on Helix
+    - bash: ./src/ProjectTemplates/build.sh --ci --nobl --pack --no-restore --no-build-deps
+      displayName: Pack Templates (for Template tests)
+    - bash: ./build.sh --no-build --ci --nobl --test -p:RunTemplateTests=true -p:RunQuarantinedTests=true -p:SkipHelixReadyTests=true
+      displayName: Run Quarantined Tests
+    - task: PublishTestResults@2
+      displayName: Publish Quarantined Test Results
+      inputs:
+        testResultsFormat: 'xUnit'
+        testResultsFiles: '*.xml'
+        searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Quarantined'
+      condition: always()
+    artifacts:
+    - name: Linux_Quarantined_Test_Logs
+      path: artifacts/log/
+      publishOnError: true
+      includeForks: true
+    - name: Linux_Quarantined_Test_Results
+      path: artifacts/TestResults/
+      publishOnError: true
+      includeForks: true

src/ProjectTemplates/BlazorTemplates.Tests/BlazorWasmTemplateTest.cs

@@ -250,10 +250,21 @@ private void ValidateAppWorksOffline(Project project, string listeningUri, bool
             TestBasicNavigation(project.ProjectName, skipFetchData: skipFetchData);
         }
 
-        [Theory]
-        [InlineData(true)]
-        [InlineData(false)]
-        public async Task BlazorWasmHostedTemplate_IndividualAuth_Works(bool useLocalDb)
+        [ConditionalFact]
+        // LocalDB doesn't work on non Windows platforms
+        [OSSkipCondition(OperatingSystems.Linux | OperatingSystems.MacOSX)]
+        public Task BlazorWasmHostedTemplate_IndividualAuth_Works_WithLocalDB()
+        {
+            return BlazorWasmHostedTemplate_IndividualAuth_Works(true);
+        }
+
+        [Fact]
+        public Task BlazorWasmHostedTemplate_IndividualAuth_Works_WithOutLocalDB()
+        {
+            return BlazorWasmHostedTemplate_IndividualAuth_Works(false);
+        }
+
+        private async Task BlazorWasmHostedTemplate_IndividualAuth_Works(bool useLocalDb)
         {
             var project = await ProjectFactory.GetOrCreateProject("blazorhostedindividual" + (useLocalDb ? "uld" : ""), Output);
 
@@ -268,11 +279,11 @@ public async Task BlazorWasmHostedTemplate_IndividualAuth_Works(bool useLocalDb)
                 Assert.Contains(".db", serverProjectFileContents);
             }
 
-            var appSettings = ReadFile(serverProject.TemplateOutputDir, "appSettings.json");
+            var appSettings = ReadFile(serverProject.TemplateOutputDir, "appsettings.json");
             var element = JsonSerializer.Deserialize<JsonElement>(appSettings);
             var clientsProperty = element.GetProperty("IdentityServer").EnumerateObject().Single().Value.EnumerateObject().Single();
             var replacedSection = element.GetRawText().Replace(clientsProperty.Name, serverProject.ProjectName.Replace(".Server", ".Client"));
-            var appSettingsPath = Path.Combine(serverProject.TemplateOutputDir, "appSettings.json");
+            var appSettingsPath = Path.Combine(serverProject.TemplateOutputDir, "appsettings.json");
             File.WriteAllText(appSettingsPath, replacedSection);
 
             var publishResult = await serverProject.RunDotNetPublishAsync();