diff --git a/azure-pipelines-PR.yml b/azure-pipelines-PR.yml index bf59708bdb1..cc88c091482 100644 --- a/azure-pipelines-PR.yml +++ b/azure-pipelines-PR.yml @@ -14,7 +14,7 @@ trigger: - docs/* - .vscode/* - .devcontainer/* - - tests/scripts/ + - tests/scripts/* - attributions.md - CODE_OF_CONDUCT.md - DEVGUIDE.md @@ -306,11 +306,13 @@ stages: name: $(DncEngPublicBuildPool) demands: ImageOverride -equals $(WindowsMachineQueueName) timeoutInMinutes: 120 + strategy: + parallel: 4 steps: - checkout: self clean: true - - script: eng\CIBuild.cmd -compressallmetadata -buildnorealsig -testDesktop -configuration Release + - script: eng\CIBuild.cmd -compressallmetadata -buildnorealsig -testDesktop -configuration Release -testBatch $(System.JobPositionInPhase) env: DOTNET_DbgEnableMiniDump: 1 DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing. @@ -318,9 +320,18 @@ stages: NativeToolsOnMachine: true displayName: Build + - task: PublishTestResults@2 + displayName: Publish Test Results + inputs: + testResultsFormat: 'XUnit' + testRunTitle: WindowsNoRealsig_testDesktop batch $(System.JobPositionInPhase) + mergeTestResults: true + testResultsFiles: '*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release' + continueOnError: true - task: PublishBuildArtifacts@1 displayName: Publish Build BinLog - condition: always() + condition: eq(variables['System.JobPositionInPhase'], 1) continueOnError: true inputs: PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' @@ -333,7 +344,7 @@ stages: continueOnError: true inputs: PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\Release' - ArtifactName: 'Windows Release WindowsNoRealsig_testDesktop process dumps' + ArtifactName: 'Windows Release WindowsNoRealsig_testDesktop process dumps $(System.JobPositionInPhase)' ArtifactType: Container parallel: true @@ -429,55 +440,36 @@ stages: demands: ImageOverride -equals $(WindowsMachineQueueName) timeoutInMinutes: 120 strategy: - maxParallel: 5 matrix: - desktop_release: - _configuration: Release - _testKind: testDesktop coreclr_release: _configuration: Release _testKind: testCoreclr + transparentCompiler: # Empty display name part. fsharpqa_release: _configuration: Release _testKind: testFSharpQA + transparentCompiler: vs_release: _configuration: Release _testKind: testVs + setupVsHive: true + transparentCompiler: transparent_compiler_release: _configuration: Release _testKind: testCoreclr - - ${{ if eq(variables['Build.Reason'], 'Flaky, disabled, was PullRequest') }}: - inttests_release: - _configuration: Release - _testKind: testIntegration + TEST_TRANSPARENT_COMPILER: 1 # Pipeline variable will map to env var. + transparentCompiler: TransparentCompiler + # inttests_release: + # _configuration: Release + # _testKind: testIntegration + # setupVsHive: true steps: - checkout: self clean: true - powershell: eng\SetupVSHive.ps1 displayName: Setup VS Hive - condition: or(eq(variables['_testKind'], 'testVs'), eq(variables['_testKind'], 'testIntegration')) - - # yes, this is miserable, but - https://github.com/dotnet/arcade/issues/13239 - - script: eng\CIBuild.cmd -compressallmetadata -configuration $(_configuration) -$(_testKind) - env: - DOTNET_DbgEnableMiniDump: 1 - DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing. - DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\$(_configuration)\$(Build.BuildId)-%e-%p-%t.dmp - NativeToolsOnMachine: true - displayName: Build / Test - condition: and( ne(variables['_testKind'], 'testIntegration'), ne(variables['System.JobName'], 'transparent_compiler_release') ) - - - script: eng\CIBuild.cmd -compressallmetadata -configuration $(_configuration) -$(_testKind) - env: - DOTNET_DbgEnableMiniDump: 1 - DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing. - DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\$(_configuration)\$(Build.BuildId)-%e-%p-%t.dmp - TEST_TRANSPARENT_COMPILER: 1 - NativeToolsOnMachine: true - displayName: Build / Test Transparent Compiler - condition: and( eq(variables['System.JobName'], 'transparent_compiler_release'), ne(variables['_testKind'], 'testIntegration') ) + condition: eq(variables.setupVsHive, 'true') - script: eng\CIBuild.cmd -compressallmetadata -configuration $(_configuration) -$(_testKind) env: @@ -485,15 +477,13 @@ stages: DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing. DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\$(_configuration)\$(Build.BuildId)-%e-%p-%t.dmp NativeToolsOnMachine: true - displayName: Build / Integration Test - continueOnError: true - condition: eq(variables['_testKind'], 'testIntegration') + displayName: Build and Test $(_testKind) $(transparentCompiler) - task: PublishTestResults@2 displayName: Publish Test Results inputs: testResultsFormat: 'XUnit' - testRunTitle: WindowsCompressedMetadata $(_testKind) + testRunTitle: WindowsCompressedMetadata $(_testKind) $(transparentCompiler) mergeTestResults: true testResultsFiles: '*.xml' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_configuration)' @@ -505,7 +495,7 @@ stages: continueOnError: true inputs: PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/$(_configuration)\Build.VisualFSharp.sln.binlog' - ArtifactName: 'Windows $(_configuration) $(_testKind) test binlogs' + ArtifactName: Windows $(_configuration) $(_testKind) $(transparentCompiler) test binlogs ArtifactType: Container parallel: true - task: PublishBuildArtifacts@1 @@ -521,7 +511,7 @@ stages: displayName: Publish Test Logs inputs: PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\$(_configuration)' - ArtifactName: 'Windows $(_configuration) $(_testKind) test logs' + ArtifactName: Windows $(_configuration) $(_testKind) $(transparentCompilerSuffix) test logs publishLocation: Container continueOnError: true condition: always() @@ -537,6 +527,84 @@ stages: continueOnError: true condition: failed() + # Windows With Compressed Metadata Desktop + - job: WindowsCompressedMetadata_Desktop + variables: + - name: XUNIT_LOGS + value: $(Build.SourcesDirectory)\artifacts\TestResults\Release + - name: __VSNeverShowWhatsNew + value: 1 + pool: + # The PR build definition sets this variable: + # WindowsMachineQueueName=Windows.vs2022.amd64.open + # and there is an alternate build definition that sets this to a queue that is always scouting the + # next preview of Visual Studio. + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(WindowsMachineQueueName) + timeoutInMinutes: 120 + strategy: + parallel: 4 + + steps: + - checkout: self + clean: true + + - script: eng\CIBuild.cmd -compressallmetadata -configuration Release -testDesktop -testBatch $(System.JobPositionInPhase) + env: + DOTNET_DbgEnableMiniDump: 1 + DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing. + DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\Release\$(Build.BuildId)-%e-%p-%t.dmp + NativeToolsOnMachine: true + displayName: Build / Test + + - task: PublishTestResults@2 + displayName: Publish Test Results + inputs: + testResultsFormat: 'XUnit' + testRunTitle: WindowsCompressedMetadata testDesktop batch $(System.JobPositionInPhase) + mergeTestResults: true + testResultsFiles: '*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release' + continueOnError: true + + - task: PublishBuildArtifacts@1 + displayName: Publish BinLog + condition: eq(variables['System.JobPositionInPhase'], 1) + continueOnError: true + inputs: + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' + ArtifactName: 'Windows testDesktop binlogs' + ArtifactType: Container + parallel: true + - task: PublishBuildArtifacts@1 + displayName: Publish Dumps + condition: failed() + continueOnError: true + inputs: + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\Release' + ArtifactName: 'Windows testDesktop process dumps $(System.JobPositionInPhase)' + ArtifactType: Container + parallel: true + - task: PublishBuildArtifacts@1 + displayName: Publish Test Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\Release' + ArtifactName: 'Windows testDesktop test logs batch $(System.JobPositionInPhase)' + publishLocation: Container + continueOnError: true + condition: always() + - script: dotnet build $(Build.SourcesDirectory)/eng/DumpPackageRoot/DumpPackageRoot.csproj + displayName: Dump NuGet cache contents + condition: failed() + - task: PublishBuildArtifacts@1 + displayName: Publish NuGet cache contents + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\NugetPackageRootContents' + ArtifactName: 'NuGetPackageContents Windows testDesktop $(System.JobPositionInPhase)' + publishLocation: Container + continueOnError: true + condition: failed() + # Mock official build - job: MockOfficial pool: diff --git a/eng/Build.ps1 b/eng/Build.ps1 index 146372ecde6..fe2fe5731bd 100644 --- a/eng/Build.ps1 +++ b/eng/Build.ps1 @@ -73,6 +73,7 @@ param ( [switch]$compressAllMetadata, [switch]$buildnorealsig = $true, [switch]$verifypackageshipstatus = $false, + [string]$testBatch = "", [parameter(ValueFromRemainingArguments = $true)][string[]]$properties) Set-StrictMode -version 2.0 @@ -356,16 +357,24 @@ function VerifyAssemblyVersionsAndSymbols() { } } -function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [string]$testadapterpath, [boolean] $asBackgroundJob = $false, [string] $settings = "") { +function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [string] $settings = "") { + $dotnetPath = InitializeDotNetCli $dotnetExe = Join-Path $dotnetPath "dotnet.exe" $projectName = [System.IO.Path]::GetFileNameWithoutExtension($testProject) + + $testBatchSuffix = "" + if ($testBatch) { + $testBatchSuffix = "_batch$testBatch" + } + # {assembly} and {framework} will expand respectively. See https://github.com/spekt/testlogger/wiki/Logger-Configuration#logfilepath # This is useful to deconflict log filenames when there are many test assemblies, e.g. when testing a whole solution. - $testLogPath = "$ArtifactsDir\TestResults\$configuration\{assembly}_{framework}.xml" - $testBinLogPath = "$LogDir\${projectName}_$targetFramework.binlog" - $args = "test $testProject -c $configuration -f $targetFramework -v n --test-adapter-path $testadapterpath --logger ""xunit;LogFilePath=$testLogPath"" /bl:$testBinLogPath" - $args += " --blame --blame-hang-timeout 5minutes --results-directory $ArtifactsDir\TestResults\$configuration -p:vstestusemsbuildoutput=true" + $testLogPath = "$ArtifactsDir\TestResults\$configuration\{assembly}_{framework}$testBatchSuffix.xml" + + $testBinLogPath = "$LogDir\${projectName}_$targetFramework$testBatch.binlog" + $args = "test $testProject -c $configuration -f $targetFramework --logger ""xunit;LogFilePath=$testLogPath"" /bl:$testBinLogPath" + $args += " --blame-hang-timeout 5minutes --results-directory $ArtifactsDir\TestResults\$configuration" if (-not $noVisualStudio -or $norestore) { $args += " --no-restore" @@ -376,22 +385,12 @@ function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [str } $args += " $settings" - - if ($asBackgroundJob) { - Write-Host - Write-Host("Starting on the background: $args") - Write-Host("------------------------------------") - Start-Job -ScriptBlock { - $argArray = $using:args -Split " " - & $using:dotnetExe $argArray - if ($LASTEXITCODE -ne 0) { - throw "Command failed to execute with exit code $($LASTEXITCODE): $using:dotnetExe $using:args" - } - } - } else { - Write-Host("$args") - Exec-Console $dotnetExe $args + if ($testBatch) { + $args += " --filter batch=$testBatch" } + + Write-Host("$args") + Exec-Console $dotnetExe $args } function Prepare-TempDir() { @@ -595,34 +594,12 @@ try { $script:BuildCategory = "Test" $script:BuildMessage = "Failure running tests" - function Receive($job) { - while($job.HasMoreData) { - Receive-Job $job | Write-Host - Start-Sleep -Seconds 1 - } - Receive-Job $job -Wait -ErrorAction Stop - } - if ($testCoreClr) { - $cpuLimit = if ($ci) { "-m:1 -- xUnit.MaxParallelThreads=1" } else { "" } - TestUsingMSBuild -testProject "$RepoRoot\FSharp.sln" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\" -settings $cpuLimit + TestUsingMSBuild -testProject "$RepoRoot\FSharp.sln" -targetFramework $script:coreclrTargetFramework } - if ($testDesktop -and $ci) { - $bgJob = TestUsingMSBuild -testProject "$RepoRoot\tests\fsharp\FSharpSuite.Tests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharpSuite.Tests\" -asBackgroundJob $true - - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Test.Utilities\FSharp.Test.Utilities.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Test.Utilities\" - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\" - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Service.Tests\FSharp.Compiler.Service.Tests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Service.Tests\" - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Private.Scripting.UnitTests\FSharp.Compiler.Private.Scripting.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Private.Scripting.UnitTests\" - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Build.UnitTests\FSharp.Build.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Build.UnitTests\" - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Core.UnitTests\FSharp.Core.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Core.UnitTests\" - - Receive -job $bgJob - } - - if ($testDesktop -and -not $ci ) { - TestUsingMSBuild -testProject "$RepoRoot\FSharp.sln" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\" + if ($testDesktop) { + TestUsingMSBuild -testProject "$RepoRoot\FSharp.sln" -targetFramework $script:desktopTargetFramework } if ($testFSharpQA) { @@ -653,50 +630,49 @@ try { } if ($testFSharpCore) { - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Core.UnitTests\FSharp.Core.UnitTests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Core.UnitTests\" - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Core.UnitTests\FSharp.Core.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Core.UnitTests\" + TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Core.UnitTests\FSharp.Core.UnitTests.fsproj" -targetFramework $script:coreclrTargetFramework + TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Core.UnitTests\FSharp.Core.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework } if ($testCompiler) { - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\" - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\" - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Service.Tests\FSharp.Compiler.Service.Tests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Service.Tests\" - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Service.Tests\FSharp.Compiler.Service.Tests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Service.Tests\" + TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:coreclrTargetFramework + TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:desktopTargetFramework + TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Service.Tests\FSharp.Compiler.Service.Tests.fsproj" -targetFramework $script:coreclrTargetFramework + TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Service.Tests\FSharp.Compiler.Service.Tests.fsproj" -targetFramework $script:desktopTargetFramework } if ($testCompilerComponentTests) { - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\" - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\" + TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:coreclrTargetFramework + TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:desktopTargetFramework } if ($testCompilerService) { - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Service.Tests\FSharp.Compiler.Service.Tests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Service.Tests\" - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Service.Tests\FSharp.Compiler.Service.Tests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Service.Tests\" + TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Service.Tests\FSharp.Compiler.Service.Tests.fsproj" -targetFramework $script:coreclrTargetFramework + TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Service.Tests\FSharp.Compiler.Service.Tests.fsproj" -targetFramework $script:desktopTargetFramework } if ($testCambridge) { - TestUsingMSBuild -testProject "$RepoRoot\tests\fsharp\FSharpSuite.Tests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharpSuite.Tests\" - TestUsingMSBuild -testProject "$RepoRoot\tests\fsharp\FSharpSuite.Tests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharpSuite.Tests\" + TestUsingMSBuild -testProject "$RepoRoot\tests\fsharp\FSharpSuite.Tests.fsproj" -targetFramework $script:coreclrTargetFramework + TestUsingMSBuild -testProject "$RepoRoot\tests\fsharp\FSharpSuite.Tests.fsproj" -targetFramework $script:desktopTargetFramework } if ($testScripting) { - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Private.Scripting.UnitTests\FSharp.Compiler.Private.Scripting.UnitTests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Private.Scripting.UnitTests\" - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Private.Scripting.UnitTests\FSharp.Compiler.Private.Scripting.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Private.Scripting.UnitTests\" + TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Private.Scripting.UnitTests\FSharp.Compiler.Private.Scripting.UnitTests.fsproj" -targetFramework $script:coreclrTargetFramework + TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Private.Scripting.UnitTests\FSharp.Compiler.Private.Scripting.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework } if ($testEditor -and -not $noVisualStudio) { - TestUsingMSBuild -testProject "$RepoRoot\vsintegration\tests\FSharp.Editor.Tests\FSharp.Editor.Tests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Editor.Tests\FSharp.Editor.Tests.fsproj" + TestUsingMSBuild -testProject "$RepoRoot\vsintegration\tests\FSharp.Editor.Tests\FSharp.Editor.Tests.fsproj" -targetFramework $script:desktopTargetFramework } if ($testVs -and -not $noVisualStudio) { - TestUsingMSBuild -testProject "$RepoRoot\vsintegration\tests\UnitTests\VisualFSharp.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\VisualFSharp.UnitTests\" + TestUsingMSBuild -testProject "$RepoRoot\vsintegration\tests\UnitTests\VisualFSharp.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework } - if ($testIntegration) { - TestUsingMSBuild -testProject "$RepoRoot\vsintegration\tests\FSharp.Editor.IntegrationTests\FSharp.Editor.IntegrationTests.csproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Editor.IntegrationTests\" + TestUsingMSBuild -testProject "$RepoRoot\vsintegration\tests\FSharp.Editor.IntegrationTests\FSharp.Editor.IntegrationTests.csproj" -targetFramework $script:desktopTargetFramework } if ($testAOT) { diff --git a/eng/build.sh b/eng/build.sh index 3c9404c8161..07c3889f33d 100755 --- a/eng/build.sh +++ b/eng/build.sh @@ -76,6 +76,7 @@ prepare_machine=false source_build=false product_build=false buildnorealsig=true +testbatch="" properties="" docker=false @@ -104,6 +105,11 @@ while [[ $# > 0 ]]; do args="$args $1" shift ;; + --testbatch) + testbatch=$2 + args="$args $1" + shift + ;; --verbosity|-v) verbosity=$2 args="$args $1" @@ -224,8 +230,16 @@ function Test() { projectname=$(basename -- "$testproject") projectname="${projectname%.*}" - testlogpath="$artifacts_dir/TestResults/$configuration/${projectname}_$targetframework.xml" - args="test \"$testproject\" --no-restore --no-build -c $configuration -f $targetframework --test-adapter-path . --logger \"xunit;LogFilePath=$testlogpath\" --blame-hang-timeout 5minutes --results-directory $artifacts_dir/TestResults/$configuration -p:vstestusemsbuildoutput=false" + testbatchsuffix="" + if [[ "$testbatch" != "" ]]; then + testbatchsuffix="_batch$testbatch" + fi + testlogpath="$artifacts_dir/TestResults/$configuration/${projectname}_$targetframework$testbatchsuffix.xml" + args="test \"$testproject\" --no-build -c $configuration -f $targetframework --logger \"xunit;LogFilePath=$testlogpath\" --blame-hang-timeout 5minutes --results-directory $artifacts_dir/TestResults/$configuration" + + if [[ "$testbatch" != "" ]]; then + args="$args --filter batch=$testbatch" + fi "$DOTNET_INSTALL_DIR/dotnet" $args || exit $? } diff --git a/tests/FSharp.Test.Utilities/XunitHelpers.fs b/tests/FSharp.Test.Utilities/XunitHelpers.fs index 50deb42ea68..6f57efef4a4 100644 --- a/tests/FSharp.Test.Utilities/XunitHelpers.fs +++ b/tests/FSharp.Test.Utilities/XunitHelpers.fs @@ -91,6 +91,23 @@ module TestCaseCustomizations = else testCase.TestMethod + let sha = Security.Cryptography.SHA256.Create() + + // We add extra trait to each test, of the form "batch=n" where n is between 1 and 4. + // It can be used to filter on in multi-agent testing in CI + // with dotnet test filter switch, for example "--filter batch=1" + // That way each agent can run test for a batch of tests. + let NumberOfBatchesInMultiAgentTesting = 4u + + let addBatchTrait (testCase: ITestCase) = + // Get a batch number stable between multiple test runs. + // UniqueID is ideal here, it does not change across many compilations of the same code + // and it will split theories with member data into many batches. + let data = Text.Encoding.UTF8.GetBytes testCase.UniqueID + let hashCode = BitConverter.ToUInt32(sha.ComputeHash(data), 0) + let batch = hashCode % NumberOfBatchesInMultiAgentTesting + 1u + testCase.Traits.Add("batch", ResizeArray [ string batch ]) + type CustomTestCase = inherit XunitTestCase // xUinit demands this constructor for deserialization. @@ -110,6 +127,7 @@ type CustomTestCase = override testCase.Initialize () = base.Initialize() testCase.TestMethod <- TestCaseCustomizations.rewriteTestMethod testCase + TestCaseCustomizations.addBatchTrait testCase type CustomTheoryTestCase = inherit XunitTheoryTestCase @@ -128,6 +146,7 @@ type CustomTheoryTestCase = override testCase.Initialize () = base.Initialize() testCase.TestMethod <- TestCaseCustomizations.rewriteTestMethod testCase + TestCaseCustomizations.addBatchTrait testCase #endif