From 38bfc1ec73f89d426b02d02f9b0f2beaee3d9a72 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Fri, 21 Feb 2025 09:49:01 -0800 Subject: [PATCH 1/3] Skip Flakey GivenDotnetTest Tests In this PR a few days ago I disabled some similar tests: https://github.com/dotnet/sdk/pull/46924 It turns out that even more of them are failing about 80%+ of the time after some recent changes to these tests. See here: https://github.com/dotnet/sdk/pull/47003#pullrequestreview-2633807953 here: https://github.com/dotnet/sdk/pull/47004#pullrequestreview-2633808248 here: https://github.com/dotnet/sdk/pull/47004/checks?check_run_id=37586911841 here: https://github.com/dotnet/sdk/pull/47014 etc. I've already made an issue to track this: https://github.com/dotnet/sdk/issues/46923 So I'll keep using that one. The maintainer has been notified. --- .../GivenDotnetTestBuildsAndRunsTestsForMultipleTFMs.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestsForMultipleTFMs.cs b/test/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestsForMultipleTFMs.cs index 171700bea1b2..8e5727569a7d 100644 --- a/test/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestsForMultipleTFMs.cs +++ b/test/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestsForMultipleTFMs.cs @@ -14,7 +14,7 @@ public GivenDotnetTestBuildsAndRunsTestsForMultipleTFMs(ITestOutputHelper log) : [InlineData(TestingConstants.Debug)] [InlineData(TestingConstants.Release)] - [Theory] + [Theory(Skip= "https://github.com/dotnet/sdk/issues/46923")] public void RunMultipleProjectWithDifferentTFMs_ShouldReturnExitCodeGenericFailure(string configuration) { TestAsset testInstance = _testAssetsManager.CopyTestAsset("ProjectSolutionForMultipleTFMs", Guid.NewGuid().ToString()) @@ -53,7 +53,7 @@ public void RunMultipleProjectWithDifferentTFMs_ShouldReturnExitCodeGenericFailu [InlineData(TestingConstants.Debug)] [InlineData(TestingConstants.Release)] - [Theory] + [Theory(Skip= "https://github.com/dotnet/sdk/issues/46923")] public void RunProjectWithMultipleTFMs_ShouldReturnExitCodeGenericFailure(string configuration) { TestAsset testInstance = _testAssetsManager.CopyTestAsset("TestProjectWithMultipleTFMsSolution", Guid.NewGuid().ToString()) @@ -100,7 +100,7 @@ public void RunProjectWithMultipleTFMs_ShouldReturnExitCodeGenericFailure(string [InlineData(TestingConstants.Debug)] [InlineData(TestingConstants.Release)] - [Theory] + [Theory(Skip= "https://github.com/dotnet/sdk/issues/46923")] public void RunProjectWithMultipleTFMsWithArchOption_ShouldReturnExitCodeGenericFailure(string configuration) { TestAsset testInstance = _testAssetsManager.CopyTestAsset("TestProjectWithMultipleTFMsSolution", Guid.NewGuid().ToString()) @@ -128,7 +128,7 @@ public void RunProjectWithMultipleTFMsWithArchOption_ShouldReturnExitCodeGeneric [InlineData(TestingConstants.Debug)] [InlineData(TestingConstants.Release)] - [Theory] + [Theory(Skip= "https://github.com/dotnet/sdk/issues/46923")] public void RunProjectWithMSTestMetaPackageAndMultipleTFMs_ShouldReturnExitCodeGenericFailure(string configuration) { TestAsset testInstance = _testAssetsManager.CopyTestAsset("MSTestMetaPackageProjectWithMultipleTFMsSolution", Guid.NewGuid().ToString()) From e24c3686518b50468701d6ede44438426702b1ba Mon Sep 17 00:00:00 2001 From: Jason Zhai Date: Mon, 24 Feb 2025 22:50:53 -0800 Subject: [PATCH 2/3] Continue skipping the failing GivenDotnetTest tests to unblock --- .../dotnet-test.Tests/GivenDotnetTestBuildsAndDiscoversTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/dotnet-test.Tests/GivenDotnetTestBuildsAndDiscoversTests.cs b/test/dotnet-test.Tests/GivenDotnetTestBuildsAndDiscoversTests.cs index e592f717be12..1a2733a57f3d 100644 --- a/test/dotnet-test.Tests/GivenDotnetTestBuildsAndDiscoversTests.cs +++ b/test/dotnet-test.Tests/GivenDotnetTestBuildsAndDiscoversTests.cs @@ -106,7 +106,7 @@ public void DiscoverMultipleTestProjectsWithTests_ShouldReturnExitCodeSuccess(st [InlineData(TestingConstants.Debug)] [InlineData(TestingConstants.Release)] - [Theory] + [Theory(Skip= "https://github.com/dotnet/sdk/issues/46923")] public void DiscoverProjectWithMSTestMetaPackageAndMultipleTFMsWithTests_ShouldReturnExitCodeSuccess(string configuration) { TestAsset testInstance = _testAssetsManager.CopyTestAsset("MSTestMetaPackageProjectWithMultipleTFMsSolution", Guid.NewGuid().ToString()) From a5c364d34448763e7ca987d0d5caa3ce76b82b58 Mon Sep 17 00:00:00 2001 From: Jacques Eloff Date: Tue, 25 Feb 2025 13:03:56 -0800 Subject: [PATCH 3/3] Reduce skipped tests and fix package feed --- NuGet.config | 2 ++ .../GivenDotnetTestBuildsAndRunsTestsForMultipleTFMs.cs | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/NuGet.config b/NuGet.config index ecf2c176d5ad..29933b739dfc 100644 --- a/NuGet.config +++ b/NuGet.config @@ -28,6 +28,8 @@ + + diff --git a/test/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestsForMultipleTFMs.cs b/test/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestsForMultipleTFMs.cs index 8e5727569a7d..171700bea1b2 100644 --- a/test/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestsForMultipleTFMs.cs +++ b/test/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestsForMultipleTFMs.cs @@ -14,7 +14,7 @@ public GivenDotnetTestBuildsAndRunsTestsForMultipleTFMs(ITestOutputHelper log) : [InlineData(TestingConstants.Debug)] [InlineData(TestingConstants.Release)] - [Theory(Skip= "https://github.com/dotnet/sdk/issues/46923")] + [Theory] public void RunMultipleProjectWithDifferentTFMs_ShouldReturnExitCodeGenericFailure(string configuration) { TestAsset testInstance = _testAssetsManager.CopyTestAsset("ProjectSolutionForMultipleTFMs", Guid.NewGuid().ToString()) @@ -53,7 +53,7 @@ public void RunMultipleProjectWithDifferentTFMs_ShouldReturnExitCodeGenericFailu [InlineData(TestingConstants.Debug)] [InlineData(TestingConstants.Release)] - [Theory(Skip= "https://github.com/dotnet/sdk/issues/46923")] + [Theory] public void RunProjectWithMultipleTFMs_ShouldReturnExitCodeGenericFailure(string configuration) { TestAsset testInstance = _testAssetsManager.CopyTestAsset("TestProjectWithMultipleTFMsSolution", Guid.NewGuid().ToString()) @@ -100,7 +100,7 @@ public void RunProjectWithMultipleTFMs_ShouldReturnExitCodeGenericFailure(string [InlineData(TestingConstants.Debug)] [InlineData(TestingConstants.Release)] - [Theory(Skip= "https://github.com/dotnet/sdk/issues/46923")] + [Theory] public void RunProjectWithMultipleTFMsWithArchOption_ShouldReturnExitCodeGenericFailure(string configuration) { TestAsset testInstance = _testAssetsManager.CopyTestAsset("TestProjectWithMultipleTFMsSolution", Guid.NewGuid().ToString()) @@ -128,7 +128,7 @@ public void RunProjectWithMultipleTFMsWithArchOption_ShouldReturnExitCodeGeneric [InlineData(TestingConstants.Debug)] [InlineData(TestingConstants.Release)] - [Theory(Skip= "https://github.com/dotnet/sdk/issues/46923")] + [Theory] public void RunProjectWithMSTestMetaPackageAndMultipleTFMs_ShouldReturnExitCodeGenericFailure(string configuration) { TestAsset testInstance = _testAssetsManager.CopyTestAsset("MSTestMetaPackageProjectWithMultipleTFMsSolution", Guid.NewGuid().ToString())