Add a flag to hostfxr_resolve_sdk2_flags_t to silence error messages (dotnet#119729)

hostfxr_resolve_sdk2 will print error messages to stderr by default. We can add a flag to the enum to allow for silencing these errors when we want to try to resolve an sdk, but a failure to resolve shouldn't be surfaced to the console.

See dotnet/sdk#50632 for context.

Author: jtschuster

docs/design/features/hosting-layer-apis.md

@@ -61,6 +61,7 @@ If the application is successfully executed, this value will return the exit cod
 enum hostfxr_resolve_sdk2_flags_t
 {
     disallow_prerelease = 0x1,
+    do_not_print_errors = 0x2,
 };
 
 enum hostfxr_resolve_sdk2_result_key_t

src/installer/tests/Assets/Projects/HostApiInvokerApp/HostFXR.cs

@@ -16,6 +16,7 @@ internal static class hostfxr
             internal enum hostfxr_resolve_sdk2_flags_t : int
             {
                 disallow_prerelease = 0x1,
+                do_not_print_errors = 0x2,
             }
 
             internal enum hostfxr_resolve_sdk2_result_key_t : int

src/installer/tests/HostActivation.Tests/NativeHostApis.cs

@@ -159,6 +159,30 @@ public void Hostfxr_resolve_sdk2_NoGlobalJson_DisallowPrerelease()
                 .And.HaveStdOutContaining($"{api} data:[{expectedData}]");
         }
 
+        [Theory]
+        [InlineData(true)]
+        [InlineData(false)]
+        public void Hostfxr_resolve_sdk2_NoGlobalJson_NoWriteToStdResolvesTheSame(bool do_not_print_errors)
+        {
+            // With no global.json and disallowing previews, pick latest non-preview
+
+            var f = sharedTestState.SdkAndFrameworkFixture;
+            string expectedData = string.Join(';', new[]
+            {
+                ("resolved_sdk_dir", Path.Combine(f.LocalSdkDir, "1.2.300")),
+                ("global_json_state", "not_found"),
+            });
+
+            string api = ApiNames.hostfxr_resolve_sdk2;
+            string flags = do_not_print_errors ? "disallow_prerelease,do_not_print_errors" : "disallow_prerelease";
+            TestContext.BuiltDotNet.Exec(sharedTestState.HostApiInvokerApp.AppDll, api, f.ExeDir, NoGlobalJson, flags)
+                .EnableTracingAndCaptureOutputs()
+                .Execute()
+                .Should().Pass()
+                .And.ReturnStatusCode(api, Constants.ErrorCode.Success)
+                .And.HaveStdOutContaining($"{api} data:[{expectedData}]");
+        }
+
         [Fact]
         public void Hostfxr_resolve_sdk2_GlobalJson_DisallowPrerelease()
         {
@@ -301,6 +325,32 @@ public void Hostfxr_resolve_sdk2_GlobalJson_InvalidDataNoFallback()
             }
         }
 
+        [Theory]
+        [InlineData(false)]
+        [InlineData(true)]
+        public void Hostfxr_resolve_sdk2_FailsToResolve_NoWriteToStd(bool do_not_print_errors)
+        {
+            var f = sharedTestState.SdkAndFrameworkFixture;
+            string api = ApiNames.hostfxr_resolve_sdk2;
+            using TestArtifact workingDir = TestArtifact.Create(nameof(Hostfxr_resolve_sdk2_FailsToResolve_NoWriteToStd));
+
+            string invalidVersion = "1.2.0"; // feature band < 1 triggers __invalid_data_no_fallback
+            GlobalJson.CreateWithVersion(workingDir.Location, invalidVersion);
+
+            var result = TestContext.BuiltDotNet.Exec(sharedTestState.HostApiInvokerApp.AppDll, api, f.ExeDir, workingDir.Location, do_not_print_errors ? "do_not_print_errors" : "0")
+                .CaptureStdOut()
+                .CaptureStdErr()
+                .Execute();
+
+            result.Should().Pass()
+                .And.ReturnStatusCode(api, Constants.ErrorCode.SdkResolveFailure);
+
+            if (do_not_print_errors)
+                result.StdErr.Should().BeEmpty();
+            else
+                result.StdErr.Should().Contain("A compatible .NET SDK was not found.");
+        }
+
         [Fact]
         public void Hostfxr_corehost_set_error_writer_test()
         {

src/native/corehost/fxr/hostfxr.cpp

@@ -166,6 +166,7 @@ SHARED_API int32_t HOSTFXR_CALLTYPE hostfxr_resolve_sdk(
 enum hostfxr_resolve_sdk2_flags_t : int32_t
 {
     disallow_prerelease = 0x1,
+    do_not_print_errors = 0x2,
 };
 
 enum class hostfxr_resolve_sdk2_result_key_t : int32_t
@@ -218,6 +219,8 @@ namespace
 //         disallow_prerelease (0x1)
 //           do not allow resolution to return a prerelease SDK version
 //           unless  prerelease version was specified via global.json.
+//         do_not_print_errors (0x2)
+//           do not write any error messages to stderr.
 //
 //   result
 //      Callback invoked to return values. It can be invoked more
@@ -282,7 +285,9 @@ SHARED_API int32_t HOSTFXR_CALLTYPE hostfxr_resolve_sdk2(
         working_dir,
         (flags & hostfxr_resolve_sdk2_flags_t::disallow_prerelease) == 0);
 
-    auto resolved_sdk_dir = resolver.resolve(exe_dir);
+    auto resolved_sdk_dir = resolver.resolve(
+        exe_dir,
+        (flags & hostfxr_resolve_sdk2_flags_t::do_not_print_errors) == 0);
     if (!resolved_sdk_dir.empty())
     {
         result(