Preserve console encoding when using --utf8output switch without swapping console streams (#17761)

* do not modify stdout

* preserve original encoding

* Revert "preserve original encoding"

This reverts commit 9665470.

* add a test

* wip

* fantomas

* add fsi test

* utf16

---------

Co-authored-by: Tomas Grosup <[email protected]>

Author: majocha

src/Compiler/Driver/fsc.fs

@@ -273,9 +273,6 @@ let SetProcessThreadLocals tcConfigB =
     | Some s -> Thread.CurrentThread.CurrentUICulture <- CultureInfo(s)
     | None -> ()
 
-    if tcConfigB.utf8output then
-        Console.OutputEncoding <- Encoding.UTF8
-
 let ProcessCommandLineFlags (tcConfigB: TcConfigBuilder, lcidFromCodePage, argv) =
     let mutable inputFilesRef = []
 
@@ -550,6 +547,17 @@ let main1
     | Some parallelReferenceResolution -> tcConfigB.parallelReferenceResolution <- parallelReferenceResolution
     | None -> ()
 
+    if tcConfigB.utf8output && Console.OutputEncoding <> Encoding.UTF8 then
+        let previousEncoding = Console.OutputEncoding
+        Console.OutputEncoding <- Encoding.UTF8
+
+        disposables.Register(
+            { new IDisposable with
+                member _.Dispose() =
+                    Console.OutputEncoding <- previousEncoding
+            }
+        )
+
     // Display the banner text, if necessary
     if not bannerAlreadyPrinted then
         Console.Write(GetBannerText tcConfigB)
@@ -1242,16 +1250,6 @@ let CompileFromCommandLineArguments
     ) =
 
     use disposables = new DisposablesTracker()
-    let savedOut = Console.Out
-
-    use _ =
-        { new IDisposable with
-            member _.Dispose() =
-                try
-                    Console.SetOut(savedOut)
-                with _ ->
-                    ()
-        }
 
     main1 (
         ctok,

src/Compiler/Interactive/fsi.fs

@@ -846,7 +846,7 @@ type internal FsiStdinSyphon(errorWriter: TextWriter) =
 /// Encapsulates functions used to write to outWriter and errorWriter
 type internal FsiConsoleOutput(tcConfigB, outWriter: TextWriter, errorWriter: TextWriter) =
 
-    let nullOut = new StreamWriter(Stream.Null) :> TextWriter
+    let nullOut = TextWriter.Null
 
     let fprintfnn (os: TextWriter) fmt =
         Printf.kfprintf
@@ -1203,11 +1203,6 @@ type internal FsiCommandLineOptions(fsi: FsiEvaluationSessionHostConfig, argv: s
         if tcConfigB.clearResultsCache then
             dependencyProvider.ClearResultsCache(tcConfigB.compilerToolPaths, getOutputDir tcConfigB, reportError rangeCmdArgs)
 
-        if tcConfigB.utf8output then
-            let prev = Console.OutputEncoding
-            Console.OutputEncoding <- Encoding.UTF8
-            System.AppDomain.CurrentDomain.ProcessExit.Add(fun _ -> Console.OutputEncoding <- prev)
-
     do
         let firstArg =
             match sourceFiles with
@@ -4646,6 +4641,20 @@ type FsiEvaluationSession
         with e ->
             warning (e)
 
+    let restoreEncoding =
+        if tcConfigB.utf8output && Console.OutputEncoding <> Text.Encoding.UTF8 then
+            let previousEncoding = Console.OutputEncoding
+            Console.OutputEncoding <- Encoding.UTF8
+
+            Some(
+                { new IDisposable with
+                    member _.Dispose() =
+                        Console.OutputEncoding <- previousEncoding
+                }
+            )
+        else
+            None
+
     do
         updateBannerText () // resetting banner text after parsing options
 
@@ -4789,6 +4798,7 @@ type FsiEvaluationSession
         member _.Dispose() =
             (tcImports :> IDisposable).Dispose()
             uninstallMagicAssemblyResolution.Dispose()
+            restoreEncoding |> Option.iter (fun x -> x.Dispose())
 
     /// Load the dummy interaction, load the initial files, and,
     /// if interacting, start the background thread to read the standard input.

src/Compiler/Service/service.fs

@@ -100,14 +100,6 @@ module CompileHelpers =
 
         diagnostics.ToArray(), result
 
-    let setOutputStreams execute =
-        // Set the output streams, if requested
-        match execute with
-        | Some(writer, error) ->
-            Console.SetOut writer
-            Console.SetError error
-        | None -> ()
-
 [<Sealed; AutoSerializable(false)>]
 // There is typically only one instance of this type in an IDE process.
 type FSharpChecker

src/fsi/fsimain.fs

@@ -358,16 +358,6 @@ let evaluateSession (argv: string[]) =
 let MainMain argv =
     ignore argv
     let argv = System.Environment.GetCommandLineArgs()
-    let savedOut = Console.Out
-
-    use __ =
-        { new IDisposable with
-            member _.Dispose() =
-                try
-                    Console.SetOut(savedOut)
-                with _ ->
-                    ()
-        }
 
     let timesFlag = argv |> Array.exists (fun x -> x = "/times" || x = "--times")
 

tests/FSharp.Compiler.ComponentTests/CompilerOptions/fsc/misc/utf8output.fs

@@ -0,0 +1,47 @@
+// Copyright (c) Microsoft Corporation.  All Rights Reserved.  See License.txt in the project root for license information.
+
+namespace CompilerOptions.Fsc
+
+open Xunit
+open FSharp.Test
+open FSharp.Test.Compiler
+open System
+
+module utf8output =
+
+    [<Fact>]
+    let ``OutputEncoding is restored after executing compilation`` () =
+        let currentEncoding = Console.OutputEncoding
+        use restoreCurrentEncodingAfterTest = { new IDisposable with member _.Dispose() = Console.OutputEncoding <- currentEncoding }
+
+        // UTF16
+        let encoding = Text.Encoding.Unicode
+
+        Console.OutputEncoding <- encoding
+
+        Fs """printfn "Hello world" """
+        |> asExe
+        |> withOptionsString "--utf8output"
+        |> compile
+        |> shouldSucceed
+        |> ignore
+
+        Console.OutputEncoding.BodyName |> Assert.shouldBe encoding.BodyName
+
+    [<Fact>]
+    let ``OutputEncoding is restored after running script`` () =
+        let currentEncoding = Console.OutputEncoding
+        use restoreCurrentEncodingAfterTest = { new IDisposable with member _.Dispose() = Console.OutputEncoding <- currentEncoding }
+
+        // UTF16
+        let encoding = Text.Encoding.Unicode
+
+        Console.OutputEncoding <- encoding
+
+        Fsx """printfn "Hello world" """
+        |> withOptionsString "--utf8output"
+        |> runFsi
+        |> shouldSucceed
+        |> ignore
+
+        Console.OutputEncoding.BodyName |> Assert.shouldBe encoding.BodyName

tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj

@@ -288,6 +288,7 @@
     <Compile Include="CompilerOptions\fsc\highentropyva.fs" />
     <Compile Include="CompilerOptions\fsc\langversion.fs" />
     <Compile Include="CompilerOptions\fsc\misc\misc.fs" />
+    <Compile Include="CompilerOptions\fsc\misc\utf8output.fs" />
     <Compile Include="CompilerOptions\fsc\noframework\noframework.fs" />
     <Compile Include="CompilerOptions\fsc\platform\platform.fs" />
     <Compile Include="CompilerOptions\fsc\times\times.fs" />