xUnit tests - improve assembly fixture (#18381)

* xUnit - move custom initialization from test framework constructor to executor

* Update tests/FSharp.Test.Utilities/XunitHelpers.fs

Co-authored-by: Tomas Grosup <[email protected]>

---------

Co-authored-by: Petr <[email protected]>
Co-authored-by: Tomas Grosup <[email protected]>

Author: 3 people

tests/FSharp.Test.Utilities/XunitHelpers.fs

@@ -21,29 +21,7 @@ open OpenTelemetry.Trace
 [<AttributeUsage(AttributeTargets.Class ||| AttributeTargets.Method, AllowMultiple = false)>]
 type RunTestCasesInSequenceAttribute() = inherit Attribute()
 
-#if !XUNIT_EXTRAS
-/// Installs console support for parallel test runs and conditionally enables optional xUnit customizations.
-type FSharpXunitFramework(sink: IMessageSink) =
-    inherit XunitTestFramework(sink)
-    do
-        // Because xUnit v2 lacks assembly fixture, the next best place to ensure things get called
-        // right at the start of the test run is here in the constructor.
-        // This gets executed once per test assembly.
-        logConfig initialConfig
-        log "FSharpXunitFramework installing TestConsole redirection"
-        TestConsole.install()
-#if !NETCOREAPP
-        AssemblyResolver.addResolver ()
-#endif
-
-    interface IDisposable with
-        member _.Dispose() =
-            match Environment.GetEnvironmentVariable("FSHARP_RETAIN_TESTBUILDS") with
-            | null -> cleanUpTemporaryDirectoryOfThisTestRun  ()
-            | _ -> ()
-            base.Dispose()
-
-#else
+#if XUNIT_EXTRAS
 
 // To use xUnit means to customize it. The following abomination adds 2 features:
 // - Capturing full console output individually for each test case, viewable in Test Explorer as test stdout.
@@ -57,7 +35,7 @@ type ConsoleCapturingTestRunner(test, messageBus, testClass, constructorArgument
     override this.InvokeTestAsync (aggregator: ExceptionAggregator) =
         task {
             use capture = new TestConsole.ExecutionCapture()
-            use _ = Activity.start test.DisplayName [  ]
+            use _ = Activity.startNoTags test.DisplayName
             let! executionTime = this.BaseInvokeTestMethodAsync aggregator
             let output =
                 seq {
@@ -142,46 +120,55 @@ type CustomTheoryTestCase =
         base.Initialize()
         testCase.TestMethod <- TestCaseCustomizations.rewriteTestMethod testCase
 
+#endif
+
 /// `XunitTestFramework` providing parallel console support and conditionally enabling optional xUnit customizations.
 type FSharpXunitFramework(sink: IMessageSink) =
     inherit XunitTestFramework(sink)
-
-    let traceProvider =
-        Sdk.CreateTracerProviderBuilder()
-                .AddSource(ActivityNames.FscSourceName)
-                .SetResourceBuilder(
-                    ResourceBuilder.CreateDefault().AddService(serviceName="F#", serviceVersion = "1.0.0"))
-                .AddOtlpExporter()
-                .Build()
-
-    interface IDisposable with
-        member _.Dispose() =
-            cleanUpTemporaryDirectoryOfThisTestRun ()
-            traceProvider.ForceFlush() |> ignore
-            traceProvider.Dispose()
-            base.Dispose()
 
-    // Group test run under single activity, to make traces more readable.
-    // Otherwise this overriden method is not necessary and can be removed.
-    override this.CreateExecutor (assemblyName) = 
+    override this.CreateExecutor (assemblyName) =
         { new XunitTestFrameworkExecutor(assemblyName, this.SourceInformationProvider, this.DiagnosticMessageSink) with
-            override _.RunTestCases(testCases, executionMessageSink, executionOptions) =
-                use _ = Activity.start $"{assemblyName.Name} {Runtime.InteropServices.RuntimeInformation.FrameworkDescription}" []
+            
+            // Because xUnit v2 lacks assembly fixture, this is a good place to ensure things get called right at the start of the test run.
+            // This gets executed once per test assembly.
+            override x.RunTestCases(testCases, executionMessageSink, executionOptions) =
+
+            #if !NETCOREAPP
+                // We need AssemblyResolver already here, because OpenTelemetry loads some assemblies dynamically.
+                AssemblyResolver.addResolver ()
+            #endif
+                
+                // Configure OpenTelemetry export. Traces can be viewed in Jaeger or other compatible tools.
+                use tracerProvider =
+                    OpenTelemetry.Sdk.CreateTracerProviderBuilder()
+                        .AddSource(ActivityNames.FscSourceName)
+                        .ConfigureResource(fun r -> r.AddService("F#") |> ignore)
+                        .AddOtlpExporter(fun o ->
+                            // Empirical values to ensure no traces are lost and no significant delay at the end of test run.
+                            o.TimeoutMilliseconds <- 200
+                            o.BatchExportProcessorOptions.MaxQueueSize <- 16384
+                            o.BatchExportProcessorOptions.ScheduledDelayMilliseconds <- 100
+                        )
+                        .Build()
 
-                // Because xUnit v2 lacks assembly fixture, the next best place to ensure things get called
-                // right at the start of the test run is here or in the FSharpXunitFramework constructor.
-                // This gets executed once per test assembly.
-                printfn $"Running tests in {assemblyName.Name} with XUNIT_EXTRAS"
                 logConfig initialConfig
                 log "Installing TestConsole redirection"
                 TestConsole.install()
+              
+                begin
+                    use _ = Activity.startNoTags $"RunTests_{assemblyName.Name} {Runtime.InteropServices.RuntimeInformation.FrameworkDescription}"
+                    // We can't just call base.RunTestCases here, because it's implementation is async void.
+                    use runner = new XunitTestAssemblyRunner (x.TestAssembly, testCases, x.DiagnosticMessageSink, executionMessageSink, executionOptions)
+                    runner.RunAsync().Wait()
+                end
 
-#if !NETCOREAPP
-                AssemblyResolver.addResolver ()
-#endif
-                base.RunTestCases(testCases, executionMessageSink, executionOptions)
+                tracerProvider.ForceFlush() |> ignore
+
+                cleanUpTemporaryDirectoryOfThisTestRun ()
         }
 
+#if XUNIT_EXTRAS
+    // Rewrites discovered test cases to support extra parallelization and capturing console as test output.
     override this.CreateDiscoverer (assemblyInfo) =
         { new XunitTestFrameworkDiscoverer(assemblyInfo, this.SourceInformationProvider, this.DiagnosticMessageSink) with
             override _.FindTestsForType (testClass, includeSourceInformation, messageBus, options) =