Add tests for multiple in-process diagnosers with varying run modes

tests/BenchmarkDotNet.IntegrationTests/Diagnosers/MockInProcessDiagnoser.cs

@@ -11,40 +11,122 @@
 
 namespace BenchmarkDotNet.IntegrationTests.Diagnosers;
 
-public sealed class MockInProcessDiagnoser : IInProcessDiagnoser
+public abstract class BaseMockInProcessDiagnoser : IInProcessDiagnoser
 {
+    public static Queue<BaseMockInProcessDiagnoser> s_completedResults = new();
+
     public Dictionary<BenchmarkCase, string> Results { get; } = [];
 
-    public IEnumerable<string> Ids => [nameof(MockInProcessDiagnoser)];
+    public abstract string DiagnoserName { get; }
+    public abstract RunMode DiagnoserRunMode { get; }
+    public abstract string ExpectedResult { get; }
+
+    public IEnumerable<string> Ids => [DiagnoserName];
 
     public IEnumerable<IExporter> Exporters => [];
 
     public IEnumerable<IAnalyser> Analysers => [];
 
-    public void DisplayResults(ILogger logger) => logger.WriteLine($"{nameof(MockInProcessDiagnoser)} results: [{string.Join(", ", Results.Values)}]");
+    public void DisplayResults(ILogger logger) => logger.WriteLine($"{DiagnoserName} results: [{string.Join(", ", Results.Values)}]");
 
-    public RunMode GetRunMode(BenchmarkCase benchmarkCase) => RunMode.NoOverhead;
+    public RunMode GetRunMode(BenchmarkCase benchmarkCase) => DiagnoserRunMode;
 
     public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { }
 
     public IEnumerable<Metric> ProcessResults(DiagnoserResults results) => [];
 
     public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters) => [];
 
-    public (Type? handlerType, string? serializedConfig) GetSeparateProcessHandlerTypeAndSerializedConfig(BenchmarkCase benchmarkCase)
-        => (typeof(MockInProcessDiagnoserHandler), null);
+    public abstract (Type? handlerType, string? serializedConfig) GetSeparateProcessHandlerTypeAndSerializedConfig(BenchmarkCase benchmarkCase);
+
+    public virtual IInProcessDiagnoserHandler? GetSameProcessHandler(BenchmarkCase benchmarkCase)
+    {
+        var (handlerType, serializedConfig) = GetSeparateProcessHandlerTypeAndSerializedConfig(benchmarkCase);
+        if (handlerType == null)
+            return null;
+        var handler = (IInProcessDiagnoserHandler)Activator.CreateInstance(handlerType);
+        handler.Initialize(serializedConfig);
+        return handler;
+    }
+
+    public void DeserializeResults(BenchmarkCase benchmarkCase, string results)
+    {
+        Results.Add(benchmarkCase, results);
+        s_completedResults.Enqueue(this);
+    }
+}
+
+public abstract class BaseMockInProcessDiagnoserHandler : IInProcessDiagnoserHandler
+{
+    private string _result;
 
-    public IInProcessDiagnoserHandler? GetSameProcessHandler(BenchmarkCase benchmarkCase)
-        => new MockInProcessDiagnoserHandler();
+    protected BaseMockInProcessDiagnoserHandler() { }
 
-    public void DeserializeResults(BenchmarkCase benchmarkCase, string results) => Results.Add(benchmarkCase, results);
+    public void Initialize(string? serializedConfig)
+    {
+        _result = serializedConfig ?? string.Empty;
+    }
+
+    public void Handle(BenchmarkSignal signal, InProcessDiagnoserActionArgs args) { }
+
+    public string SerializeResults() => _result;
 }
 
-public sealed class MockInProcessDiagnoserHandler : IInProcessDiagnoserHandler
+public sealed class MockInProcessDiagnoser : BaseMockInProcessDiagnoser
 {
-    public void Initialize(string? serializedConfig) { }
+    public override string DiagnoserName => nameof(MockInProcessDiagnoser);
+    public override RunMode DiagnoserRunMode => RunMode.NoOverhead;
+    public override string ExpectedResult => "MockResult";
 
-    public void Handle(BenchmarkSignal signal, InProcessDiagnoserActionArgs args) { }
+    public override (Type? handlerType, string? serializedConfig) GetSeparateProcessHandlerTypeAndSerializedConfig(BenchmarkCase benchmarkCase)
+        => (typeof(MockInProcessDiagnoserHandler), ExpectedResult);
+}
+
+public sealed class MockInProcessDiagnoserHandler : BaseMockInProcessDiagnoserHandler
+{
+}
+
+public sealed class MockInProcessDiagnoserExtraRun : BaseMockInProcessDiagnoser
+{
+    public override string DiagnoserName => nameof(MockInProcessDiagnoserExtraRun);
+    public override RunMode DiagnoserRunMode => RunMode.ExtraRun;
+    public override string ExpectedResult => "ExtraRunResult";
+
+    public override (Type? handlerType, string? serializedConfig) GetSeparateProcessHandlerTypeAndSerializedConfig(BenchmarkCase benchmarkCase)
+        => (typeof(MockInProcessDiagnoserExtraRunHandler), ExpectedResult);
+}
 
-    public string SerializeResults() => "MockResult";
+public sealed class MockInProcessDiagnoserExtraRunHandler : BaseMockInProcessDiagnoserHandler
+{
+}
+
+public sealed class MockInProcessDiagnoserNone : BaseMockInProcessDiagnoser
+{
+    public override string DiagnoserName => nameof(MockInProcessDiagnoserNone);
+    public override RunMode DiagnoserRunMode => RunMode.None;
+    public override string ExpectedResult => "NoneResult";
+
+    public override (Type? handlerType, string? serializedConfig) GetSeparateProcessHandlerTypeAndSerializedConfig(BenchmarkCase benchmarkCase)
+        => default; // Returns default when RunMode is None
+
+    public override IInProcessDiagnoserHandler? GetSameProcessHandler(BenchmarkCase benchmarkCase)
+        => null; // Returns null when RunMode is None
+}
+
+public sealed class MockInProcessDiagnoserNoneHandler : BaseMockInProcessDiagnoserHandler
+{
+}
+
+public sealed class MockInProcessDiagnoserSeparateLogic : BaseMockInProcessDiagnoser
+{
+    public override string DiagnoserName => nameof(MockInProcessDiagnoserSeparateLogic);
+    public override RunMode DiagnoserRunMode => RunMode.SeparateLogic;
+    public override string ExpectedResult => "SeparateLogicResult";
+
+    public override (Type? handlerType, string? serializedConfig) GetSeparateProcessHandlerTypeAndSerializedConfig(BenchmarkCase benchmarkCase)
+        => (typeof(MockInProcessDiagnoserSeparateLogicHandler), ExpectedResult);
+}
+
+public sealed class MockInProcessDiagnoserSeparateLogicHandler : BaseMockInProcessDiagnoserHandler
+{
 }

tests/BenchmarkDotNet.IntegrationTests/InProcessDiagnoserTests.cs

@@ -0,0 +1,150 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Columns;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.IntegrationTests.Diagnosers;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Tests.Loggers;
+using BenchmarkDotNet.Toolchains;
+using BenchmarkDotNet.Toolchains.InProcess.Emit;
+using BenchmarkDotNet.Toolchains.InProcess.NoEmit;
+using Xunit;
+using Xunit.Abstractions;
+using RunMode = BenchmarkDotNet.Diagnosers.RunMode;
+
+namespace BenchmarkDotNet.IntegrationTests;
+
+public class InProcessDiagnoserTests(ITestOutputHelper output) : BenchmarkTestExecutor(output)
+{
+    // For test explorer since it doesn't handle interfaces well.
+    public enum ToolchainType
+    {
+        Default,
+        InProcessEmit,
+        InProcessNoEmit,
+    }
+
+    private static IEnumerable<RunMode[]> GetRunModeCombinations(int count)
+    {
+        var runModes = (RunMode[]) Enum.GetValues(typeof(RunMode));
+
+        if (count == 1)
+        {
+            foreach (var runMode in runModes)
+            {
+                yield return [runMode];
+            }
+        }
+        else if (count == 3)
+        {
+            foreach (var runMode1 in runModes)
+            {
+                foreach (var runMode2 in runModes)
+                {
+                    foreach (var runMode3 in runModes)
+                    {
+                        yield return [runMode1, runMode2, runMode3];
+                    }
+                }
+            }
+        }
+    }
+
+    public static IEnumerable<object[]> GetTestCombinations()
+    {
+        var toolchains = (ToolchainType[]) Enum.GetValues(typeof(ToolchainType));
+        var counts = new[] { 1, 3 };
+
+        foreach (var toolchain in toolchains)
+        {
+            foreach (var count in counts)
+            {
+                foreach (var runModes in GetRunModeCombinations(count))
+                {
+                    yield return [runModes, toolchain];
+                }
+            }
+        }
+    }
+
+    private static BaseMockInProcessDiagnoser CreateDiagnoser(RunMode runMode)
+        => runMode switch
+        {
+            RunMode.None => new MockInProcessDiagnoserNone(),
+            RunMode.NoOverhead => new MockInProcessDiagnoser(),
+            RunMode.ExtraRun => new MockInProcessDiagnoserExtraRun(),
+            RunMode.SeparateLogic => new MockInProcessDiagnoserSeparateLogic(),
+            _ => throw new ArgumentException($"Unsupported run mode: {runMode}")
+        };
+
+    private ManualConfig CreateConfig(ToolchainType toolchain)
+    {
+        var job = toolchain switch
+        {
+            ToolchainType.InProcessEmit => Job.Dry.WithToolchain(InProcessEmitToolchain.Instance),
+            ToolchainType.InProcessNoEmit => Job.Dry.WithToolchain(InProcessNoEmitToolchain.Instance),
+            _ => Job.Dry
+        };
+
+        return new ManualConfig()
+            .AddLogger(new OutputLogger(Output))
+            .AddColumnProvider(DefaultColumnProviders.Instance)
+            .AddJob(job);
+    }
+
+    [Theory]
+    [MemberData(nameof(GetTestCombinations))]
+    public void MultipleInProcessDiagnosersWork(RunMode[] runModes, ToolchainType toolchain)
+    {
+        var diagnosers = runModes.Select(CreateDiagnoser).ToArray();
+        var config = CreateConfig(toolchain);
+
+        foreach (var diagnoser in diagnosers)
+        {
+            config = config.AddDiagnoser(diagnoser);
+        }
+
+        var summary = CanExecute<SimpleBenchmark>(config);
+
+        foreach (var diagnoser in diagnosers)
+        {
+            if (diagnoser.DiagnoserRunMode == RunMode.None)
+            {
+                Assert.Empty(diagnoser.Results);
+            }
+            else
+            {
+                Assert.NotEmpty(diagnoser.Results);
+                Assert.Equal(summary.BenchmarksCases.Length, diagnoser.Results.Count);
+                Assert.All(diagnoser.Results.Values, result => Assert.Equal(diagnoser.ExpectedResult, result));
+            }
+        }
+        Assert.Equal(
+            BaseMockInProcessDiagnoser.s_completedResults,
+            diagnosers
+                .Where(d => d.DiagnoserRunMode != RunMode.None)
+                .OrderBy(d => d.DiagnoserRunMode switch
+                {
+                    RunMode.NoOverhead => 0,
+                    RunMode.ExtraRun => 1,
+                    RunMode.SeparateLogic => 2,
+                    _ => 3
+                })
+        );
+        BaseMockInProcessDiagnoser.s_completedResults.Clear();
+    }
+
+    public class SimpleBenchmark
+    {
+        private int counter;
+
+        [Benchmark]
+        public void BenchmarkMethod()
+        {
+            Interlocked.Increment(ref counter);
+        }
+    }
+}

tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs

@@ -131,19 +131,6 @@ public void InProcessBenchmarkEmitsSameIL(Type benchmarkType)
             Assert.DoesNotContain("No benchmarks found", logger.GetLog());
         }
 
-        [Fact]
-        public void InProcessEmitSupportsInProcessDiagnosers()
-        {
-            var logger = new OutputLogger(Output);
-            var diagnoser = new MockInProcessDiagnoser();
-            var config = CreateInProcessConfig(logger).AddDiagnoser(diagnoser);
-
-            var summary = CanExecute<BenchmarkAllCases>(config);
-
-            var expected = Enumerable.Repeat("MockResult", summary.BenchmarksCases.Length);
-            Assert.Equal(expected, diagnoser.Results.Values);
-        }
-
         [UsedImplicitly(ImplicitUseTargetFlags.WithMembers)]
         public class BenchmarkAllCases
         {

tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs

@@ -218,19 +218,6 @@ public void InProcessBenchmarkAllCasesSupported()
             }
         }
 
-        [Fact]
-        public void InProcessNoEmitSupportsInProcessDiagnosers()
-        {
-            var logger = new OutputLogger(Output);
-            var diagnoser = new MockInProcessDiagnoser();
-            var config = CreateInProcessConfig(logger).AddDiagnoser(diagnoser);
-
-            var summary = CanExecute<BenchmarkAllCases>(config);
-
-            var expected = Enumerable.Repeat("MockResult", summary.BenchmarksCases.Length);
-            Assert.Equal(expected, diagnoser.Results.Values);
-        }
-
         [UsedImplicitly(ImplicitUseTargetFlags.WithMembers)]
         public class BenchmarkAllCases
         {

tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs

@@ -79,7 +79,9 @@ public void NativeAotSupportsInProcessDiagnosers()
                 throw;
             }
 
-            Assert.Equal(["MockResult"], diagnoser.Results.Values);
+            Assert.Equal([diagnoser.ExpectedResult], diagnoser.Results.Values);
+            Assert.Equal([diagnoser], BaseMockInProcessDiagnoser.s_completedResults);
+            BaseMockInProcessDiagnoser.s_completedResults.Clear();
         }
 
         private static bool GetShouldRunTest()

tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs

@@ -56,7 +56,9 @@ public void WasmSupportsInProcessDiagnosers()
 
             CanExecute<WasmBenchmark>(config);
 
-            Assert.Equal(["MockResult"], diagnoser.Results.Values);
+            Assert.Equal([diagnoser.ExpectedResult], diagnoser.Results.Values);
+            Assert.Equal([diagnoser], BaseMockInProcessDiagnoser.s_completedResults);
+            BaseMockInProcessDiagnoser.s_completedResults.Clear();
         }
 
         public class WasmBenchmark