Dotnet test integration (first part)

Directory.Packages.props

@@ -63,8 +63,6 @@
     <PackageVersion Include="Microsoft.TemplateEngine.Orchestrator.RunnableProjects" Version="$(MicrosoftTemplateEngineOrchestratorRunnableProjectsPackageVersion)" />
     <PackageVersion Include="Microsoft.TemplateEngine.TestHelper" Version="$(MicrosoftTemplateEngineTestHelperPackageVersion)" />
     <PackageVersion Include="Microsoft.TemplateSearch.Common" Version="$(MicrosoftTemplateSearchCommonPackageVersion)" />
-    <PackageVersion Include="Microsoft.Testing.Platform" Version="$(MicrosoftTestingPlatformVersion)" />
-    <PackageVersion Include="Microsoft.Testing.Extensions.TrxReport" Version="$(MicrosoftTestingPlatformVersion)" />
     <PackageVersion Include="Microsoft.TestPlatform.Build" Version="$(MicrosoftTestPlatformBuildPackageVersion)" />
     <PackageVersion Include="Microsoft.TestPlatform.CLI" Version="$(MicrosoftTestPlatformCLIPackageVersion)" />
     <PackageVersion Include="Microsoft.VisualStudio.Composition" Version="17.4.16" />

eng/Version.Details.xml

@@ -634,11 +634,7 @@
       <Uri>https://github.com/dotnet/runtime</Uri>
       <Sha>e77011b31a3e5c47d931248a64b47f9b2d47853d</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.Testing.Platform" Version="1.6.0-preview.25059.14">
-      <Uri>https://github.com/microsoft/testfx</Uri>
-      <Sha>53488cf463d33882113ca7a6fbbb6f93e06251df</Sha>
-    </Dependency>
-    <Dependency Name="MSTest" Version="3.8.0-preview.25059.14">
+    <Dependency Name="MSTest" Version="3.8.0-preview.25057.8">
       <Uri>https://github.com/microsoft/testfx</Uri>
       <Sha>53488cf463d33882113ca7a6fbbb6f93e06251df</Sha>
     </Dependency>

eng/Versions.props

@@ -28,7 +28,6 @@
     <UsingToolMicrosoftNetCompilers Condition="'$(DotNetBuildSourceOnly)' == 'true'">true</UsingToolMicrosoftNetCompilers>
     <MicrosoftIORedistPackageVersion>6.0.1</MicrosoftIORedistPackageVersion>
     <FlagNetStandard1XDependencies Condition="'$(DotNetBuildSourceOnly)' == 'true'">true</FlagNetStandard1XDependencies>
-    <MicrosoftTestingPlatformVersion>1.6.0-preview.25059.14</MicrosoftTestingPlatformVersion>
   </PropertyGroup>
   <PropertyGroup Label="Servicing version information">
     <VersionFeature21>30</VersionFeature21>

src/Cli/dotnet/commands/dotnet-test/IPC/Models/TestResultMessages.cs

@@ -5,7 +5,9 @@ namespace Microsoft.DotNet.Tools.Test
 {
     internal sealed record SuccessfulTestResultMessage(string? Uid, string? DisplayName, byte? State, long? Duration, string? Reason, string? StandardOutput, string? ErrorOutput, string? SessionUid);
 
-    internal sealed record FailedTestResultMessage(string? Uid, string? DisplayName, byte? State, long? Duration, string? Reason, string? ErrorMessage, string? ErrorStackTrace, string? StandardOutput, string? ErrorOutput, string? SessionUid);
+    internal sealed record FailedTestResultMessage(string? Uid, string? DisplayName, byte? State, long? Duration, string? Reason, ExceptionMessage[]? Exceptions, string? StandardOutput, string? ErrorOutput, string? SessionUid);
+
+    internal sealed record ExceptionMessage(string? ErrorMessage, string? ErrorType, string? StackTrace);
 
     internal sealed record TestResultMessages(string? ExecutionId, SuccessfulTestResultMessage[] SuccessfulTestMessages, FailedTestResultMessage[] FailedTestMessages) : IRequest;
 }

src/Cli/dotnet/commands/dotnet-test/IPC/ObjectFieldIds.cs

@@ -85,13 +85,19 @@ internal static class FailedTestResultMessageFieldsId
         public const ushort State = 3;
         public const ushort Duration = 4;
         public const ushort Reason = 5;
-        public const ushort ErrorMessage = 6;
-        public const ushort ErrorStackTrace = 7;
+        public const ushort ExceptionMessageList = 6;
         public const ushort StandardOutput = 8;
         public const ushort ErrorOutput = 9;
         public const ushort SessionUid = 10;
     }
 
+    internal static class ExceptionMessageFieldsId
+    {
+        public const ushort ErrorMessage = 1;
+        public const ushort ErrorType = 2;
+        public const ushort StackTrace = 3;
+    }
+
     internal static class FileArtifactMessagesFieldsId
     {
         public const int MessagesSerializerId = 7;

src/Cli/dotnet/commands/dotnet-test/IPC/Serializers/TestResultMessagesSerializer.cs

@@ -217,8 +217,8 @@ private static List<FailedTestResultMessage> ReadFailedTestMessagesPayload(Strea
             int length = ReadInt(stream);
             for (int i = 0; i < length; i++)
             {
-                string? uid = null, displayName = null, reason = null, sessionUid = null,
-                    errorMessage = null, errorStackTrace = null, standardOutput = null, errorOutput = null;
+                string? uid = null, displayName = null, reason = null, sessionUid = null, standardOutput = null, errorOutput = null;
+                List<ExceptionMessage> exceptionMessages = [];
                 byte? state = null;
                 long? duration = null;
 
@@ -251,13 +251,44 @@ private static List<FailedTestResultMessage> ReadFailedTestMessagesPayload(Strea
                             reason = ReadStringValue(stream, fieldSize);
                             break;
 
-                        case FailedTestResultMessageFieldsId.ErrorMessage:
-                            errorMessage = ReadStringValue(stream, fieldSize);
-                            break;
+                        case FailedTestResultMessageFieldsId.ExceptionMessageList:
+                            {
+                                int length2 = ReadInt(stream);
+                                for (int k = 0; k < length2; k++)
+                                {
 
-                        case FailedTestResultMessageFieldsId.ErrorStackTrace:
-                            errorStackTrace = ReadStringValue(stream, fieldSize);
-                            break;
+                                    int fieldCount2 = ReadShort(stream);
+
+                                    string? errorMessage = null;
+                                    string? errorType = null;
+                                    string? stackTrace = null;
+
+                                    for (int l = 0; l < fieldCount2; l++)
+                                    {
+                                        int fieldId2 = ReadShort(stream);
+                                        int fieldSize2 = ReadInt(stream);
+
+                                        switch (fieldId2)
+                                        {
+                                            case ExceptionMessageFieldsId.ErrorMessage:
+                                                errorMessage = ReadStringValue(stream, fieldSize2);
+                                                break;
+
+                                            case ExceptionMessageFieldsId.ErrorType:
+                                                errorType = ReadStringValue(stream, fieldSize2);
+                                                break;
+
+                                            case ExceptionMessageFieldsId.StackTrace:
+                                                stackTrace = ReadStringValue(stream, fieldSize2);
+                                                break;
+                                        }
+                                    }
+
+                                    exceptionMessages.Add(new ExceptionMessage(errorMessage, errorType, stackTrace));
+                                }
+
+                                break;
+                            }
 
                         case FailedTestResultMessageFieldsId.StandardOutput:
                             standardOutput = ReadStringValue(stream, fieldSize);
@@ -277,7 +308,7 @@ private static List<FailedTestResultMessage> ReadFailedTestMessagesPayload(Strea
                     }
                 }
 
-                failedTestResultMessages.Add(new FailedTestResultMessage(uid, displayName, state, duration, reason, errorMessage, errorStackTrace, standardOutput, errorOutput, sessionUid));
+                failedTestResultMessages.Add(new FailedTestResultMessage(uid, displayName, state, duration, reason, exceptionMessages.ToArray(), standardOutput, errorOutput, sessionUid));
             }
 
             return failedTestResultMessages;
@@ -354,8 +385,7 @@ private static void WriteFailedTestMessagesPayload(Stream stream, FailedTestResu
                 WriteField(stream, FailedTestResultMessageFieldsId.State, failedTestResultMessage.State);
                 WriteField(stream, FailedTestResultMessageFieldsId.Duration, failedTestResultMessage.Duration);
                 WriteField(stream, FailedTestResultMessageFieldsId.Reason, failedTestResultMessage.Reason);
-                WriteField(stream, FailedTestResultMessageFieldsId.ErrorMessage, failedTestResultMessage.ErrorMessage);
-                WriteField(stream, FailedTestResultMessageFieldsId.ErrorStackTrace, failedTestResultMessage.ErrorStackTrace);
+                WriteExceptionMessagesPayload(stream, failedTestResultMessage.Exceptions);
                 WriteField(stream, FailedTestResultMessageFieldsId.StandardOutput, failedTestResultMessage.StandardOutput);
                 WriteField(stream, FailedTestResultMessageFieldsId.ErrorOutput, failedTestResultMessage.ErrorOutput);
                 WriteField(stream, FailedTestResultMessageFieldsId.SessionUid, failedTestResultMessage.SessionUid);
@@ -366,6 +396,35 @@ private static void WriteFailedTestMessagesPayload(Stream stream, FailedTestResu
             WriteAtPosition(stream, (int)(stream.Position - before), before - sizeof(int));
         }
 
+        private static void WriteExceptionMessagesPayload(Stream stream, ExceptionMessage[]? exceptionMessages)
+        {
+            if (exceptionMessages is null || exceptionMessages.Length == 0)
+            {
+                return;
+            }
+
+            WriteShort(stream, FailedTestResultMessageFieldsId.ExceptionMessageList);
+
+            // We will reserve an int (4 bytes)
+            // so that we fill the size later, once we write the payload
+            WriteInt(stream, 0);
+
+            long before = stream.Position;
+            WriteInt(stream, exceptionMessages.Length);
+            foreach (ExceptionMessage exceptionMessage in exceptionMessages)
+            {
+                WriteShort(stream, GetFieldCount(exceptionMessage));
+
+                WriteField(stream, ExceptionMessageFieldsId.ErrorMessage, exceptionMessage.ErrorMessage);
+                WriteField(stream, ExceptionMessageFieldsId.ErrorType, exceptionMessage.ErrorType);
+                WriteField(stream, ExceptionMessageFieldsId.StackTrace, exceptionMessage.StackTrace);
+            }
+
+            // NOTE: We are able to seek only if we are using a MemoryStream
+            // thus, the seek operation is fast as we are only changing the value of a property
+            WriteAtPosition(stream, (int)(stream.Position - before), before - sizeof(int));
+        }
+
         private static ushort GetFieldCount(TestResultMessages testResultMessages) =>
             (ushort)((testResultMessages.ExecutionId is null ? 0 : 1) +
             (IsNullOrEmpty(testResultMessages.SuccessfulTestMessages) ? 0 : 1) +
@@ -387,10 +446,14 @@ private static ushort GetFieldCount(FailedTestResultMessage failedTestResultMess
             (failedTestResultMessage.State is null ? 0 : 1) +
             (failedTestResultMessage.Duration is null ? 0 : 1) +
             (failedTestResultMessage.Reason is null ? 0 : 1) +
-            (failedTestResultMessage.ErrorMessage is null ? 0 : 1) +
-            (failedTestResultMessage.ErrorStackTrace is null ? 0 : 1) +
+            (IsNullOrEmpty(failedTestResultMessage.Exceptions) ? 0 : 1) +
             (failedTestResultMessage.StandardOutput is null ? 0 : 1) +
             (failedTestResultMessage.ErrorOutput is null ? 0 : 1) +
             (failedTestResultMessage.SessionUid is null ? 0 : 1));
+
+        private static ushort GetFieldCount(ExceptionMessage exceptionMessage) =>
+            (ushort)((exceptionMessage.ErrorMessage is null ? 0 : 1) +
+            (exceptionMessage.ErrorType is null ? 0 : 1) +
+            (exceptionMessage.StackTrace is null ? 0 : 1));
     }
 }

src/Cli/dotnet/commands/dotnet-test/LocalizableStrings.resx

@@ -304,10 +304,10 @@ Examples:
     <value>NAME="VALUE"</value>
   </data>
   <data name="NoSerializerRegisteredWithIdErrorMessage" xml:space="preserve">
-      <value>No serializer registered with ID '{0}'</value>
+    <value>No serializer registered with ID '{0}'</value>
   </data>
   <data name="NoSerializerRegisteredWithTypeErrorMessage" xml:space="preserve">
-      <value>No serializer registered with type '{0}'</value>
+    <value>No serializer registered with type '{0}'</value>
   </data>
   <data name="CmdMaxParallelTestModulesDescription" xml:space="preserve">
     <value>The max number of test modules that can run in parallel.</value>
@@ -332,6 +332,114 @@ Examples:
   <data name="CmdUnsupportedVSTestTestApplicationsDescription" xml:space="preserve">
     <value>Test application(s) that support VSTest are not supported.</value>
   </data>
+  <data name="Aborted" xml:space="preserve">
+    <value>Aborted</value>
+  </data>
+  <data name="ActiveTestsRunning_FullTestsCount" xml:space="preserve">
+    <value>{0} tests running</value>
+  </data>
+  <data name="ActiveTestsRunning_MoreTestsCount" xml:space="preserve">
+    <value>and {0} more</value>
+  </data>
+  <data name="Actual" xml:space="preserve">
+    <value>Actual</value>
+  </data>
+  <data name="CancelledLowercase" xml:space="preserve">
+    <value>canceled</value>
+  </data>
+  <data name="CancellingTestSession" xml:space="preserve">
+    <value>Canceling the test session...</value>
+  </data>
+  <data name="ConsoleIsAlreadyInBatchingMode" xml:space="preserve">
+    <value>Console is already in batching mode.</value>
+    <comment>Exception that is thrown when console is already collecting input into a batch (into a string builder), and code asks to enable batching mode again.</comment>
+  </data>
+  <data name="DiscoveredTestsInAssembly" xml:space="preserve">
+    <value>Discovered {0} tests in assembly</value>
+    <comment>0 is count, the sentence is followed by the path of the assebly</comment>
+  </data>
+  <data name="DiscoveringTestsFrom" xml:space="preserve">
+    <value>Discovering tests from</value>
+  </data>
+  <data name="ExitCode" xml:space="preserve">
+    <value>Exit code</value>
+  </data>
+  <data name="Expected" xml:space="preserve">
+    <value>Expected</value>
+  </data>
+  <data name="Failed" xml:space="preserve">
+    <value>Failed</value>
+  </data>
+  <data name="FailedLowercase" xml:space="preserve">
+    <value>failed</value>
+  </data>
+  <data name="FailedWithErrors" xml:space="preserve">
+    <value>failed with {0} error(s)</value>
+  </data>
+  <data name="FailedWithErrorsAndWarnings" xml:space="preserve">
+    <value>failed with {0} error(s) and {1} warning(s)</value>
+  </data>
+  <data name="FailedWithWarnings" xml:space="preserve">
+    <value>failed with {0} warning(s)</value>
+  </data>
+  <data name="ForTest" xml:space="preserve">
+    <value>For test</value>
+    <comment>is followed by test name</comment>
+  </data>
+  <data name="FromFile" xml:space="preserve">
+    <value>from</value>
+    <comment>from followed by a file name to point to the file from which test is originating</comment>
+  </data>
+  <data name="InProcessArtifactsProduced" xml:space="preserve">
+    <value>In process file artifacts produced:</value>
+  </data>
+  <data name="MinimumExpectedTestsPolicyViolation" xml:space="preserve">
+    <value>Minimum expected tests policy violation, tests ran {0}, minimum expected {1}</value>
+    <comment>{0}, {1} number of tests</comment>
+  </data>
+  <data name="OutOfProcessArtifactsProduced" xml:space="preserve">
+    <value>Out of process file artifacts produced:</value>
+  </data>
+  <data name="Passed" xml:space="preserve">
+    <value>Passed</value>
+  </data>
+  <data name="PassedLowercase" xml:space="preserve">
+    <value>passed</value>
+  </data>
+  <data name="RunningTestsFrom" xml:space="preserve">
+    <value>Running tests from</value>
+  </data>
+  <data name="SkippedLowercase" xml:space="preserve">
+    <value>skipped</value>
+  </data>
+  <data name="StackFrameAt" xml:space="preserve">
+    <value>at</value>
+    <comment>at that is used for a stack frame location in a stack trace, is followed by a class and method name</comment>
+  </data>
+  <data name="StackFrameIn" xml:space="preserve">
+    <value>in</value>
+    <comment>in that is used in stack frame it is followed by file name</comment>
+  </data>
+  <data name="StandardError" xml:space="preserve">
+    <value>Error output</value>
+  </data>
+  <data name="StandardOutput" xml:space="preserve">
+    <value>Standard output</value>
+  </data>
+  <data name="TestDiscoverySummary" xml:space="preserve">
+    <value>Discovered {0} tests in {1} assemblies.</value>
+    <comment>0 is number of tests, 1 is count of assemblies</comment>
+  </data>
+  <data name="TestDiscoverySummarySingular" xml:space="preserve">
+    <value>Discovered {0} tests.</value>
+    <comment>0 is number of tests</comment>
+  </data>
+  <data name="TestRunSummary" xml:space="preserve">
+    <value>Test run summary:</value>
+  </data>
+  <data name="ZeroTestsRan" xml:space="preserve">
+    <value>Zero tests ran</value>
+  </data>
   <data name="CmdUnsupportedTestRunnerDescription" xml:space="preserve">
     <value>Test runner not supported: {0}.</value>
   </data>

src/Cli/dotnet/commands/dotnet-test/Models.cs

@@ -13,7 +13,9 @@ internal sealed record DiscoveredTest(string? Uid, string? DisplayName);
 
     internal sealed record SuccessfulTestResult(string? Uid, string? DisplayName, byte? State, long? Duration, string? Reason, string? StandardOutput, string? ErrorOutput, string? SessionUid);
 
-    internal sealed record FailedTestResult(string? Uid, string? DisplayName, byte? State, long? Duration, string? Reason, string? ErrorMessage, string? ErrorStackTrace, string? StandardOutput, string? ErrorOutput, string? SessionUid);
+    internal sealed record FailedTestResult(string? Uid, string? DisplayName, byte? State, long? Duration, string? Reason, FlatException[]? Exceptions, string? StandardOutput, string? ErrorOutput, string? SessionUid);
+
+    internal sealed record FlatException(string? ErrorMessage, string? ErrorType, string? StackTrace);
 
     internal sealed record FileArtifact(string? FullPath, string? DisplayName, string? Description, string? TestUid, string? TestDisplayName, string? SessionUid);
 

src/Cli/dotnet/commands/dotnet-test/Options.cs

@@ -3,7 +3,7 @@
 
 namespace Microsoft.DotNet.Cli
 {
-    internal record BuildConfigurationOptions(bool HasNoRestore, bool HasNoBuild, string Configuration, string Architecture);
+    internal record BuildConfigurationOptions(bool HasNoRestore, bool HasNoBuild, bool HasListTests, string Configuration, string Architecture);
 
     internal record BuildPathsOptions(string ProjectPath, string SolutionPath, string DirectoryPath);
 }