Fix failure to capture test failures

Author: sharwell

test/Microsoft.ML.TestFramework/BaseTestBaseline.cs

@@ -5,6 +5,7 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Runtime.ExceptionServices;
 using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading;
@@ -78,7 +79,7 @@ protected BaseTestBaseline(ITestOutputHelper output) : base(output)
         protected IHostEnvironment Env => _env;
         protected MLContext ML;
         private bool _normal;
-        private bool _passed;
+        private readonly List<Exception> _failures = new List<Exception>();
 
         protected override void Initialize()
         {
@@ -96,7 +97,6 @@ protected override void Initialize()
 
             string logPath = Path.Combine(logDir, FullTestName + LogSuffix);
             LogWriter = OpenWriter(logPath);
-            _passed = true;
             _env = new ConsoleEnvironment(42, outWriter: LogWriter, errWriter: LogWriter)
                 .AddStandardComponents();
             ML = new MLContext(42);
@@ -113,7 +113,7 @@ protected override void Cleanup()
             Contracts.Assert(IsActive);
             Log("Test {0}: {1}: {2}", TestName,
                 _normal ? "completed normally" : "aborted",
-                _passed ? "passed" : "failed");
+                IsPassing ? "passed" : "failed");
 
             Contracts.AssertValue(LogWriter);
             LogWriter.Dispose();
@@ -124,7 +124,7 @@ protected override void Cleanup()
 
         protected bool IsActive { get { return LogWriter != null; } }
 
-        protected bool IsPassing { get { return _passed; } }
+        protected bool IsPassing { get { return _failures.Count == 0; } }
 
         // Called by a test to signal normal completion. If this is not called before the
         // TestScope is disposed, we assume the test was aborted.
@@ -134,7 +134,18 @@ protected void Done()
             Contracts.Assert(!_normal, "Done() should only be called once!");
             _normal = true;
 
-            Assert.True(_passed);
+            switch (_failures.Count)
+            {
+                case 0:
+                    break;
+
+                case 1:
+                    ExceptionDispatchInfo.Capture(_failures[0]).Throw();
+                    break;
+
+                default:
+                    throw new AggregateException(_failures.ToArray());
+            }
         }
 
         protected bool Check(bool f, string msg)
@@ -164,8 +175,14 @@ protected void Fail(string fmt, params object[] args)
         protected void Fail(bool relax, string fmt, params object[] args)
         {
             Contracts.Assert(IsActive);
-            if (!relax)
-                _passed = false;
+            try
+            {
+                throw new InvalidOperationException(string.Format(fmt, args));
+            }
+            catch (Exception ex) when (relax)
+            {
+                _failures.Add(ex);
+            }
 
             Log("*** Failure: " + fmt, args);
         }
@@ -210,12 +227,6 @@ protected string GetBaselinePath(string subDir, string name)
             return Path.GetFullPath(Path.Combine(_baselineBuildStringDir, subDir, name));
         }
 
-        // Inverts the _passed flag. Do not ever use this except in rare conditions. Eg. Recording failure of a test as a success.
-        protected void DoNotEverUseInvertPass()
-        {
-            _passed = !_passed;
-        }
-
         // These are used to normalize output.
         private static readonly Regex _matchDataRoot = new Regex(DataRootRegExp, RegexOptions.IgnoreCase | RegexOptions.Compiled);
         private static readonly Regex _matchDataUnixRoot = new Regex(DataRootUnixRegExp, RegexOptions.IgnoreCase | RegexOptions.Compiled);