Merge branch 'frank-dong-ms-retry-fail-tests'

Author: frank-dong-ms

test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs

@@ -19,6 +19,7 @@
 using Microsoft.ML.Runtime;
 using Microsoft.ML.TestFramework.Attributes;
 using Microsoft.ML.TestFrameworkCommon;
+using Microsoft.ML.TestFrameworkCommon.Attributes;
 using Microsoft.ML.Trainers;
 using Microsoft.ML.Trainers.Ensemble;
 using Microsoft.ML.Trainers.FastTree;
@@ -2758,7 +2759,7 @@ public void EntryPointSDCAMulticlass()
             TestEntryPointRoutine("iris.txt", "Trainers.StochasticDualCoordinateAscentClassifier");
         }
 
-        [Fact()]
+        [RetryFact]
         public void EntryPointSDCARegression()
         {
             TestEntryPointRoutine(TestDatasets.generatedRegressionDatasetmacro.trainFilename, "Trainers.StochasticDualCoordinateAscentRegressor", loader: TestDatasets.generatedRegressionDatasetmacro.loaderSettings);
@@ -3845,7 +3846,7 @@ public void EntryPointChainedTrainTestMacros()
             validateAuc(metrics);
         }
 
-        [Fact]
+        [RetryFact]
         public void EntryPointChainedCrossValMacros()
         {
             string inputGraph = @"
@@ -6025,7 +6026,7 @@ public void TestCrossValidationMacroWithNonDefaultNames()
             }
         }
 
-        [Fact]
+        [RetryFact]
         public void TestOvaMacro()
         {
             var dataPath = GetDataPath(@"iris.txt");
@@ -6189,7 +6190,7 @@ public void TestOvaMacro()
             }
         }
 
-        [Fact]
+        [RetryFact]
         public void TestOvaMacroWithUncalibratedLearner()
         {
             var dataPath = GetDataPath(@"iris.txt");

test/Microsoft.ML.Predictor.Tests/TestPredictors.cs

@@ -24,6 +24,7 @@ namespace Microsoft.ML.RunTests
     using Xunit.Abstractions;
     using TestLearners = TestLearnersBase;
     using Microsoft.ML.TestFrameworkCommon;
+    using Microsoft.ML.TestFrameworkCommon.Attributes;
 
     /// <summary>
     /// Tests using maml commands (IDV) functionality.
@@ -181,7 +182,7 @@ public void MulticlassSdcaTest()
         /// <summary>
         /// Multiclass Logistic Regression test with a tree featurizer.
         /// </summary>
-        [X64Fact("x86 output differs from Baseline")]
+        [RetryX64Fact("x86 output differs from Baseline")]
         [TestCategory("Multiclass")]
         [TestCategory("Logistic Regression")]
         [TestCategory("FastTree")]
@@ -270,7 +271,7 @@ public void BinaryClassifierLogisticRegressionTest()
             Done();
         }
 
-        [X64Fact("x86 output differs from Baseline")]
+        [RetryX64Fact("x86 output differs from Baseline")]
         [TestCategory("Binary")]
         public void BinaryClassifierSymSgdTest()
         {
@@ -321,7 +322,7 @@ public void BinaryClassifierLogisticRegressionNonNegativeTest()
         /// <summary>
         ///A test for binary classifiers
         ///</summary>
-        [LessThanNetCore30OrNotNetCoreFact("netcoreapp3.0 output differs from Baseline")]
+        [RetryLessThanNetCore30OrNotNetCoreFact("netcoreapp3.0 output differs from Baseline")]
         [TestCategory("Binary")]
         public void BinaryClassifierLogisticRegressionBinNormTest()
         {

test/Microsoft.ML.TestFrameworkCommon/Attributes/RetryFactAttribute.cs

@@ -0,0 +1,85 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using Xunit;
+using Xunit.Sdk;
+
+namespace Microsoft.ML.TestFrameworkCommon.Attributes
+{
+    /// <summary>
+    /// ML.NET facts that will retry several flaky test cases, use default timeout settings
+    /// </summary>
+    [XunitTestCaseDiscoverer("Microsoft.ML.TestFrameworkCommon.RetryFactDiscoverer", "Microsoft.ML.TestFrameworkCommon")]
+    public class RetryFactAttribute : FactAttribute
+    {
+        /// <summary>
+        /// Number of retries allowed for a failed test. If unset (or set less than 1), will
+        /// default to 2 attempts.
+        /// </summary>
+        public int MaxRetries { get; set; }
+    }
+
+
+    /// <summary>
+    /// ML.NET facts that will retry several flaky test cases, use default timeout settings
+    /// </summary>
+    [XunitTestCaseDiscoverer("Microsoft.ML.TestFrameworkCommon.RetryFactDiscoverer", "Microsoft.ML.TestFrameworkCommon")]
+    public class RetryLessThanNetCore30OrNotNetCoreFactAttribute : EnvironmentSpecificFactAttribute
+    {
+        public RetryLessThanNetCore30OrNotNetCoreFactAttribute(string skipMessage) : base(skipMessage)
+        {
+        }
+
+        /// <inheritdoc />
+        protected override bool IsEnvironmentSupported()
+        {
+            return AppDomain.CurrentDomain.GetData("FX_PRODUCT_VERSION") == null;
+        }
+        /// <summary>
+        /// Number of retries allowed for a failed test. If unset (or set less than 1), will
+        /// default to 2 attempts.
+        /// </summary>
+        public int MaxRetries { get; set; }
+    }
+
+    /// <summary>
+    /// A fact for tests requiring X64 environment.
+    /// </summary>
+    [XunitTestCaseDiscoverer("Microsoft.ML.TestFrameworkCommon.RetryFactDiscoverer", "Microsoft.ML.TestFrameworkCommon")]
+    public sealed class RetryX64FactAttribute : EnvironmentSpecificFactAttribute
+    {
+        public RetryX64FactAttribute(string skipMessage) : base(skipMessage)
+        {
+        }
+
+        /// <inheritdoc />
+        protected override bool IsEnvironmentSupported()
+        {
+            return Environment.Is64BitProcess;
+        }
+
+        public int MaxRetries { get; set; }
+    }
+
+    /// <summary>
+    /// A fact for tests requiring TensorFlow.
+    /// </summary>
+    [XunitTestCaseDiscoverer("Microsoft.ML.TestFrameworkCommon.RetryFactDiscoverer", "Microsoft.ML.TestFrameworkCommon")]
+    public sealed class RetryTensorFlowFactAttribute : EnvironmentSpecificFactAttribute
+    {
+        public RetryTensorFlowFactAttribute() : base("TensorFlow is 64-bit only")
+        {
+        }
+
+        /// <inheritdoc />
+        protected override bool IsEnvironmentSupported()
+        {
+            return Environment.Is64BitProcess;
+        }
+
+        public int MaxRetries { get; set; }
+    }
+}
+

test/Microsoft.ML.TestFrameworkCommon/DelayedMessageBus.cs

@@ -0,0 +1,42 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+using System.Reflection;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace Microsoft.ML.TestFrameworkCommon
+{
+    /// <summary>
+    /// Used to capture messages to potentially be forwarded later. Messages are forwarded by
+    /// disposing of the message bus.
+    /// </summary>
+    public class DelayedMessageBus : IMessageBus
+    {
+        private readonly IMessageBus innerBus;
+        public readonly List<IMessageSinkMessage> messages = new List<IMessageSinkMessage>();
+
+        public DelayedMessageBus(IMessageBus innerBus)
+        {
+            this.innerBus = innerBus;
+        }
+
+        public bool QueueMessage(IMessageSinkMessage message)
+        {
+            lock (messages)
+                messages.Add(message);
+
+            // No way to ask the inner bus if they want to cancel without sending them the message, so
+            // we just go ahead and continue always.
+            return true;
+        }
+
+        public void Dispose()
+        {
+            foreach (var message in messages)
+                innerBus.QueueMessage(message);
+        }
+    }
+}

test/Microsoft.ML.TestFrameworkCommon/RetryFactDiscoverer.cs

@@ -0,0 +1,32 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace Microsoft.ML.TestFrameworkCommon
+{
+    public class RetryFactDiscoverer : IXunitTestCaseDiscoverer
+    {
+        readonly IMessageSink diagnosticMessageSink;
+
+        public RetryFactDiscoverer(IMessageSink diagnosticMessageSink)
+        {
+            this.diagnosticMessageSink = diagnosticMessageSink;
+        }
+
+        public IEnumerable<IXunitTestCase> Discover(ITestFrameworkDiscoveryOptions discoveryOptions, 
+            ITestMethod testMethod, IAttributeInfo factAttribute)
+        {
+            //by default, retry failed tests at max 2 times
+            var maxRetries = factAttribute.GetNamedArgument<int>("MaxRetries");
+            if (maxRetries < 1)
+                maxRetries = 2;
+
+            yield return new RetryTestCase(diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, maxRetries);
+        }
+    }
+}
+

test/Microsoft.ML.TestFrameworkCommon/RetryTestCase.cs

@@ -0,0 +1,120 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.ComponentModel;
+using System.Threading;
+using System.Threading.Tasks;
+using Xunit.Abstractions;
+using Xunit.Sdk;
+
+namespace Microsoft.ML.TestFrameworkCommon
+{
+    [Serializable]
+    public class RetryTestCase : XunitTestCase
+    {
+        private int maxRetries;
+
+        [EditorBrowsable(EditorBrowsableState.Never)]
+        [Obsolete("Called by the de-serializer", true)]
+        public RetryTestCase() { }
+
+        public RetryTestCase(IMessageSink diagnosticMessageSink, TestMethodDisplay testMethodDisplay, 
+            ITestMethod testMethod, int maxRetries)
+            : base(diagnosticMessageSink, testMethodDisplay, TestMethodDisplayOptions.None, testMethod, testMethodArguments: null)
+        {
+            this.maxRetries = maxRetries;
+        }
+
+        // This method is called by the xUnit test framework classes to run the test case. We will do the
+        // loop here, forwarding on to the implementation in XunitTestCase to do the heavy lifting. We will
+        // continue to re-run the test until the aggregator has an error (meaning that some internal error
+        // condition happened), or the test runs without failure, or we've hit the maximum number of tries.
+        public override async Task<RunSummary> RunAsync(IMessageSink diagnosticMessageSink,
+                                                        IMessageBus messageBus,
+                                                        object[] constructorArguments,
+                                                        ExceptionAggregator aggregator,
+                                                        CancellationTokenSource cancellationTokenSource)
+        {
+            var runCount = 0;
+
+            while (true)
+            {
+                // This is really the only tricky bit: we need to capture and delay messages (since those will
+                // contain run status) until we know we've decided to accept the final result;
+                var delayedMessageBus = new DelayedMessageBus(messageBus);
+
+                RunSummary summary = await base.RunAsync(diagnosticMessageSink, delayedMessageBus, constructorArguments, aggregator, cancellationTokenSource);
+                if (aggregator.HasExceptions || summary.Failed > 0)
+                {
+                    var details = ExtractTestFailDetailsFromMessageBus(delayedMessageBus);
+                    var errorMessage = $"Execution of '{DisplayName}' failed (attempt #{runCount + 1}) with details {details}.";
+
+                    diagnosticMessageSink.OnMessage(new DiagnosticMessage(errorMessage));
+                    Console.WriteLine(errorMessage);
+                }
+
+                if (summary.Failed == 0 || ++runCount >= maxRetries)
+                {
+                    delayedMessageBus.Dispose();  // Sends all the delayed messages
+                    return summary;
+                }
+            }
+        }
+
+        private static string ExtractTestFailDetailsFromMessageBus(DelayedMessageBus delayedMessageBus)
+        {
+            string details = "";
+
+            foreach (var message in delayedMessageBus.messages)
+            {
+                if (message.ToString() == "Xunit.Sdk.TestFailed")
+                {
+                    try
+                    {
+                        var messages = (string[])message.GetType().GetProperty("Messages").GetValue(message);
+                        var exceptionTypes = (string[])message.GetType().GetProperty("ExceptionTypes").GetValue(message);
+                        var stackTraces = (string[])message.GetType().GetProperty("StackTraces").GetValue(message);
+
+                        if (messages != null && messages.Length > 0)
+                        {
+                            details += "Messages: " + string.Join(";", messages) + ". ";
+                        }
+
+                        if (exceptionTypes != null && exceptionTypes.Length > 0)
+                        {
+                            details += "ExceptionTypes: " + string.Join(";", exceptionTypes) + ". ";
+                        }
+
+                        if (stackTraces != null && stackTraces.Length > 0)
+                        {
+                            details += "StackTraces: " + string.Join(";", stackTraces) + ".";
+                        }
+                    }
+                    catch
+                    {
+                        Console.WriteLine($"Fail to read test fail message from message bus.");
+                    }
+                }
+            }
+
+            return details;
+        }
+
+        public override void Serialize(IXunitSerializationInfo data)
+        {
+            base.Serialize(data);
+
+            data.AddValue("MaxRetries", maxRetries);
+        }
+
+        public override void Deserialize(IXunitSerializationInfo data)
+        {
+            base.Deserialize(data);
+
+            maxRetries = data.GetValue<int>("MaxRetries");
+        }
+    }
+}
+

test/Microsoft.ML.Tests/FeatureContributionTests.cs

@@ -10,6 +10,7 @@
 using Microsoft.ML.Internal.Utilities;
 using Microsoft.ML.RunTests;
 using Microsoft.ML.TestFramework.Attributes;
+using Microsoft.ML.TestFrameworkCommon.Attributes;
 using Microsoft.ML.Trainers;
 using Xunit;
 using Xunit.Abstractions;
@@ -89,7 +90,7 @@ public void TestPoissonRegression()
                 new LbfgsPoissonRegressionTrainer.Options { NumberOfThreads = 1 }), GetSparseDataset(numberOfInstances: 100), "PoissonRegression");
         }
 
-        [Fact]
+        [RetryFact]
         public void TestGAMRegression()
         {
             TestFeatureContribution(ML.Regression.Trainers.Gam(), GetSparseDataset(numberOfInstances: 100), "GAMRegression");

test/Microsoft.ML.Tests/OnnxConversionTest.cs

@@ -200,7 +200,7 @@ public void KmeansOnnxConversionTest()
             Done();
         }
 
-        [Fact]
+        [RetryFact]
         public void RegressionTrainersOnnxConversionTest()
         {
             var mlContext = new MLContext(seed: 1);