From 7dd0d0d31976318d3c139e9c6faeec1b28321f80 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Thu, 21 Dec 2023 14:49:19 -0800 Subject: [PATCH] Make double assertions compare with tolerance instead of precision Precision might cause small differences to round to a different number. Instead compare with a tolerance which is not sensitive to rounding. --- .../UnitTests/TestEntryPoints.cs | 20 +++--- .../UnitTests/TestLoss.cs | 6 +- .../UnitTests/TestVBuffer.cs | 2 +- .../UnitTests.cs | 18 +++--- .../MetricTest.cs | 22 +++---- .../UtilityTest.cs | 8 +-- .../IntrospectiveTraining.cs | 4 +- test/Microsoft.ML.IntegrationTests/ONNX.cs | 4 +- .../Microsoft.ML.IntegrationTests/Training.cs | 2 +- .../OnnxTransformTests.cs | 6 +- .../TestGamPublicInterfaces.cs | 2 +- .../TestPredictors.cs | 6 +- .../AnomalyDetectionTests.cs | 4 +- test/Microsoft.ML.Tests/EvaluateTests.cs | 2 +- .../LearningRateSchedulerTest.cs | 2 +- .../Scenarios/ClusteringTests.cs | 2 +- .../Scenarios/IrisPlantClassificationTests.cs | 28 ++++---- ...PlantClassificationWithStringLabelTests.cs | 28 ++++---- .../IrisPlantClassificationTests.cs | 26 ++++---- .../TensorflowTests.cs | 22 +++---- .../MatrixFactorizationTests.cs | 28 ++++---- .../TrainerEstimators/SdcaTests.cs | 16 ++--- .../TreeEnsembleFeaturizerTest.cs | 8 +-- .../TrainerEstimators/TreeEstimators.cs | 16 ++--- .../Transformers/NormalizerTests.cs | 64 +++++++++---------- .../TimeSeriesDirectApi.cs | 54 ++++++++-------- .../TimeSeriesSimpleApiTests.cs | 8 +-- 27 files changed, 204 insertions(+), 204 deletions(-) diff --git a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs index b927df44c2..80acaf5d70 100644 --- a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs +++ b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs @@ -94,7 +94,7 @@ public void EntryPointTrainTestSplit() int testRows = CountRows(splitOutput.TestData); Assert.Equal(totalRows, trainRows + testRows); - Assert.Equal(0.9, (double)trainRows / totalRows, 1); + Assert.Equal(0.9, (double)trainRows / totalRows, 0.1); } private static int CountRows(IDataView dataView) @@ -5005,7 +5005,7 @@ public void TestSimpleTrainExperiment() Assert.True(b); double auc = 0; getter(ref auc); - Assert.Equal(0.93, auc, 2); + Assert.Equal(0.93, auc, 0.01); b = cursor.MoveNext(); Assert.False(b); } @@ -5210,7 +5210,7 @@ public void TestCrossValidationMacro() if (w == 1) Assert.Equal(1.585, stdev, .001); else - Assert.Equal(1.39, stdev, 2); + Assert.Equal(1.39, stdev, 0.01); isWeightedGetter(ref isWeighted); Assert.True(isWeighted == (w == 1)); } @@ -5379,7 +5379,7 @@ public void TestCrossValidationMacroWithMulticlass() getter(ref stdev); foldGetter(ref fold); Assert.True(ReadOnlyMemoryUtils.EqualsStr("Standard Deviation", fold)); - Assert.Equal(0.024809923969586353, stdev, 3); + Assert.Equal(0.024809923969586353, stdev, 0.001); double sum = 0; double val = 0; @@ -5788,7 +5788,7 @@ public void TestCrossValidationMacroWithStratification() getter(ref stdev); foldGetter(ref fold); Assert.True(ReadOnlyMemoryUtils.EqualsStr("Standard Deviation", fold)); - Assert.Equal(0.02582, stdev, 5); + Assert.Equal(0.02582, stdev, 0.00001); double sum = 0; double val = 0; @@ -6089,9 +6089,9 @@ public void TestCrossValidationMacroWithNonDefaultNames() foldGetter(ref fold); Assert.True(ReadOnlyMemoryUtils.EqualsStr("Standard Deviation", fold)); var stdevValues = stdev.GetValues(); - Assert.Equal(0.02462, stdevValues[0], 5); - Assert.Equal(0.02763, stdevValues[1], 5); - Assert.Equal(0.03273, stdevValues[2], 5); + Assert.Equal(0.02462, stdevValues[0], 0.00001); + Assert.Equal(0.02763, stdevValues[1], 0.00001); + Assert.Equal(0.03273, stdevValues[2], 0.00001); var sumBldr = new BufferBuilder(R8Adder.Instance); sumBldr.Reset(avg.Length, true); @@ -6291,7 +6291,7 @@ public void TestOvaMacro() Assert.True(b); double acc = 0; getter(ref acc); - Assert.Equal(0.96, acc, 2); + Assert.Equal(0.96, acc, 0.01); b = cursor.MoveNext(); Assert.False(b); } @@ -6463,7 +6463,7 @@ public void TestOvaMacroWithUncalibratedLearner() Assert.True(b); double acc = 0; getter(ref acc); - Assert.Equal(0.71, acc, 2); + Assert.Equal(0.71, acc, 0.01); b = cursor.MoveNext(); Assert.False(b); } diff --git a/test/Microsoft.ML.Core.Tests/UnitTests/TestLoss.cs b/test/Microsoft.ML.Core.Tests/UnitTests/TestLoss.cs index d7428dbed3..868c1f679c 100644 --- a/test/Microsoft.ML.Core.Tests/UnitTests/TestLoss.cs +++ b/test/Microsoft.ML.Core.Tests/UnitTests/TestLoss.cs @@ -39,8 +39,8 @@ private void TestHelper(IScalarLoss lossFunc, double label, double output, doubl { Double loss = lossFunc.Loss((float)output, (float)label); float derivative = lossFunc.Derivative((float)output, (float)label); - Assert.Equal(expectedLoss, loss, 5); - Assert.Equal(expectedUpdate, -derivative, 5); + Assert.Equal(expectedLoss, loss, 0.00001); + Assert.Equal(expectedUpdate, -derivative, 0.00001); if (differentiable) { @@ -48,7 +48,7 @@ private void TestHelper(IScalarLoss lossFunc, double label, double output, doubl // Use a simple finite difference method to see if it's in the right ballpark. float almostOutput = Math.Max((float)output * (1 + _epsilon), (float)output + _epsilon); Double almostLoss = lossFunc.Loss(almostOutput, (float)label); - Assert.Equal((almostLoss - loss) / (almostOutput - output), derivative, 1); + Assert.Equal((almostLoss - loss) / (almostOutput - output), derivative, 0.1); } } diff --git a/test/Microsoft.ML.Core.Tests/UnitTests/TestVBuffer.cs b/test/Microsoft.ML.Core.Tests/UnitTests/TestVBuffer.cs index be2af7f5a4..c12786125c 100644 --- a/test/Microsoft.ML.Core.Tests/UnitTests/TestVBuffer.cs +++ b/test/Microsoft.ML.Core.Tests/UnitTests/TestVBuffer.cs @@ -293,7 +293,7 @@ public void TestSparsifyNormalize(int startRange, bool normalize, float[] expect var actualValues = a.GetValues().ToArray(); Assert.Equal(expectedValues.Length, actualValues.Length); for (int i = 0; i < expectedValues.Length; i++) - Assert.Equal(expectedValues[i], actualValues[i], precision: 6); + Assert.Equal(expectedValues[i], actualValues[i], 0.000001); } /// diff --git a/test/Microsoft.ML.CpuMath.UnitTests/UnitTests.cs b/test/Microsoft.ML.CpuMath.UnitTests/UnitTests.cs index cbae22eb1e..edf99ef5c9 100644 --- a/test/Microsoft.ML.CpuMath.UnitTests/UnitTests.cs +++ b/test/Microsoft.ML.CpuMath.UnitTests/UnitTests.cs @@ -645,7 +645,7 @@ public void SumTest(string mode, string test, Dictionary environ } var actual = CpuMathUtils.Sum(src); - Assert.Equal((double)expected, (double)actual, 2); + Assert.Equal((double)expected, (double)actual, 0.01); return RemoteExecutor.SuccessExitCode; }, mode, test, options); } @@ -668,7 +668,7 @@ public void SumSqUTest(string mode, string test, Dictionary envi } var actual = CpuMathUtils.SumSq(src); - Assert.Equal((double)expected, (double)actual, 2); + Assert.Equal((double)expected, (double)actual, 0.01); return RemoteExecutor.SuccessExitCode; }, mode, test, options); } @@ -693,7 +693,7 @@ public void SumSqDiffUTest(string mode, string test, string scale, Dictionary env } var actual = CpuMathUtils.SumAbs(src); - Assert.Equal((double)expected, (double)actual, 2); + Assert.Equal((double)expected, (double)actual, 0.01); return RemoteExecutor.SuccessExitCode; }, mode, test, options); } @@ -741,7 +741,7 @@ public void SumAbsDiffUTest(string mode, string test, string scale, Dictionary env } } - Assert.Equal((double)expected, (double)actual, 2); + Assert.Equal((double)expected, (double)actual, 0.01); return RemoteExecutor.SuccessExitCode; }, mode, test, options); } @@ -797,7 +797,7 @@ public void MaxAbsDiffUTest(string mode, string test, string scale, Dictionary enviro } var actual = CpuMathUtils.DotProductDense(src, dst, dst.Length); - Assert.Equal((double)expected, (double)actual, 1); + Assert.Equal((double)expected, (double)actual, 0.1); return RemoteExecutor.SuccessExitCode; }, mode, test, options); } @@ -861,7 +861,7 @@ public void DotSUTest(string mode, string test, Dictionary envir } var actual = CpuMathUtils.DotProductSparse(src, dst, idx, limit); - Assert.Equal((double)expected, (double)actual, 2); + Assert.Equal((double)expected, (double)actual, 0.01); return RemoteExecutor.SuccessExitCode; }, mode, test, options); } diff --git a/test/Microsoft.ML.Fairlearn.Tests/MetricTest.cs b/test/Microsoft.ML.Fairlearn.Tests/MetricTest.cs index a51c8ae06f..26e5283342 100644 --- a/test/Microsoft.ML.Fairlearn.Tests/MetricTest.cs +++ b/test/Microsoft.ML.Fairlearn.Tests/MetricTest.cs @@ -38,17 +38,17 @@ public void RegressionMetricTest() { RegressionGroupMetric regressionMetric = mlContext.Fairlearn().Metric.Regression(eval: data, labelColumn: "Price", scoreColumn: "Score", sensitiveFeatureColumn: "Gender"); var metricByGroup = regressionMetric.ByGroup(); - Assert.Equal(-2.30578, Convert.ToSingle(metricByGroup["RSquared"][0]), 3); - Assert.Equal(-2039.81453, Convert.ToSingle(metricByGroup["RSquared"][1]), 3); - Assert.Equal(1.00000, Convert.ToSingle(metricByGroup["RMS"][0]), 3); - Assert.Equal(15.811388, Convert.ToSingle(metricByGroup["RMS"][1]), 3); + Assert.Equal(-2.30578, Convert.ToSingle(metricByGroup["RSquared"][0]), 0.001); + Assert.Equal(-2039.81453, Convert.ToSingle(metricByGroup["RSquared"][1]), 0.001); + Assert.Equal(1.00000, Convert.ToSingle(metricByGroup["RMS"][0]), 0.001); + Assert.Equal(15.811388, Convert.ToSingle(metricByGroup["RMS"][1]), 0.001); metricByGroup.Description(); Dictionary metricOverall = regressionMetric.Overall(); - Assert.Equal(125.5, metricOverall["MSE"], 1); - Assert.Equal(11.202678, metricOverall["RMS"], 4); + Assert.Equal(125.5, metricOverall["MSE"], 0.1); + Assert.Equal(11.202678, metricOverall["RMS"], 0.0001); Dictionary diff = regressionMetric.DifferenceBetweenGroups(); - Assert.Equal(14.81138, diff["RMS"], 4); - Assert.Equal(2037.5, diff["RSquared"], 1); + Assert.Equal(14.81138, diff["RMS"], 0.0001); + Assert.Equal(2037.5, diff["RSquared"], 0.1); } @@ -70,10 +70,10 @@ public void BinaryClassificationMetricTest() BinaryGroupMetric metrics = mlContext.Fairlearn().Metric.BinaryClassification(eval: df, labelColumn: "label", predictedColumn: "PredictedLabel", sensitiveFeatureColumn: "group_id"); var metricByGroup = metrics.ByGroup(); - Assert.Equal(0.8, Convert.ToSingle(metricByGroup["Accuracy"][0]), 1); - Assert.Equal(0.6, Convert.ToSingle(metricByGroup["Accuracy"][1]), 1); + Assert.Equal(0.8, Convert.ToSingle(metricByGroup["Accuracy"][0]), 0.1); + Assert.Equal(0.6, Convert.ToSingle(metricByGroup["Accuracy"][1]), 0.1); var metricOverall = metrics.Overall(); - Assert.Equal(0.7, Convert.ToSingle(metricOverall["Accuracy"]), 1); + Assert.Equal(0.7, Convert.ToSingle(metricOverall["Accuracy"]), 0.1); } } } diff --git a/test/Microsoft.ML.Fairlearn.Tests/UtilityTest.cs b/test/Microsoft.ML.Fairlearn.Tests/UtilityTest.cs index 3a0354755d..faca33296e 100644 --- a/test/Microsoft.ML.Fairlearn.Tests/UtilityTest.cs +++ b/test/Microsoft.ML.Fairlearn.Tests/UtilityTest.cs @@ -31,10 +31,10 @@ public void DemographyParityTest() PrimitiveDataFrameColumn ypred = new PrimitiveDataFrameColumn("pred", fl); var gSinged = dp.Gamma(ypred); - Assert.Equal(0.1, Convert.ToSingle(gSinged["value"][0]), 1); - Assert.Equal(-0.1, Convert.ToSingle(gSinged["value"][1]), 1); - Assert.Equal(-0.1, Convert.ToSingle(gSinged["value"][2]), 1); - Assert.Equal(0.1, Convert.ToSingle(gSinged["value"][3]), 1); + Assert.Equal(0.1, Convert.ToSingle(gSinged["value"][0]), 0.1); + Assert.Equal(-0.1, Convert.ToSingle(gSinged["value"][1]), 0.1); + Assert.Equal(-0.1, Convert.ToSingle(gSinged["value"][2]), 0.1); + Assert.Equal(0.1, Convert.ToSingle(gSinged["value"][3]), 0.1); } } } diff --git a/test/Microsoft.ML.IntegrationTests/IntrospectiveTraining.cs b/test/Microsoft.ML.IntegrationTests/IntrospectiveTraining.cs index 018c75d046..e58498b42c 100644 --- a/test/Microsoft.ML.IntegrationTests/IntrospectiveTraining.cs +++ b/test/Microsoft.ML.IntegrationTests/IntrospectiveTraining.cs @@ -119,8 +119,8 @@ public void InspectFastTreeModelParameters() var expectedThresholds = new float[] { 0.0911167f, 0.06509889f, 0.019873254f, 0.0361835f }; for (int i = 0; i < finalTree.NumberOfNodes; ++i) { - Assert.Equal(expectedSplitGains[i], finalTree.SplitGains[i], 6); - Assert.Equal((double)expectedThresholds[i], (double)finalTree.NumericalSplitThresholds[i], 6); + Assert.Equal(expectedSplitGains[i], finalTree.SplitGains[i], 0.000001); + Assert.Equal((double)expectedThresholds[i], (double)finalTree.NumericalSplitThresholds[i], 0.000001); } } diff --git a/test/Microsoft.ML.IntegrationTests/ONNX.cs b/test/Microsoft.ML.IntegrationTests/ONNX.cs index 3a598b8c21..fea40744d1 100644 --- a/test/Microsoft.ML.IntegrationTests/ONNX.cs +++ b/test/Microsoft.ML.IntegrationTests/ONNX.cs @@ -71,7 +71,7 @@ public void SaveOnnxModelLoadAndScoreFastTree() var originalPrediction = originalPredictionEngine.Predict(row); var onnxPrediction = onnxPredictionEngine.Predict(row); // Check that the predictions are identical. - Assert.Equal(originalPrediction.Score, onnxPrediction.Score[0], precision: 4); + Assert.Equal(originalPrediction.Score, onnxPrediction.Score[0], 0.0001); } } @@ -170,7 +170,7 @@ public void SaveOnnxModelLoadAndScoreSDCA() var originalPrediction = originalPredictionEngine.Predict(row); var onnxPrediction = onnxPredictionEngine.Predict(row); // Check that the predictions are identical. - Assert.Equal(originalPrediction.Score, onnxPrediction.Score[0], precision: 4); + Assert.Equal(originalPrediction.Score, onnxPrediction.Score[0], 0.0001); } } } diff --git a/test/Microsoft.ML.IntegrationTests/Training.cs b/test/Microsoft.ML.IntegrationTests/Training.cs index 51f3338110..49783f1d86 100644 --- a/test/Microsoft.ML.IntegrationTests/Training.cs +++ b/test/Microsoft.ML.IntegrationTests/Training.cs @@ -498,7 +498,7 @@ public void MetacomponentsFunctionWithKeyHandling() // Evaluate the model. var binaryClassificationMetrics = mlContext.MulticlassClassification.Evaluate(binaryClassificationPredictions); - Assert.Equal(0.4367, binaryClassificationMetrics.LogLoss, 4); + Assert.Equal(0.4367, binaryClassificationMetrics.LogLoss, 0.0001); } } } diff --git a/test/Microsoft.ML.OnnxTransformerTest/OnnxTransformTests.cs b/test/Microsoft.ML.OnnxTransformerTest/OnnxTransformTests.cs index e69fffb63a..8d6646fe39 100644 --- a/test/Microsoft.ML.OnnxTransformerTest/OnnxTransformTests.cs +++ b/test/Microsoft.ML.OnnxTransformerTest/OnnxTransformTests.cs @@ -476,9 +476,9 @@ public void TestOnnxNoneDimValue() var transformedValues = onnxTransformer.Transform(idv); var predictions = mlContext.Data.CreateEnumerable(transformedValues, reuseRowObject: false).ToArray(); - Assert.Equal(-0.080, Math.Round(predictions[0].variable[0], 3)); - Assert.Equal(1.204, Math.Round(predictions[1].variable[0], 3)); - Assert.Equal(2.27, Math.Round(predictions[2].variable[0], 3)); + Assert.Equal(-0.080, predictions[0].variable[0], 0.001); + Assert.Equal(1.204, predictions[1].variable[0], 0.001); + Assert.Equal(2.27, predictions[2].variable[0], 0.001); } /// diff --git a/test/Microsoft.ML.Predictor.Tests/TestGamPublicInterfaces.cs b/test/Microsoft.ML.Predictor.Tests/TestGamPublicInterfaces.cs index 8984e96882..a18b0fa945 100644 --- a/test/Microsoft.ML.Predictor.Tests/TestGamPublicInterfaces.cs +++ b/test/Microsoft.ML.Predictor.Tests/TestGamPublicInterfaces.cs @@ -43,7 +43,7 @@ public void TestGamDirectInstantiation() Assert.Equal(binUpperBounds.Length, gam.NumberOfShapeFunctions); // Check the intercept - Assert.Equal(intercept, gam.Bias, 6); + Assert.Equal(intercept, gam.Bias, 0.000001); // Check that the binUpperBounds were made correctly CheckArrayOfArrayEquality(binUpperBounds, gam.GetBinUpperBounds()); diff --git a/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs b/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs index 681cfd6b23..2e7b194fd1 100644 --- a/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs +++ b/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs @@ -737,8 +737,8 @@ private void CombineAndTestTreeEnsembles(IDataView idv, PredictorModel[] fastTre probGetters[i](ref probs[i]); predGetters[i](ref preds[i]); } - Assert.Equal(score, 0.4 * scores.Sum() / predCount, 5); - Assert.Equal(prob, 1 / (1 + Math.Exp(-score)), 6); + Assert.Equal(score, 0.4 * scores.Sum() / predCount, 0.00001); + Assert.Equal(prob, 1 / (1 + Math.Exp(-score)), 0.000001); Assert.True(pred == score > 0); } } @@ -953,7 +953,7 @@ private void CombineAndTestEnsembles(IDataView idv, string name, string options, for (int j = 0; j < predCount; j++) sum += vectorScores[j].GetItemOrDefault(i); if (float.IsNaN(sum)) - Assert.Equal((double)vectorScore.GetItemOrDefault(i), (double)sum / predCount, 3); + Assert.Equal((double)vectorScore.GetItemOrDefault(i), (double)sum / predCount, 0.001); } Assert.Equal(probs.Count(p => p >= prob), probs.Count(p => p <= prob)); } diff --git a/test/Microsoft.ML.Tests/AnomalyDetectionTests.cs b/test/Microsoft.ML.Tests/AnomalyDetectionTests.cs index 3d00592191..3581af5198 100644 --- a/test/Microsoft.ML.Tests/AnomalyDetectionTests.cs +++ b/test/Microsoft.ML.Tests/AnomalyDetectionTests.cs @@ -33,8 +33,8 @@ public void RandomizedPcaTrainerBaselineTest() // Evaluate var metrics = ML.AnomalyDetection.Evaluate(transformedData, falsePositiveCount: 5); - Assert.Equal(0.98667, metrics.AreaUnderRocCurve, 5); - Assert.Equal(0.90000, metrics.DetectionRateAtFalsePositiveCount, 5); + Assert.Equal(0.98667, metrics.AreaUnderRocCurve, 0.00001); + Assert.Equal(0.90000, metrics.DetectionRateAtFalsePositiveCount, 0.00001); } /// diff --git a/test/Microsoft.ML.Tests/EvaluateTests.cs b/test/Microsoft.ML.Tests/EvaluateTests.cs index 28a6cfa3a5..ae404b9bd5 100644 --- a/test/Microsoft.ML.Tests/EvaluateTests.cs +++ b/test/Microsoft.ML.Tests/EvaluateTests.cs @@ -65,7 +65,7 @@ public void MulticlassEvaluatorTopKArray() var metrics2 = mlContext.MulticlassClassification.Evaluate(inputDV2, topKPredictionCount: 4); var output2 = metrics2.TopKAccuracyForAllK.ToArray(); for (int i = 0; i < expectedTopKArray2.Length; i++) - Assert.Equal(expectedTopKArray2[i], output2[i], precision: 7); + Assert.Equal(expectedTopKArray2[i], output2[i], 0.0000001); } } } diff --git a/test/Microsoft.ML.Tests/LearningRateSchedulerTest.cs b/test/Microsoft.ML.Tests/LearningRateSchedulerTest.cs index 57ca637b7f..779911d85c 100644 --- a/test/Microsoft.ML.Tests/LearningRateSchedulerTest.cs +++ b/test/Microsoft.ML.Tests/LearningRateSchedulerTest.cs @@ -48,7 +48,7 @@ internal void TestPolynomialDecay(float[] expectedValues, bool cycle) trainState.CurrentBatchIndex = i % trainState.BatchesPerEpoch; trainState.CurrentEpoch = i / trainState.BatchesPerEpoch; float decayedLR = learningRateScheduler.GetLearningRate(trainState); - Assert.Equal((double)expectedValues[i], (double)decayedLR, 4); + Assert.Equal((double)expectedValues[i], (double)decayedLR, 0.0001); } } } diff --git a/test/Microsoft.ML.Tests/Scenarios/ClusteringTests.cs b/test/Microsoft.ML.Tests/Scenarios/ClusteringTests.cs index cdc157a244..2eccbb4db4 100644 --- a/test/Microsoft.ML.Tests/Scenarios/ClusteringTests.cs +++ b/test/Microsoft.ML.Tests/Scenarios/ClusteringTests.cs @@ -87,7 +87,7 @@ public void PredictClusters() Assert.Equal(double.NaN, metrics.NormalizedMutualInformation); //Calculate dbi is false by default so Dbi would be 0 Assert.Equal(0d, metrics.DaviesBouldinIndex); - Assert.Equal(0d, metrics.AverageDistance, 5); + Assert.Equal(0d, metrics.AverageDistance, 0.00001); } } } diff --git a/test/Microsoft.ML.Tests/Scenarios/IrisPlantClassificationTests.cs b/test/Microsoft.ML.Tests/Scenarios/IrisPlantClassificationTests.cs index 6357cad60e..9e94c0f6b5 100644 --- a/test/Microsoft.ML.Tests/Scenarios/IrisPlantClassificationTests.cs +++ b/test/Microsoft.ML.Tests/Scenarios/IrisPlantClassificationTests.cs @@ -55,9 +55,9 @@ public void TrainAndPredictIrisModelTest() PetalWidth = 0.2f, }); - Assert.Equal(1d, prediction.PredictedLabels[0], 2); - Assert.Equal(0d, prediction.PredictedLabels[1], 2); - Assert.Equal(0d, prediction.PredictedLabels[2], 2); + Assert.Equal(1d, prediction.PredictedLabels[0], 0.01); + Assert.Equal(0d, prediction.PredictedLabels[1], 0.01); + Assert.Equal(0d, prediction.PredictedLabels[2], 0.01); prediction = predictFunction.Predict(new IrisData() { @@ -67,9 +67,9 @@ public void TrainAndPredictIrisModelTest() PetalWidth = 2.2f, }); - Assert.Equal(0d, prediction.PredictedLabels[0], 2); - Assert.Equal(0d, prediction.PredictedLabels[1], 2); - Assert.Equal(1d, prediction.PredictedLabels[2], 2); + Assert.Equal(0d, prediction.PredictedLabels[0], 0.01); + Assert.Equal(0d, prediction.PredictedLabels[1], 0.01); + Assert.Equal(1d, prediction.PredictedLabels[2], 0.01); prediction = predictFunction.Predict(new IrisData() { @@ -79,23 +79,23 @@ public void TrainAndPredictIrisModelTest() PetalWidth = 1.2f, }); - Assert.Equal(.2, prediction.PredictedLabels[0], 1); - Assert.Equal(.8, prediction.PredictedLabels[1], 1); - Assert.Equal(0d, prediction.PredictedLabels[2], 2); + Assert.Equal(.2, prediction.PredictedLabels[0], 0.1); + Assert.Equal(.8, prediction.PredictedLabels[1], 0.1); + Assert.Equal(0d, prediction.PredictedLabels[2], 0.01); // Evaluate the trained pipeline var predicted = trainedModel.Transform(testData); var metrics = mlContext.MulticlassClassification.Evaluate(predicted, topKPredictionCount: 3); Assert.Equal(.98, metrics.MacroAccuracy); - Assert.Equal(.98, metrics.MicroAccuracy, 2); - Assert.Equal(.06, metrics.LogLoss, 2); + Assert.Equal(.98, metrics.MicroAccuracy, 0.01); + Assert.Equal(.06, metrics.LogLoss, 0.01); Assert.Equal(1, metrics.TopKAccuracy); Assert.Equal(3, metrics.PerClassLogLoss.Count); - Assert.Equal(0d, metrics.PerClassLogLoss[0], 1); - Assert.Equal(.1, metrics.PerClassLogLoss[1], 1); - Assert.Equal(.1, metrics.PerClassLogLoss[2], 1); + Assert.Equal(0d, metrics.PerClassLogLoss[0], 0.1); + Assert.Equal(.1, metrics.PerClassLogLoss[1], 0.1); + Assert.Equal(.1, metrics.PerClassLogLoss[2], 0.1); } public class IrisData diff --git a/test/Microsoft.ML.Tests/Scenarios/IrisPlantClassificationWithStringLabelTests.cs b/test/Microsoft.ML.Tests/Scenarios/IrisPlantClassificationWithStringLabelTests.cs index d2eb57dee2..4e3592851f 100644 --- a/test/Microsoft.ML.Tests/Scenarios/IrisPlantClassificationWithStringLabelTests.cs +++ b/test/Microsoft.ML.Tests/Scenarios/IrisPlantClassificationWithStringLabelTests.cs @@ -54,9 +54,9 @@ public void TrainAndPredictIrisModelWithStringLabelTest() PetalWidth = 0.2f, }); - Assert.Equal(1d, prediction.PredictedScores[0], 2); - Assert.Equal(0d, prediction.PredictedScores[1], 2); - Assert.Equal(0d, prediction.PredictedScores[2], 2); + Assert.Equal(1d, prediction.PredictedScores[0], 0.01); + Assert.Equal(0d, prediction.PredictedScores[1], 0.01); + Assert.Equal(0d, prediction.PredictedScores[2], 0.01); Assert.True(prediction.PredictedPlant == "Iris-setosa"); prediction = predictFunction.Predict(new IrisDataWithStringLabel() @@ -67,9 +67,9 @@ public void TrainAndPredictIrisModelWithStringLabelTest() PetalWidth = 2.2f, }); - Assert.Equal(0d, prediction.PredictedScores[0], 2); - Assert.Equal(0d, prediction.PredictedScores[1], 2); - Assert.Equal(1d, prediction.PredictedScores[2], 2); + Assert.Equal(0d, prediction.PredictedScores[0], 0.01); + Assert.Equal(0d, prediction.PredictedScores[1], 0.01); + Assert.Equal(1d, prediction.PredictedScores[2], 0.01); Assert.True(prediction.PredictedPlant == "Iris-virginica"); prediction = predictFunction.Predict(new IrisDataWithStringLabel() @@ -80,9 +80,9 @@ public void TrainAndPredictIrisModelWithStringLabelTest() PetalWidth = 1.2f, }); - Assert.Equal(.2, prediction.PredictedScores[0], 1); - Assert.Equal(.8, prediction.PredictedScores[1], 1); - Assert.Equal(0d, prediction.PredictedScores[2], 2); + Assert.Equal(.2, prediction.PredictedScores[0], 0.1); + Assert.Equal(.8, prediction.PredictedScores[1], 0.1); + Assert.Equal(0d, prediction.PredictedScores[2], 0.01); Assert.True(prediction.PredictedPlant == "Iris-versicolor"); // Evaluate the trained pipeline @@ -90,15 +90,15 @@ public void TrainAndPredictIrisModelWithStringLabelTest() var metrics = mlContext.MulticlassClassification.Evaluate(predicted, topKPredictionCount: 3); Assert.Equal(.98, metrics.MacroAccuracy); - Assert.Equal(.98, metrics.MicroAccuracy, 2); - Assert.Equal(.06, metrics.LogLoss, 2); + Assert.Equal(.98, metrics.MicroAccuracy, 0.01); + Assert.Equal(.06, metrics.LogLoss, 0.01); Assert.InRange(metrics.LogLossReduction, 0.94, 0.96); Assert.Equal(1, metrics.TopKAccuracy); Assert.Equal(3, metrics.PerClassLogLoss.Count); - Assert.Equal(0d, metrics.PerClassLogLoss[0], 1); - Assert.Equal(.1, metrics.PerClassLogLoss[1], 1); - Assert.Equal(.1, metrics.PerClassLogLoss[2], 1); + Assert.Equal(0d, metrics.PerClassLogLoss[0], 0.1); + Assert.Equal(.1, metrics.PerClassLogLoss[1], 0.1); + Assert.Equal(.1, metrics.PerClassLogLoss[2], 0.1); } private class IrisDataWithStringLabel diff --git a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/IrisPlantClassificationTests.cs b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/IrisPlantClassificationTests.cs index 8b393afeaf..20866f494c 100644 --- a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/IrisPlantClassificationTests.cs +++ b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/IrisPlantClassificationTests.cs @@ -61,9 +61,9 @@ private void ComparePredictions(PredictionEngine model PetalWidth = 0.2f, }); - Assert.Equal(1d, prediction.PredictedLabels[0], 2); - Assert.Equal(0d, prediction.PredictedLabels[1], 2); - Assert.Equal(0d, prediction.PredictedLabels[2], 2); + Assert.Equal(1d, prediction.PredictedLabels[0], 0.01); + Assert.Equal(0d, prediction.PredictedLabels[1], 0.01); + Assert.Equal(0d, prediction.PredictedLabels[2], 0.01); prediction = model.Predict(new IrisData() { @@ -73,9 +73,9 @@ private void ComparePredictions(PredictionEngine model PetalWidth = 2.2f, }); - Assert.Equal(0d, prediction.PredictedLabels[0], 2); - Assert.Equal(0d, prediction.PredictedLabels[1], 2); - Assert.Equal(1d, prediction.PredictedLabels[2], 2); + Assert.Equal(0d, prediction.PredictedLabels[0], 0.01); + Assert.Equal(0d, prediction.PredictedLabels[1], 0.01); + Assert.Equal(1d, prediction.PredictedLabels[2], 0.01); prediction = model.Predict(new IrisData() { @@ -85,22 +85,22 @@ private void ComparePredictions(PredictionEngine model PetalWidth = 1.2f, }); - Assert.Equal(.2, prediction.PredictedLabels[0], 1); - Assert.Equal(.8, prediction.PredictedLabels[1], 1); - Assert.Equal(0d, prediction.PredictedLabels[2], 2); + Assert.Equal(.2, prediction.PredictedLabels[0], 0.1); + Assert.Equal(.8, prediction.PredictedLabels[1], 0.1); + Assert.Equal(0d, prediction.PredictedLabels[2], 0.01); } private void CompareMetrics(MulticlassClassificationMetrics metrics) { Assert.Equal(.98, metrics.MacroAccuracy); - Assert.Equal(.98, metrics.MicroAccuracy, 2); + Assert.Equal(.98, metrics.MicroAccuracy, 0.01); Assert.InRange(metrics.LogLoss, .05, .06); Assert.InRange(metrics.LogLossReduction, 0.94, 0.96); Assert.Equal(3, metrics.PerClassLogLoss.Count); - Assert.Equal(0d, metrics.PerClassLogLoss[0], 1); - Assert.Equal(.1, metrics.PerClassLogLoss[1], 1); - Assert.Equal(.1, metrics.PerClassLogLoss[2], 1); + Assert.Equal(0d, metrics.PerClassLogLoss[0], 0.1); + Assert.Equal(.1, metrics.PerClassLogLoss[1], 0.1); + Assert.Equal(.1, metrics.PerClassLogLoss[2], 0.1); } } } diff --git a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs index 81eec25f9a..38b3ab97ec 100644 --- a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs +++ b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs @@ -161,24 +161,24 @@ public void TensorFlowTransforCifarEndToEndTest2() var predictions = transformer.Transform(data); var metrics = _mlContext.MulticlassClassification.Evaluate(predictions); - Assert.Equal(1, metrics.MicroAccuracy, 2); + Assert.Equal(1, metrics.MicroAccuracy, 0.01); var predictFunction = _mlContext.Model.CreatePredictionEngine(transformer); var prediction = predictFunction.Predict(new CifarData() { ImagePath = GetDataPath("images/banana.jpg") }); - Assert.Equal(0d, prediction.PredictedScores[0], 2); - Assert.Equal(1d, prediction.PredictedScores[1], 2); - Assert.Equal(0d, prediction.PredictedScores[2], 2); + Assert.Equal(0d, prediction.PredictedScores[0], 0.01); + Assert.Equal(1d, prediction.PredictedScores[1], 0.01); + Assert.Equal(0d, prediction.PredictedScores[2], 0.01); prediction = predictFunction.Predict(new CifarData() { ImagePath = GetDataPath("images/hotdog.jpg") }); - Assert.Equal(0d, prediction.PredictedScores[0], 2); - Assert.Equal(0d, prediction.PredictedScores[1], 2); - Assert.Equal(1d, prediction.PredictedScores[2], 2); + Assert.Equal(0d, prediction.PredictedScores[0], 0.01); + Assert.Equal(0d, prediction.PredictedScores[1], 0.01); + Assert.Equal(1d, prediction.PredictedScores[2], 0.01); (transformer as IDisposable)?.Dispose(); } @@ -677,7 +677,7 @@ public void TensorFlowTransformMNISTConvTest() var metrics = _mlContext.MulticlassClassification.Evaluate(predicted); Assert.Equal(0.99, metrics.MicroAccuracy, .01); - Assert.Equal(0.93, metrics.MacroAccuracy, 2); + Assert.Equal(0.93, metrics.MacroAccuracy, 0.01); var oneSample = GetOneMNISTExample(); @@ -902,7 +902,7 @@ public void TensorFlowTransformMNISTConvSavedModelTest() // First group of checks Assert.Equal(0.99, metrics.MicroAccuracy, .01); - Assert.Equal(.93, metrics.MacroAccuracy, 2); + Assert.Equal(.93, metrics.MacroAccuracy, 0.01); // An in-memory example. Its label is predicted below. var oneSample = GetOneMNISTExample(); @@ -1172,7 +1172,7 @@ public void TensorFlowSaveAndLoadSavedModel() var outputSchema = transformer.GetOutputSchema(data.Schema); var metrics = _mlContext.MulticlassClassification.Evaluate(transformedData); - Assert.Equal(1, metrics.MicroAccuracy, 2); + Assert.Equal(1, metrics.MicroAccuracy, 0.01); var predictFunction = _mlContext.Model.CreatePredictionEngine(transformer); var predictions = new[] @@ -1207,7 +1207,7 @@ public void TensorFlowSaveAndLoadSavedModel() for (var i = 0; i < predictions.Length; i++) { for (var j = 0; j < predictions[i].PredictedScores.Length; j++) - Assert.Equal((double)predictions[i].PredictedScores[j], (double)testPredictions[i].PredictedScores[j], 2); + Assert.Equal((double)predictions[i].PredictedScores[j], (double)testPredictions[i].PredictedScores[j], 0.01); } (testTransformer as IDisposable)?.Dispose(); testPredictFunction.Dispose(); diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs index aa0713e63d..df6ee1e2b3 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs @@ -96,12 +96,12 @@ public void MatrixFactorizationSimpleTrainAndPredict() if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { if (RuntimeInformation.ProcessArchitecture == Architecture.Arm64) - Assert.Equal(0.3041052520275116, leftMatrix[0], 4); + Assert.Equal(0.3041052520275116, leftMatrix[0], 0.0001); else - Assert.Equal(0.309137582778931, leftMatrix[0], 4); - Assert.Equal(0.468956589698792, leftMatrix[leftMatrix.Count - 1], 4); - Assert.Equal(0.303486406803131, rightMatrix[0], 4); - Assert.Equal(0.503888845443726, rightMatrix[rightMatrix.Count - 1], 4); + Assert.Equal(0.309137582778931, leftMatrix[0], 0.0001); + Assert.Equal(0.468956589698792, leftMatrix[leftMatrix.Count - 1], 0.0001); + Assert.Equal(0.303486406803131, rightMatrix[0], 0.0001); + Assert.Equal(0.503888845443726, rightMatrix[rightMatrix.Count - 1], 0.0001); } // Read the test data set as an IDataView var testData = reader.Load(new MultiFileSource(GetDataPath(TestDatasets.trivialMatrixFactorization.testFilename))); @@ -687,13 +687,13 @@ public void OneClassMatrixFactorizationSample() Assert.Equal(1u, firstElement.MatrixColumnIndex); Assert.Equal(1u, firstElement.MatrixRowIndex); - Assert.Equal(0.987113833, firstElement.Score, 3); - Assert.Equal(1d, firstElement.Value, 3); + Assert.Equal(0.987113833, firstElement.Score, 0.001); + Assert.Equal(1d, firstElement.Value, 0.001); Assert.Equal(60u, lastElement.MatrixColumnIndex); Assert.Equal(100u, lastElement.MatrixRowIndex); - Assert.Equal(0.149993762, lastElement.Score, 3); - Assert.Equal(0.15, lastElement.Value, 3); + Assert.Equal(0.149993762, lastElement.Score, 0.001); + Assert.Equal(0.15, lastElement.Value, 0.001); // Two columns with highest predicted score to the 2nd row (indexed by 1). If we view row index as user ID and column as game ID, // the following list contains the games recommended by the trained model. Note that sometime, you may want to exclude training @@ -705,13 +705,13 @@ public void OneClassMatrixFactorizationSample() Assert.Equal(1u, firstElement.MatrixColumnIndex); Assert.Equal(1u, firstElement.MatrixRowIndex); - Assert.Equal(0.987113833, firstElement.Score, 3); - Assert.Equal(1d, firstElement.Value, 3); + Assert.Equal(0.987113833, firstElement.Score, 0.001); + Assert.Equal(1d, firstElement.Value, 0.001); Assert.Equal(11u, lastElement.MatrixColumnIndex); Assert.Equal(1u, lastElement.MatrixRowIndex); - Assert.Equal(0.987113833, lastElement.Score, 3); - Assert.Equal(1d, lastElement.Value, 3); + Assert.Equal(0.987113833, lastElement.Score, 0.001); + Assert.Equal(1d, lastElement.Value, 0.001); } // A data structure used to encode a single value in matrix @@ -842,7 +842,7 @@ public void InspectMatrixFactorizationModel() // Check if results computed by SSE code and MF predictor are the same. for (int i = 0; i < predictions.Count(); ++i) - Assert.Equal((double)predictions[i].Score, (double)valuesAtSecondColumn[i], 3); + Assert.Equal((double)predictions[i].Score, (double)valuesAtSecondColumn[i], 0.001); } } } diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/SdcaTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/SdcaTests.cs index 5b10ca358c..159f341071 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/SdcaTests.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/SdcaTests.cs @@ -129,10 +129,10 @@ public void SdcaLogisticRegressionWithWeight() // Verify the metrics produced are different. var metrics1 = mlContext.BinaryClassification.Evaluate(prediction1); var metrics2 = mlContext.BinaryClassification.Evaluate(prediction2); - Assert.Equal(0.9658, metrics1.AreaUnderRocCurve, 4); - Assert.Equal(0.3488, metrics1.LogLoss, 4); - Assert.Equal(0.9596, metrics2.AreaUnderRocCurve, 4); - Assert.Equal(0.3591, metrics2.LogLoss, 4); + Assert.Equal(0.9658, metrics1.AreaUnderRocCurve, 0.0001); + Assert.Equal(0.3488, metrics1.LogLoss, 0.0001); + Assert.Equal(0.9596, metrics2.AreaUnderRocCurve, 0.0001); + Assert.Equal(0.3591, metrics2.LogLoss, 0.0001); // Verify the raw scores are different. var scores1 = prediction1.GetColumn(prediction1.Schema["Score"]).ToArray(); @@ -188,10 +188,10 @@ public void SdcaMaximumEntropyWithWeight() // Verify the metrics produced are different. var metrics1 = mlContext.MulticlassClassification.Evaluate(prediction1, labelColumnName: "LabelIndex", topKPredictionCount: 1); var metrics2 = mlContext.MulticlassClassification.Evaluate(prediction2, labelColumnName: "LabelIndex", topKPredictionCount: 1); - Assert.Equal(0.9100, metrics1.TopKAccuracy, 4); - Assert.Equal(0.2411, metrics1.LogLoss, 4); - Assert.Equal(0.8800, metrics2.TopKAccuracy, 4); - Assert.Equal(0.2464, metrics2.LogLoss, 4); + Assert.Equal(0.9100, metrics1.TopKAccuracy, 0.0001); + Assert.Equal(0.2411, metrics1.LogLoss, 0.0001); + Assert.Equal(0.8800, metrics2.TopKAccuracy, 0.0001); + Assert.Equal(0.2464, metrics2.LogLoss, 0.0001); // Verify the raw scores are different. var scores1 = prediction1.GetColumn(prediction1.Schema["Score"]).ToArray(); diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEnsembleFeaturizerTest.cs b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEnsembleFeaturizerTest.cs index 28392c7ac1..9719fefb26 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEnsembleFeaturizerTest.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEnsembleFeaturizerTest.cs @@ -627,8 +627,8 @@ public void TestSaveAndLoadTreeFeaturizer() var loadedPrediction = loadedModel.Transform(dataView); var loadedMetrics = ML.Regression.Evaluate(loadedPrediction); - Assert.Equal(metrics.MeanAbsoluteError, loadedMetrics.MeanAbsoluteError, 5); - Assert.Equal(metrics.MeanSquaredError, loadedMetrics.MeanSquaredError, 5); + Assert.Equal(metrics.MeanAbsoluteError, loadedMetrics.MeanAbsoluteError, 0.00001); + Assert.Equal(metrics.MeanSquaredError, loadedMetrics.MeanSquaredError, 0.00001); } [Fact] @@ -687,8 +687,8 @@ public void TestSaveAndLoadDoubleTreeFeaturizer() var loadedMetrics = ML.Regression.Evaluate(loadedPrediction); // Check if the loaded model produces the same result as the trained model. - Assert.Equal(metrics.MeanAbsoluteError, loadedMetrics.MeanAbsoluteError, 5); - Assert.Equal(metrics.MeanSquaredError, loadedMetrics.MeanSquaredError, 5); + Assert.Equal(metrics.MeanAbsoluteError, loadedMetrics.MeanAbsoluteError, 0.00001); + Assert.Equal(metrics.MeanSquaredError, loadedMetrics.MeanSquaredError, 0.00001); var secondPipeline = ML.Transforms.CopyColumns("CopiedFeatures", "Features") .Append(ML.Transforms.NormalizeBinning("CopiedFeatures")) diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs index 6a618dfc24..42540f478f 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs @@ -557,7 +557,7 @@ public void LightGbmMulticlassEstimatorCompareOva() double sum = 0; for (int j = 0; j < _classNumber; ++j) { - Assert.Equal(nativeResult0[j + i * _classNumber], mlnetPredictions[i].Score[j], 6); + Assert.Equal(nativeResult0[j + i * _classNumber], mlnetPredictions[i].Score[j], 0.000001); if (float.IsNaN((float)nativeResult1[j + i * _classNumber])) continue; sum += MathUtils.SigmoidSlow(sigmoidScale * (float)nativeResult1[j + i * _classNumber]); @@ -565,7 +565,7 @@ public void LightGbmMulticlassEstimatorCompareOva() for (int j = 0; j < _classNumber; ++j) { double prob = MathUtils.SigmoidSlow(sigmoidScale * (float)nativeResult1[j + i * _classNumber]); - Assert.Equal(prob / sum, mlnetPredictions[i].Score[j], 6); + Assert.Equal(prob / sum, mlnetPredictions[i].Score[j], 0.000001); } } @@ -593,7 +593,7 @@ public void LightGbmMulticlassEstimatorCompareOvaUsingSigmoids() double sum = 0; for (int j = 0; j < _classNumber; ++j) { - Assert.Equal(nativeResult0[j + i * _classNumber], mlnetPredictions[i].Score[j], 6); + Assert.Equal(nativeResult0[j + i * _classNumber], mlnetPredictions[i].Score[j], 0.000001); if (float.IsNaN((float)nativeResult1[j + i * _classNumber])) continue; sum += MathUtils.SigmoidSlow((float)sigmoidScale * (float)nativeResult1[j + i * _classNumber]); @@ -601,7 +601,7 @@ public void LightGbmMulticlassEstimatorCompareOvaUsingSigmoids() for (int j = 0; j < _classNumber; ++j) { double prob = MathUtils.SigmoidSlow((float)sigmoidScale * (float)nativeResult1[j + i * _classNumber]); - Assert.Equal(prob / sum, mlnetPredictions[i].Score[j], 6); + Assert.Equal(prob / sum, mlnetPredictions[i].Score[j], 0.000001); } } @@ -664,13 +664,13 @@ public void LightGbmMulticlassEstimatorCompareSoftMax() double sum = 0; for (int j = 0; j < _classNumber; ++j) { - Assert.Equal(nativeResult0[j + i * _classNumber], mlnetPredictions[i].Score[j], 6); + Assert.Equal(nativeResult0[j + i * _classNumber], mlnetPredictions[i].Score[j], 0.000001); sum += Math.Exp((float)nativeResult1[j + i * _classNumber]); } for (int j = 0; j < _classNumber; ++j) { double prob = Math.Exp(nativeResult1[j + i * _classNumber]); - Assert.Equal(prob / sum, mlnetPredictions[i].Score[j], 6); + Assert.Equal(prob / sum, mlnetPredictions[i].Score[j], 0.000001); } } @@ -693,13 +693,13 @@ public void LightGbmMulticlassEstimatorCompareUnbalanced() double sum = 0; for (int j = 0; j < _classNumber; ++j) { - Assert.Equal(nativeResult0[j + i * _classNumber], mlnetPredictions[i].Score[j], 6); + Assert.Equal(nativeResult0[j + i * _classNumber], mlnetPredictions[i].Score[j], 0.000001); sum += Math.Exp((float)nativeResult1[j + i * _classNumber]); } for (int j = 0; j < _classNumber; ++j) { double prob = Math.Exp(nativeResult1[j + i * _classNumber]); - Assert.Equal(prob / sum, mlnetPredictions[i].Score[j], 6); + Assert.Equal(prob / sum, mlnetPredictions[i].Score[j], 0.000001); } } diff --git a/test/Microsoft.ML.Tests/Transformers/NormalizerTests.cs b/test/Microsoft.ML.Tests/Transformers/NormalizerTests.cs index 4d1f89bb67..b33d69574d 100644 --- a/test/Microsoft.ML.Tests/Transformers/NormalizerTests.cs +++ b/test/Microsoft.ML.Tests/Transformers/NormalizerTests.cs @@ -395,36 +395,36 @@ public void NormalizerParametersMultiColumnApi() var robustScalerTransformer = robustScalerEstimator.Fit(data); floatAffineModel = ((NormalizingTransformer)robustScalerTransformer).Columns[0].ModelParameters as NormalizingTransformer.AffineNormalizerModelParameters; - Assert.Equal(1 / 1.8, floatAffineModel.Scale, 2); - Assert.Equal(5.8d, floatAffineModel.Offset, 2); + Assert.Equal(1 / 1.8, floatAffineModel.Scale, 0.01); + Assert.Equal(5.8d, floatAffineModel.Offset, 0.01); floatAffineModelVec = ((NormalizingTransformer)robustScalerTransformer).Columns[1].ModelParameters as NormalizingTransformer.AffineNormalizerModelParameters>; Assert.Equal(4, floatAffineModelVec.Scale.Length); - Assert.Equal(.5555556, floatAffineModelVec.Scale[0], 2); - Assert.Equal(.8333333, floatAffineModelVec.Scale[1], 2); - Assert.Equal(.3389830, floatAffineModelVec.Scale[2], 2); - Assert.Equal(.8333333, floatAffineModelVec.Scale[3], 2); + Assert.Equal(.5555556, floatAffineModelVec.Scale[0], 0.01); + Assert.Equal(.8333333, floatAffineModelVec.Scale[1], 0.01); + Assert.Equal(.3389830, floatAffineModelVec.Scale[2], 0.01); + Assert.Equal(.8333333, floatAffineModelVec.Scale[3], 0.01); - Assert.Equal(5.8, floatAffineModelVec.Offset[0], 2); - Assert.Equal(3d, floatAffineModelVec.Offset[1], 2); - Assert.Equal(4.4, floatAffineModelVec.Offset[2], 2); - Assert.Equal(1.3, floatAffineModelVec.Offset[3], 2); + Assert.Equal(5.8, floatAffineModelVec.Offset[0], 0.01); + Assert.Equal(3d, floatAffineModelVec.Offset[1], 0.01); + Assert.Equal(4.4, floatAffineModelVec.Offset[2], 0.01); + Assert.Equal(1.3, floatAffineModelVec.Offset[3], 0.01); doubleAffineModel = ((NormalizingTransformer)robustScalerTransformer).Columns[2].ModelParameters as NormalizingTransformer.AffineNormalizerModelParameters; - Assert.Equal(1 / 1.8, doubleAffineModel.Scale, 2); - Assert.Equal(5.8, doubleAffineModel.Offset, 2); + Assert.Equal(1 / 1.8, doubleAffineModel.Scale, 0.01); + Assert.Equal(5.8, doubleAffineModel.Offset, 0.01); doubleAffineModelVector = ((NormalizingTransformer)robustScalerTransformer).Columns[3].ModelParameters as NormalizingTransformer.AffineNormalizerModelParameters>; Assert.Equal(4, doubleAffineModelVector.Scale.Length); - Assert.Equal(.5555556, doubleAffineModelVector.Scale[0], 2); - Assert.Equal(.8333333, doubleAffineModelVector.Scale[1], 2); - Assert.Equal(.3389830, doubleAffineModelVector.Scale[2], 2); - Assert.Equal(.8333333, doubleAffineModelVector.Scale[3], 2); + Assert.Equal(.5555556, doubleAffineModelVector.Scale[0], 0.01); + Assert.Equal(.8333333, doubleAffineModelVector.Scale[1], 0.01); + Assert.Equal(.3389830, doubleAffineModelVector.Scale[2], 0.01); + Assert.Equal(.8333333, doubleAffineModelVector.Scale[3], 0.01); - Assert.Equal(5.8, doubleAffineModelVector.Offset[0], 2); - Assert.Equal(3, doubleAffineModelVector.Offset[1], 2); - Assert.Equal(4.4, doubleAffineModelVector.Offset[2], 2); - Assert.Equal(1.3, doubleAffineModelVector.Offset[3], 2); + Assert.Equal(5.8, doubleAffineModelVector.Offset[0], 0.01); + Assert.Equal(3, doubleAffineModelVector.Offset[1], 0.01); + Assert.Equal(4.4, doubleAffineModelVector.Offset[2], 0.01); + Assert.Equal(1.3, doubleAffineModelVector.Offset[3], 0.01); // Robust scaler no offset robustScalerEstimator = context.Transforms.NormalizeRobustScaling( @@ -435,28 +435,28 @@ public void NormalizerParametersMultiColumnApi() robustScalerTransformer = robustScalerEstimator.Fit(data); floatAffineModel = ((NormalizingTransformer)robustScalerTransformer).Columns[0].ModelParameters as NormalizingTransformer.AffineNormalizerModelParameters; - Assert.Equal(1 / 1.8, floatAffineModel.Scale, 2); - Assert.Equal(0d, floatAffineModel.Offset, 2); + Assert.Equal(1 / 1.8, floatAffineModel.Scale, 0.01); + Assert.Equal(0d, floatAffineModel.Offset, 0.01); floatAffineModelVec = ((NormalizingTransformer)robustScalerTransformer).Columns[1].ModelParameters as NormalizingTransformer.AffineNormalizerModelParameters>; Assert.Equal(4, floatAffineModelVec.Scale.Length); - Assert.Equal(.5555556, floatAffineModelVec.Scale[0], 2); - Assert.Equal(.8333333, floatAffineModelVec.Scale[1], 2); - Assert.Equal(.3389830, floatAffineModelVec.Scale[2], 2); - Assert.Equal(.8333333, floatAffineModelVec.Scale[3], 2); + Assert.Equal(.5555556, floatAffineModelVec.Scale[0], 0.01); + Assert.Equal(.8333333, floatAffineModelVec.Scale[1], 0.01); + Assert.Equal(.3389830, floatAffineModelVec.Scale[2], 0.01); + Assert.Equal(.8333333, floatAffineModelVec.Scale[3], 0.01); Assert.Empty(floatAffineModelVec.Offset); doubleAffineModel = ((NormalizingTransformer)robustScalerTransformer).Columns[2].ModelParameters as NormalizingTransformer.AffineNormalizerModelParameters; - Assert.Equal(1 / 1.8, doubleAffineModel.Scale, 2); - Assert.Equal(0, doubleAffineModel.Offset, 2); + Assert.Equal(1 / 1.8, doubleAffineModel.Scale, 0.01); + Assert.Equal(0, doubleAffineModel.Offset, 0.01); doubleAffineModelVector = ((NormalizingTransformer)robustScalerTransformer).Columns[3].ModelParameters as NormalizingTransformer.AffineNormalizerModelParameters>; Assert.Equal(4, doubleAffineModelVector.Scale.Length); - Assert.Equal(.5555556, doubleAffineModelVector.Scale[0], 2); - Assert.Equal(.8333333, doubleAffineModelVector.Scale[1], 2); - Assert.Equal(.3389830, doubleAffineModelVector.Scale[2], 2); - Assert.Equal(.8333333, doubleAffineModelVector.Scale[3], 2); + Assert.Equal(.5555556, doubleAffineModelVector.Scale[0], 0.01); + Assert.Equal(.8333333, doubleAffineModelVector.Scale[1], 0.01); + Assert.Equal(.3389830, doubleAffineModelVector.Scale[2], 0.01); + Assert.Equal(.8333333, doubleAffineModelVector.Scale[3], 0.01); Assert.Empty(doubleAffineModelVector.Offset); diff --git a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs index c38e426b65..ce727c5daa 100644 --- a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs +++ b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesDirectApi.cs @@ -195,10 +195,10 @@ public void ChangePointDetectionWithSeasonality() while (enumerator.MoveNext() && index < expectedValues.Count) { row = enumerator.Current; - Assert.Equal(expectedValues[index++], row.Change[0], precision: 7); // Alert - Assert.Equal(expectedValues[index++], row.Change[1], precision: 7); // Raw score - Assert.Equal(expectedValues[index++], row.Change[2], precision: 7); // P-Value score - Assert.Equal(expectedValues[index++], row.Change[3], precision: 7); // Martingale score + Assert.Equal(expectedValues[index++], row.Change[0], 0.0000001); // Alert + Assert.Equal(expectedValues[index++], row.Change[1], 0.0000001); // Raw score + Assert.Equal(expectedValues[index++], row.Change[2], 0.0000001); // P-Value score + Assert.Equal(expectedValues[index++], row.Change[3], 0.0000001); // Martingale score } } @@ -255,10 +255,10 @@ public void ChangePointDetectionWithSeasonalityPredictionEngineNoColumn() var engine2 = model2.CreateTimeSeriesEngine(ml); var prediction2 = engine2.Predict(new Data(1)); //Raw score after first input. - Assert.Equal(1.1661833524703979, prediction2.Change[1], precision: 5); // Raw score + Assert.Equal(1.1661833524703979, prediction2.Change[1], 0.00001); // Raw score prediction2 = engine2.Predict(new Data(1)); //Raw score after second input. - Assert.Equal(0.12216401100158691, prediction2.Change[1], precision: 5); // Raw score + Assert.Equal(0.12216401100158691, prediction2.Change[1], 0.00001); // Raw score //Even though time series column is not requested it will // pass the observation through time series transform and update the state with the first input. @@ -275,7 +275,7 @@ public void ChangePointDetectionWithSeasonalityPredictionEngineNoColumn() //and raw score should match the raw score obtained by passing the two input in the first model. var engine3 = model3.CreateTimeSeriesEngine(ml); var prediction3 = engine3.Predict(new Data(1)); - Assert.Equal(0.12216401100158691, prediction2.Change[1], precision: 5); // Raw score + Assert.Equal(0.12216401100158691, prediction2.Change[1], 0.00001); // Raw score } [NativeDependencyFact("MklImports")] @@ -318,10 +318,10 @@ public void ChangePointDetectionWithSeasonalityPredictionEngine() //Model 1: Prediction #1. var engine = model.CreateTimeSeriesEngine(ml); var prediction = engine.Predict(new Data(1)); - Assert.Equal(0, prediction.Change[0], precision: 7); // Alert - Assert.Equal(1.1661833524703979, prediction.Change[1], precision: 5); // Raw score - Assert.Equal(0.5, prediction.Change[2], precision: 7); // P-Value score - Assert.Equal(5.1200000000000114E-08, prediction.Change[3], precision: 7); // Martingale score + Assert.Equal(0, prediction.Change[0], 0.0000001); // Alert + Assert.Equal(1.1661833524703979, prediction.Change[1], 0.00001); // Raw score + Assert.Equal(0.5, prediction.Change[2], 0.0000001); // P-Value score + Assert.Equal(5.1200000000000114E-08, prediction.Change[3], 0.0000001); // Martingale score //Model 1: Checkpoint. var modelPath = "temp.zip"; @@ -329,10 +329,10 @@ public void ChangePointDetectionWithSeasonalityPredictionEngine() //Model 1: Prediction #2 prediction = engine.Predict(new Data(1)); - Assert.Equal(0, prediction.Change[0], precision: 7); // Alert - Assert.Equal(0.12216401100158691, prediction.Change[1], precision: 5); // Raw score - Assert.Equal(0.14823824685192111, prediction.Change[2], precision: 5); // P-Value score - Assert.Equal(1.5292508189989167E-07, prediction.Change[3], precision: 7); // Martingale score + Assert.Equal(0, prediction.Change[0], 0.0000001); // Alert + Assert.Equal(0.12216401100158691, prediction.Change[1], 0.00001); // Raw score + Assert.Equal(0.14823824685192111, prediction.Change[2], 0.00001); // P-Value score + Assert.Equal(1.5292508189989167E-07, prediction.Change[3], 0.0000001); // Martingale score // Load Model 1. ITransformer model2 = null; @@ -342,10 +342,10 @@ public void ChangePointDetectionWithSeasonalityPredictionEngine() //Predict and expect the same result after checkpointing(Prediction #2). engine = model2.CreateTimeSeriesEngine(ml); prediction = engine.Predict(new Data(1)); - Assert.Equal(0, prediction.Change[0], precision: 7); // Alert - Assert.Equal(0.12216401100158691, prediction.Change[1], precision: 5); // Raw score - Assert.Equal(0.14823824685192111, prediction.Change[2], precision: 5); // P-Value score - Assert.Equal(1.5292508189989167E-07, prediction.Change[3], precision: 5); // Martingale score + Assert.Equal(0, prediction.Change[0], 0.0000001); // Alert + Assert.Equal(0.12216401100158691, prediction.Change[1], 0.00001); // Raw score + Assert.Equal(0.14823824685192111, prediction.Change[2], 0.00001); // P-Value score + Assert.Equal(1.5292508189989167E-07, prediction.Change[3], 0.00001); // Martingale score } [NativeDependencyFact("MklImports")] @@ -405,9 +405,9 @@ public void SsaForecast() for (int localIndex = 0; localIndex < 4; localIndex++) { - Assert.Equal(expectedForecast[localIndex], row.Forecast[localIndex], precision: 7); - Assert.Equal(minCnf[localIndex], row.MinCnf[localIndex], precision: 7); - Assert.Equal(maxCnf[localIndex], row.MaxCnf[localIndex], precision: 7); + Assert.Equal(expectedForecast[localIndex], row.Forecast[localIndex], 0.0000001); + Assert.Equal(minCnf[localIndex], row.MinCnf[localIndex], 0.0000001); + Assert.Equal(maxCnf[localIndex], row.MaxCnf[localIndex], 0.0000001); } } @@ -645,7 +645,7 @@ public void TestSrCnnBatchAnomalyDetector( if (k == 20) { Assert.Equal(1, prediction.Prediction[0]); - Assert.Equal(5.00, prediction.Prediction[3], 2); + Assert.Equal(5.00, prediction.Prediction[3], 0.01); } else Assert.Equal(0, prediction.Prediction[0]); @@ -655,10 +655,10 @@ public void TestSrCnnBatchAnomalyDetector( if (k == 20) { Assert.Equal(1, prediction.Prediction[0]); - Assert.Equal(5.00, prediction.Prediction[3], 2); - Assert.Equal(5.00, prediction.Prediction[4], 2); - Assert.Equal(5.01, prediction.Prediction[5], 2); - Assert.Equal(4.99, prediction.Prediction[6], 2); + Assert.Equal(5.00, prediction.Prediction[3], 0.01); + Assert.Equal(5.00, prediction.Prediction[4], 0.01); + Assert.Equal(5.01, prediction.Prediction[5], 0.01); + Assert.Equal(4.99, prediction.Prediction[6], 0.01); Assert.True(prediction.Prediction[6] > data[k].Value || data[k].Value > prediction.Prediction[5]); } else diff --git a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesSimpleApiTests.cs b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesSimpleApiTests.cs index 7852eb36ec..b1758ac691 100644 --- a/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesSimpleApiTests.cs +++ b/test/Microsoft.ML.TimeSeries.Tests/TimeSeriesSimpleApiTests.cs @@ -66,10 +66,10 @@ public void ChangeDetection() { row = enumerator.Current; - Assert.Equal(expectedValues[index++], row.Data[0], precision: 7); - Assert.Equal(expectedValues[index++], row.Data[1], precision: 7); - Assert.Equal(expectedValues[index++], row.Data[2], precision: 7); - Assert.Equal(expectedValues[index++], row.Data[3], precision: 7); + Assert.Equal(expectedValues[index++], row.Data[0], 0.0000001); + Assert.Equal(expectedValues[index++], row.Data[1], 0.0000001); + Assert.Equal(expectedValues[index++], row.Data[2], 0.0000001); + Assert.Equal(expectedValues[index++], row.Data[3], 0.0000001); } }