From cfb7fa8c9f542a03260b8a32d20c48fd5b413ab3 Mon Sep 17 00:00:00 2001 From: Mustafa Bal Date: Fri, 14 Feb 2020 14:38:12 -0800 Subject: [PATCH 1/7] Updating expected unix error --- .../TrainerEstimators/MatrixFactorizationTests.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs index 2552c5be98..bbf23feb6e 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs @@ -54,8 +54,6 @@ public void MatrixFactorization_Estimator() } [MatrixFactorizationFact] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - [Trait("Category", "SkipInCI")] public void MatrixFactorizationSimpleTrainAndPredict() { var mlContext = new MLContext(seed: 1); @@ -128,7 +126,7 @@ public void MatrixFactorizationSimpleTrainAndPredict() if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { // Linux case - var expectedUnixL2Error = 0.614457914950479; // Linux baseline + var expectedUnixL2Error = 0.614440959250773; // Linux baseline Assert.InRange(metrices.MeanSquaredError, expectedUnixL2Error - tolerance, expectedUnixL2Error + tolerance); } else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) From 5fc14598dcaddd9134af78890894fb092bb61540 Mon Sep 17 00:00:00 2001 From: Mustafa Bal Date: Fri, 14 Feb 2020 15:27:48 -0800 Subject: [PATCH 2/7] Update MatrixFactorizationTests.cs --- .../TrainerEstimators/MatrixFactorizationTests.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs index 2552c5be98..e82a8663af 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs @@ -54,8 +54,6 @@ public void MatrixFactorization_Estimator() } [MatrixFactorizationFact] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - [Trait("Category", "SkipInCI")] public void MatrixFactorizationSimpleTrainAndPredict() { var mlContext = new MLContext(seed: 1); @@ -94,7 +92,7 @@ public void MatrixFactorizationSimpleTrainAndPredict() var rightMatrix = model.Model.RightFactorMatrix; Assert.Equal(leftMatrix.Count, model.Model.NumberOfRows * model.Model.ApproximationRank); Assert.Equal(rightMatrix.Count, model.Model.NumberOfColumns * model.Model.ApproximationRank); - // MF produce different matrixes on different platforms, so at least test thier content on windows. + // MF produce different matrixes on different platforms, so at least test their content on windows. if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { Assert.Equal(0.33491, leftMatrix[0], 5); @@ -124,12 +122,14 @@ public void MatrixFactorizationSimpleTrainAndPredict() var metrices = mlContext.Recommendation().Evaluate(prediction, labelColumnName: labelColumnName, scoreColumnName: scoreColumnName); // Determine if the selected metric is reasonable for different platforms - double tolerance = Math.Pow(10, -7); + double windowsTolerance = Math.Pow(10, -7); + // 1e-7 is too small for Linux, so we try 1e-5 + double linuxTolerance = Math.Pow(10, -5); if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { // Linux case var expectedUnixL2Error = 0.614457914950479; // Linux baseline - Assert.InRange(metrices.MeanSquaredError, expectedUnixL2Error - tolerance, expectedUnixL2Error + tolerance); + Assert.InRange(metrices.MeanSquaredError, expectedUnixL2Error - linuxTolerance, expectedUnixL2Error + linuxTolerance); } else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { @@ -142,7 +142,7 @@ public void MatrixFactorizationSimpleTrainAndPredict() { // Windows case var expectedWindowsL2Error = 0.6098110249191965; // Windows baseline - Assert.InRange(metrices.MeanSquaredError, expectedWindowsL2Error - tolerance, expectedWindowsL2Error + tolerance); + Assert.InRange(metrices.MeanSquaredError, expectedWindowsL2Error - windowsTolerance, expectedWindowsL2Error + windowsTolerance); } var modelWithValidation = pipeline.Fit(data, testData); From e75751f312ec181328cd2e9a9674531dd145efcc Mon Sep 17 00:00:00 2001 From: Mustafa Bal Date: Fri, 14 Feb 2020 15:29:32 -0800 Subject: [PATCH 3/7] Testing 1000 iterations of MatrixFactorizationSimpleTrainAndPredict --- .vsts-dotnet-ci.yml | 16 ++++------------ .../MatrixFactorizationTests.cs | 10 ++++++++-- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml index 5825f42af2..31b39c7cbd 100644 --- a/.vsts-dotnet-ci.yml +++ b/.vsts-dotnet-ci.yml @@ -25,7 +25,6 @@ jobs: _configuration: Release-netcoreapp3_0 _config_short: RI _includeBenchmarkData: true - innerLoop: true pool: name: Hosted Ubuntu 1604 @@ -34,7 +33,6 @@ jobs: name: Ubuntu_x64_NetCoreApp21 buildScript: ./build.sh container: UbuntuContainer - innerLoop: true pool: name: Hosted Ubuntu 1604 @@ -42,7 +40,6 @@ jobs: parameters: name: MacOS_x64_NetCoreApp21 buildScript: ./build.sh - innerLoop: true pool: name: Hosted macOS High Sierra @@ -59,18 +56,15 @@ jobs: _configuration: Release-netcoreapp3_0 _config_short: RI _includeBenchmarkData: true - innerLoop: true pool: - name: NetCorePublic-Pool - queue: buildpool.windows.10.amd64.vs2017.open + name: MachineLearning Test - template: /build/ci/job-template.yml parameters: name: Windows_x64_NetCoreApp21 buildScript: build.cmd - innerLoop: true pool: - name: Hosted VS2017 + name: MachineLearning Test - template: /build/ci/job-template.yml parameters: @@ -85,15 +79,13 @@ jobs: _configuration: Release-netfx _config_short: RFX _includeBenchmarkData: false - innerLoop: true pool: - name: Hosted VS2017 + name: MachineLearning Test - template: /build/ci/job-template.yml parameters: name: Windows_x86_NetCoreApp21 architecture: x86 buildScript: build.cmd - innerLoop: true pool: - name: Hosted VS2017 + name: MachineLearning Test diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs index e82a8663af..4d2e10dfc2 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs @@ -53,8 +53,10 @@ public void MatrixFactorization_Estimator() Done(); } - [MatrixFactorizationFact] - public void MatrixFactorizationSimpleTrainAndPredict() + [Theory] + [Trait("Category", "RunSpecificTest")] + [IterationData(1000)] + public void MatrixFactorizationSimpleTrainAndPredict(int iterations) { var mlContext = new MLContext(seed: 1); @@ -130,6 +132,10 @@ public void MatrixFactorizationSimpleTrainAndPredict() // Linux case var expectedUnixL2Error = 0.614457914950479; // Linux baseline Assert.InRange(metrices.MeanSquaredError, expectedUnixL2Error - linuxTolerance, expectedUnixL2Error + linuxTolerance); + if (iterations > 100) + { + Console.WriteLine(iterations); + } } else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { From 6365e319718a395a24031a2bc821b10ebacc91e1 Mon Sep 17 00:00:00 2001 From: Mustafa Bal Date: Fri, 14 Feb 2020 15:35:36 -0800 Subject: [PATCH 4/7] Update job-template.yml --- build/ci/job-template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/ci/job-template.yml b/build/ci/job-template.yml index dc4f6499ed..fc82ad64f4 100644 --- a/build/ci/job-template.yml +++ b/build/ci/job-template.yml @@ -7,7 +7,7 @@ parameters: codeCoverage: false nightlyBuild: false innerLoop: false - runSpecific: false + runSpecific: true container: '' jobs: From 9e3d8ec47c09f0ece64eb4dbcb198f232c0b6218 Mon Sep 17 00:00:00 2001 From: Mustafa Bal Date: Fri, 14 Feb 2020 15:50:02 -0800 Subject: [PATCH 5/7] Updating Linux tolerance to 1e-4 --- .../TrainerEstimators/MatrixFactorizationTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs index 4d2e10dfc2..944e85fd9b 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs @@ -125,8 +125,8 @@ public void MatrixFactorizationSimpleTrainAndPredict(int iterations) // Determine if the selected metric is reasonable for different platforms double windowsTolerance = Math.Pow(10, -7); - // 1e-7 is too small for Linux, so we try 1e-5 - double linuxTolerance = Math.Pow(10, -5); + // 1e-7 is too small for Linux, so we try 1e-4 + double linuxTolerance = Math.Pow(10, -4); if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { // Linux case From 0e453075a50ce9406c26707c647b03a3496bb379 Mon Sep 17 00:00:00 2001 From: Mustafa Bal Date: Mon, 17 Feb 2020 12:36:42 -0800 Subject: [PATCH 6/7] Reverting debugging code --- .vsts-dotnet-ci.yml | 16 ++++++++++++---- build/ci/job-template.yml | 2 +- .../MatrixFactorizationTests.cs | 8 +------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml index 31b39c7cbd..5825f42af2 100644 --- a/.vsts-dotnet-ci.yml +++ b/.vsts-dotnet-ci.yml @@ -25,6 +25,7 @@ jobs: _configuration: Release-netcoreapp3_0 _config_short: RI _includeBenchmarkData: true + innerLoop: true pool: name: Hosted Ubuntu 1604 @@ -33,6 +34,7 @@ jobs: name: Ubuntu_x64_NetCoreApp21 buildScript: ./build.sh container: UbuntuContainer + innerLoop: true pool: name: Hosted Ubuntu 1604 @@ -40,6 +42,7 @@ jobs: parameters: name: MacOS_x64_NetCoreApp21 buildScript: ./build.sh + innerLoop: true pool: name: Hosted macOS High Sierra @@ -56,15 +59,18 @@ jobs: _configuration: Release-netcoreapp3_0 _config_short: RI _includeBenchmarkData: true + innerLoop: true pool: - name: MachineLearning Test + name: NetCorePublic-Pool + queue: buildpool.windows.10.amd64.vs2017.open - template: /build/ci/job-template.yml parameters: name: Windows_x64_NetCoreApp21 buildScript: build.cmd + innerLoop: true pool: - name: MachineLearning Test + name: Hosted VS2017 - template: /build/ci/job-template.yml parameters: @@ -79,13 +85,15 @@ jobs: _configuration: Release-netfx _config_short: RFX _includeBenchmarkData: false + innerLoop: true pool: - name: MachineLearning Test + name: Hosted VS2017 - template: /build/ci/job-template.yml parameters: name: Windows_x86_NetCoreApp21 architecture: x86 buildScript: build.cmd + innerLoop: true pool: - name: MachineLearning Test + name: Hosted VS2017 diff --git a/build/ci/job-template.yml b/build/ci/job-template.yml index fc82ad64f4..dc4f6499ed 100644 --- a/build/ci/job-template.yml +++ b/build/ci/job-template.yml @@ -7,7 +7,7 @@ parameters: codeCoverage: false nightlyBuild: false innerLoop: false - runSpecific: true + runSpecific: false container: '' jobs: diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs index 944e85fd9b..20267335db 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs @@ -53,9 +53,7 @@ public void MatrixFactorization_Estimator() Done(); } - [Theory] - [Trait("Category", "RunSpecificTest")] - [IterationData(1000)] + [MatrixFactorizationFact] public void MatrixFactorizationSimpleTrainAndPredict(int iterations) { var mlContext = new MLContext(seed: 1); @@ -132,10 +130,6 @@ public void MatrixFactorizationSimpleTrainAndPredict(int iterations) // Linux case var expectedUnixL2Error = 0.614457914950479; // Linux baseline Assert.InRange(metrices.MeanSquaredError, expectedUnixL2Error - linuxTolerance, expectedUnixL2Error + linuxTolerance); - if (iterations > 100) - { - Console.WriteLine(iterations); - } } else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { From 127d37322b2419e9e129303bdd0844aa1c8d85a2 Mon Sep 17 00:00:00 2001 From: Mustafa Bal Date: Mon, 17 Feb 2020 17:16:55 -0800 Subject: [PATCH 7/7] Update MatrixFactorizationTests.cs --- .../TrainerEstimators/MatrixFactorizationTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs index 20267335db..594b95bd03 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs @@ -54,7 +54,7 @@ public void MatrixFactorization_Estimator() } [MatrixFactorizationFact] - public void MatrixFactorizationSimpleTrainAndPredict(int iterations) + public void MatrixFactorizationSimpleTrainAndPredict() { var mlContext = new MLContext(seed: 1);