Bugfix/hardwired sigmoid (dotnet#3850)

* fixed Hardcoded sigmoid value. Let Microsoft.ML.Tests see internals of Microsoft.ML.StandardTrainers

* Removed extra whitespace between comment and code

* changed which parameter was internal

* changed unneeded internal parameter back to private

* Put comments in correct format, sorted the using list

* Added direct tests between ML.NET and LightGBM with non-default sigmoid.

* Update test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs

Co-Authored-By: Wei-Sheng Chin <[email protected]>

* Update test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs

Co-Authored-By: Wei-Sheng Chin <[email protected]>

* Update test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs

Co-Authored-By: Wei-Sheng Chin <[email protected]>

* renamed sigmoid to sigmoidScale

* changed sigmoid from a default parameter so it can come before the out params

Author: michaelgsharp

src/Microsoft.ML.LightGbm/LightGbmBinaryTrainer.cs

@@ -232,7 +232,7 @@ private protected override CalibratedModelParametersBase<LightGbmBinaryModelPara
             Host.Check(TrainedEnsemble != null, "The predictor cannot be created before training is complete");
             var innerArgs = LightGbmInterfaceUtils.JoinParameters(base.GbmOptions);
             var pred = new LightGbmBinaryModelParameters(Host, TrainedEnsemble, FeatureCount, innerArgs);
-            var cali = new PlattCalibrator(Host, -0.5, 0);
+            var cali = new PlattCalibrator(Host, -LightGbmTrainerOptions.Sigmoid, 0);
             return new FeatureWeightsCalibratedModelParameters<LightGbmBinaryModelParameters, PlattCalibrator>(Host, pred, cali);
         }
 

src/Microsoft.ML.LightGbm/LightGbmMulticlassTrainer.cs

@@ -185,7 +185,7 @@ private protected override OneVersusAllModelParameters CreatePredictor()
             for (int i = 0; i < _tlcNumClass; ++i)
             {
                 var pred = CreateBinaryPredictor(i, innerArgs);
-                var cali = new PlattCalibrator(Host, -0.5, 0);
+                var cali = new PlattCalibrator(Host, -LightGbmTrainerOptions.Sigmoid, 0);
                 predictors[i] = new FeatureWeightsCalibratedModelParameters<LightGbmBinaryModelParameters, PlattCalibrator>(Host, pred, cali);
             }
             string obj = (string)GetGbmParameters()["objective"];

src/Microsoft.ML.StandardTrainers/Properties/AssemblyInfo.cs

@@ -8,6 +8,7 @@
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Ensemble" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.StaticPipe" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Core.Tests" + PublicKey.TestValue)]
+[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Tests" + PublicKey.TestValue)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Predictor.Tests" + PublicKey.TestValue)]
 
 [assembly: InternalsVisibleTo(assemblyName: "RunTests" + InternalPublicKey.Value)]

src/Microsoft.ML.StandardTrainers/Standard/MulticlassClassification/OneVersusAllTrainer.cs

@@ -267,7 +267,7 @@ private static VersionInfo GetVersionInfo()
         /// <summary>
         /// Retrieves the model parameters.
         /// </summary>
-        private ImmutableArray<object> SubModelParameters => _impl.Predictors.Cast<object>().ToImmutableArray();
+        internal ImmutableArray<object> SubModelParameters => _impl.Predictors.Cast<object>().ToImmutableArray();
 
         /// <summary>
         /// The type of the prediction task.

test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs

@@ -6,13 +6,14 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
+using Microsoft.ML.Calibrators;
 using Microsoft.ML.Data;
 using Microsoft.ML.Internal.Utilities;
-using Microsoft.ML.Trainers.LightGbm;
 using Microsoft.ML.RunTests;
 using Microsoft.ML.Runtime;
 using Microsoft.ML.TestFramework.Attributes;
 using Microsoft.ML.Trainers.FastTree;
+using Microsoft.ML.Trainers.LightGbm;
 using Microsoft.ML.Transforms;
 using Xunit;
 
@@ -64,6 +65,31 @@ public void LightGBMBinaryEstimator()
             Done();
         }
 
+        /// <summary>
+        /// LightGBMBinaryTrainer CorrectSigmoid test 
+        /// </summary>
+        [LightGBMFact]
+        public void LightGBMBinaryEstimatorCorrectSigmoid()
+        {
+            var (pipe, dataView) = GetBinaryClassificationPipeline();
+            var sigmoid = .789;
+
+            var trainer = ML.BinaryClassification.Trainers.LightGbm(new LightGbmBinaryTrainer.Options
+            {
+                NumberOfLeaves = 10,
+                NumberOfThreads = 1,
+                MinimumExampleCountPerLeaf = 2,
+                Sigmoid = sigmoid
+            });
+
+            var transformedDataView = pipe.Fit(dataView).Transform(dataView);
+            var model = trainer.Fit(transformedDataView, transformedDataView);
+
+            // The slope in the model calibrator should be equal to the negative of the sigmoid passed into the trainer.
+            Assert.Equal(sigmoid, -model.Model.Calibrator.Slope);
+            Done();
+        }
+
 
         [Fact]
         public void GAMClassificationEstimator()
@@ -251,6 +277,32 @@ public void LightGbmMulticlassEstimator()
             Done();
         }
 
+        /// <summary>
+        /// LightGbmMulticlass CorrectSigmoid test 
+        /// </summary>
+        [LightGBMFact]
+        public void LightGbmMulticlassEstimatorCorrectSigmoid()
+        {
+            var (pipeline, dataView) = GetMulticlassPipeline();
+            var sigmoid = .789;
+
+            var trainer = ML.MulticlassClassification.Trainers.LightGbm(new LightGbmMulticlassTrainer.Options
+            {
+                Sigmoid = sigmoid
+            });
+
+            var pipe = pipeline.Append(trainer)
+                    .Append(new KeyToValueMappingEstimator(Env, "PredictedLabel"));
+
+            var transformedDataView = pipe.Fit(dataView).Transform(dataView);
+            var model = trainer.Fit(transformedDataView, transformedDataView);
+
+            // The slope in the all the calibrators should be equal to the negative of the sigmoid passed into the trainer.
+            Assert.True(model.Model.SubModelParameters.All(predictor =>
+            ((FeatureWeightsCalibratedModelParameters<LightGbmBinaryModelParameters, PlattCalibrator>)predictor).Calibrator.Slope == -sigmoid));
+            Done();
+        }
+
         // Number of examples
         private const int _rowNumber = 1000;
         // Number of features
@@ -267,7 +319,7 @@ private class GbmExample
             public float[] Score;
         }
 
-        private void LightGbmHelper(bool useSoftmax, out string modelString, out List<GbmExample> mlnetPredictions, out double[] lgbmRawScores, out double[] lgbmProbabilities)
+        private void LightGbmHelper(bool useSoftmax, double sigmoid, out string modelString, out List<GbmExample> mlnetPredictions, out double[] lgbmRawScores, out double[] lgbmProbabilities)
         {
             // Prepare data and train LightGBM model via ML.NET
             // Training matrix. It contains all feature vectors.
@@ -300,7 +352,8 @@ private void LightGbmHelper(bool useSoftmax, out string modelString, out List<Gb
                     NumberOfIterations = numberOfTrainingIterations,
                     MinimumExampleCountPerGroup = 1,
                     MinimumExampleCountPerLeaf = 1,
-                    UseSoftmax = useSoftmax
+                    UseSoftmax = useSoftmax,
+                    Sigmoid = sigmoid // Custom sigmoid value.
                 });
 
             var gbm = gbmTrainer.Fit(dataView);
@@ -376,14 +429,15 @@ private void LightGbmHelper(bool useSoftmax, out string modelString, out List<Gb
         [LightGBMFact]
         public void LightGbmMulticlassEstimatorCompareOva()
         {
+            float sigmoidScale = 0.5f; // Constant used train LightGBM. See gbmParams["sigmoid"] in the helper function.
+
             // Train ML.NET LightGBM and native LightGBM and apply the trained models to the training set.
-            LightGbmHelper(useSoftmax: false, out string modelString, out List<GbmExample> mlnetPredictions, out double[] nativeResult1, out double[] nativeResult0);
+            LightGbmHelper(useSoftmax: false, sigmoid: sigmoidScale, out string modelString, out List<GbmExample> mlnetPredictions, out double[] nativeResult1, out double[] nativeResult0);
 
             // The i-th predictor returned by LightGBM produces the raw score, denoted by z_i, of the i-th class.
             // Assume that we have n classes in total. The i-th class probability can be computed via
             // p_i = sigmoid(sigmoidScale * z_i) / (sigmoid(sigmoidScale * z_1) + ... + sigmoid(sigmoidScale * z_n)).
             Assert.True(modelString != null);
-            float sigmoidScale = 0.5f; // Constant used train LightGBM. See gbmParams["sigmoid"] in the helper function.
             // Compare native LightGBM's and ML.NET's LightGBM results example by example
             for (int i = 0; i < _rowNumber; ++i)
             {
@@ -405,11 +459,87 @@ public void LightGbmMulticlassEstimatorCompareOva()
             Done();
         }
 
+        /// <summary>
+        /// Test LightGBM's sigmoid parameter with a custom value. This test checks if ML.NET and LightGBM produce the same result.
+        /// </summary>
+        [LightGBMFact]
+        public void LightGbmMulticlassEstimatorCompareOvaUsingSigmoids()
+        {
+            var sigmoidScale = .790;
+            // Train ML.NET LightGBM and native LightGBM and apply the trained models to the training set.
+            LightGbmHelper(useSoftmax: false, sigmoid: sigmoidScale, out string modelString, out List<GbmExample> mlnetPredictions, out double[] nativeResult1, out double[] nativeResult0);
+
+            // The i-th predictor returned by LightGBM produces the raw score, denoted by z_i, of the i-th class.
+            // Assume that we have n classes in total. The i-th class probability can be computed via
+            // p_i = sigmoid(sigmoidScale * z_i) / (sigmoid(sigmoidScale * z_1) + ... + sigmoid(sigmoidScale * z_n)).
+            Assert.True(modelString != null);
+
+            // Compare native LightGBM's and ML.NET's LightGBM results example by example
+            for (int i = 0; i < _rowNumber; ++i)
+            {
+                double sum = 0;
+                for (int j = 0; j < _classNumber; ++j)
+                {
+                    Assert.Equal(nativeResult0[j + i * _classNumber], mlnetPredictions[i].Score[j], 6);
+                    if (float.IsNaN((float)nativeResult1[j + i * _classNumber]))
+                        continue;
+                    sum += MathUtils.SigmoidSlow((float)sigmoidScale * (float)nativeResult1[j + i * _classNumber]);
+                }
+                for (int j = 0; j < _classNumber; ++j)
+                {
+                    double prob = MathUtils.SigmoidSlow((float)sigmoidScale * (float)nativeResult1[j + i * _classNumber]);
+                    Assert.Equal(prob / sum, mlnetPredictions[i].Score[j], 6);
+                }
+            }
+
+            Done();
+        }
+
+        /// <summary>
+        /// Make sure different sigmoid parameters produce different scores. In this test, two LightGBM models are trained with two different sigmoid values.
+        /// </summary>
+        [LightGBMFact]
+        public void LightGbmMulticlassEstimatorCompareOvaUsingDifferentSigmoids()
+        {
+            // Run native implemenation twice, see that results are different with different sigmoid values.
+            var firstSigmoidScale = .790;
+            var secondSigmoidScale = .2;
+
+            // Train native LightGBM with both sigmoid values and apply the trained models to the training set.
+            LightGbmHelper(useSoftmax: false, sigmoid: firstSigmoidScale, out string firstModelString, out List<GbmExample> firstMlnetPredictions, out double[] firstNativeResult1, out double[] firstNativeResult0);
+            LightGbmHelper(useSoftmax: false, sigmoid: secondSigmoidScale, out string secondModelString, out List<GbmExample> secondMlnetPredictions, out double[] secondNativeResult1, out double[] secondNativeResult0);
+
+            // Compare native LightGBM's results when 2 different sigmoid values are used.
+            for (int i = 0; i < _rowNumber; ++i)
+            {
+                var areEqual = true;
+                for (int j = 0; j < _classNumber; ++j)
+                {
+                    if (float.IsNaN((float)firstNativeResult1[j + i * _classNumber]))
+                        continue;
+                    if (float.IsNaN((float)secondNativeResult1[j + i * _classNumber]))
+                        continue;
+
+                    // Testing to make sure that at least 1 value is different. This avoids false positives when values are 0
+                    // even for the same sigmoid value.
+                    areEqual &= firstMlnetPredictions[i].Score[j].Equals(secondMlnetPredictions[i].Score[j]);
+
+                    // Testing that the native result is different before we apply the sigmoid.
+                    Assert.NotEqual((float)firstNativeResult1[j + i * _classNumber], (float)secondNativeResult1[j + i * _classNumber], 6);
+                }
+
+                // There should be at least 1 value that is different in the row.
+                Assert.False(areEqual);
+            }
+
+            Done();
+        }
+
         [LightGBMFact]
         public void LightGbmMulticlassEstimatorCompareSoftMax()
         {
             // Train ML.NET LightGBM and native LightGBM and apply the trained models to the training set.
-            LightGbmHelper(useSoftmax: true, out string modelString, out List<GbmExample> mlnetPredictions, out double[] nativeResult1, out double[] nativeResult0);
+            LightGbmHelper(useSoftmax: true, sigmoid: .5, out string modelString, out List<GbmExample> mlnetPredictions, out double[] nativeResult1, out double[] nativeResult0);
 
             // The i-th predictor returned by LightGBM produces the raw score, denoted by z_i, of the i-th class.
             // Assume that we have n classes in total. The i-th class probability can be computed via