diff --git a/src/Microsoft.ML.LightGbm/LightGbmBinaryTrainer.cs b/src/Microsoft.ML.LightGbm/LightGbmBinaryTrainer.cs index 861bbd1528..2efc892ae2 100644 --- a/src/Microsoft.ML.LightGbm/LightGbmBinaryTrainer.cs +++ b/src/Microsoft.ML.LightGbm/LightGbmBinaryTrainer.cs @@ -232,7 +232,7 @@ private protected override CalibratedModelParametersBase(Host, pred, cali); } diff --git a/src/Microsoft.ML.LightGbm/LightGbmMulticlassTrainer.cs b/src/Microsoft.ML.LightGbm/LightGbmMulticlassTrainer.cs index 2c0791fab5..40a0082fde 100644 --- a/src/Microsoft.ML.LightGbm/LightGbmMulticlassTrainer.cs +++ b/src/Microsoft.ML.LightGbm/LightGbmMulticlassTrainer.cs @@ -185,7 +185,7 @@ private protected override OneVersusAllModelParameters CreatePredictor() for (int i = 0; i < _tlcNumClass; ++i) { var pred = CreateBinaryPredictor(i, innerArgs); - var cali = new PlattCalibrator(Host, -0.5, 0); + var cali = new PlattCalibrator(Host, -LightGbmTrainerOptions.Sigmoid, 0); predictors[i] = new FeatureWeightsCalibratedModelParameters(Host, pred, cali); } string obj = (string)GetGbmParameters()["objective"]; diff --git a/src/Microsoft.ML.StandardTrainers/Properties/AssemblyInfo.cs b/src/Microsoft.ML.StandardTrainers/Properties/AssemblyInfo.cs index 8b2172b4f4..ecb343b55b 100644 --- a/src/Microsoft.ML.StandardTrainers/Properties/AssemblyInfo.cs +++ b/src/Microsoft.ML.StandardTrainers/Properties/AssemblyInfo.cs @@ -8,6 +8,7 @@ [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Ensemble" + PublicKey.Value)] [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.StaticPipe" + PublicKey.Value)] [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Core.Tests" + PublicKey.TestValue)] +[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Tests" + PublicKey.TestValue)] [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Predictor.Tests" + PublicKey.TestValue)] [assembly: InternalsVisibleTo(assemblyName: "RunTests" + InternalPublicKey.Value)] diff --git a/src/Microsoft.ML.StandardTrainers/Standard/MulticlassClassification/OneVersusAllTrainer.cs b/src/Microsoft.ML.StandardTrainers/Standard/MulticlassClassification/OneVersusAllTrainer.cs index 602497f630..2ae05af908 100644 --- a/src/Microsoft.ML.StandardTrainers/Standard/MulticlassClassification/OneVersusAllTrainer.cs +++ b/src/Microsoft.ML.StandardTrainers/Standard/MulticlassClassification/OneVersusAllTrainer.cs @@ -267,7 +267,7 @@ private static VersionInfo GetVersionInfo() /// /// Retrieves the model parameters. /// - private ImmutableArray SubModelParameters => _impl.Predictors.Cast().ToImmutableArray(); + internal ImmutableArray SubModelParameters => _impl.Predictors.Cast().ToImmutableArray(); /// /// The type of the prediction task. diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs index beaec1a3a3..bf8eba4ce7 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs @@ -6,13 +6,14 @@ using System.Collections.Generic; using System.Linq; using System.Threading; +using Microsoft.ML.Calibrators; using Microsoft.ML.Data; using Microsoft.ML.Internal.Utilities; -using Microsoft.ML.Trainers.LightGbm; using Microsoft.ML.RunTests; using Microsoft.ML.Runtime; using Microsoft.ML.TestFramework.Attributes; using Microsoft.ML.Trainers.FastTree; +using Microsoft.ML.Trainers.LightGbm; using Microsoft.ML.Transforms; using Xunit; @@ -64,6 +65,31 @@ public void LightGBMBinaryEstimator() Done(); } + /// + /// LightGBMBinaryTrainer CorrectSigmoid test + /// + [LightGBMFact] + public void LightGBMBinaryEstimatorCorrectSigmoid() + { + var (pipe, dataView) = GetBinaryClassificationPipeline(); + var sigmoid = .789; + + var trainer = ML.BinaryClassification.Trainers.LightGbm(new LightGbmBinaryTrainer.Options + { + NumberOfLeaves = 10, + NumberOfThreads = 1, + MinimumExampleCountPerLeaf = 2, + Sigmoid = sigmoid + }); + + var transformedDataView = pipe.Fit(dataView).Transform(dataView); + var model = trainer.Fit(transformedDataView, transformedDataView); + + // The slope in the model calibrator should be equal to the negative of the sigmoid passed into the trainer. + Assert.Equal(sigmoid, -model.Model.Calibrator.Slope); + Done(); + } + [Fact] public void GAMClassificationEstimator() @@ -251,6 +277,32 @@ public void LightGbmMulticlassEstimator() Done(); } + /// + /// LightGbmMulticlass CorrectSigmoid test + /// + [LightGBMFact] + public void LightGbmMulticlassEstimatorCorrectSigmoid() + { + var (pipeline, dataView) = GetMulticlassPipeline(); + var sigmoid = .789; + + var trainer = ML.MulticlassClassification.Trainers.LightGbm(new LightGbmMulticlassTrainer.Options + { + Sigmoid = sigmoid + }); + + var pipe = pipeline.Append(trainer) + .Append(new KeyToValueMappingEstimator(Env, "PredictedLabel")); + + var transformedDataView = pipe.Fit(dataView).Transform(dataView); + var model = trainer.Fit(transformedDataView, transformedDataView); + + // The slope in the all the calibrators should be equal to the negative of the sigmoid passed into the trainer. + Assert.True(model.Model.SubModelParameters.All(predictor => + ((FeatureWeightsCalibratedModelParameters)predictor).Calibrator.Slope == -sigmoid)); + Done(); + } + // Number of examples private const int _rowNumber = 1000; // Number of features @@ -267,7 +319,7 @@ private class GbmExample public float[] Score; } - private void LightGbmHelper(bool useSoftmax, out string modelString, out List mlnetPredictions, out double[] lgbmRawScores, out double[] lgbmProbabilities) + private void LightGbmHelper(bool useSoftmax, double sigmoid, out string modelString, out List mlnetPredictions, out double[] lgbmRawScores, out double[] lgbmProbabilities) { // Prepare data and train LightGBM model via ML.NET // Training matrix. It contains all feature vectors. @@ -300,7 +352,8 @@ private void LightGbmHelper(bool useSoftmax, out string modelString, out List mlnetPredictions, out double[] nativeResult1, out double[] nativeResult0); + LightGbmHelper(useSoftmax: false, sigmoid: sigmoidScale, out string modelString, out List mlnetPredictions, out double[] nativeResult1, out double[] nativeResult0); // The i-th predictor returned by LightGBM produces the raw score, denoted by z_i, of the i-th class. // Assume that we have n classes in total. The i-th class probability can be computed via // p_i = sigmoid(sigmoidScale * z_i) / (sigmoid(sigmoidScale * z_1) + ... + sigmoid(sigmoidScale * z_n)). Assert.True(modelString != null); - float sigmoidScale = 0.5f; // Constant used train LightGBM. See gbmParams["sigmoid"] in the helper function. // Compare native LightGBM's and ML.NET's LightGBM results example by example for (int i = 0; i < _rowNumber; ++i) { @@ -405,11 +459,87 @@ public void LightGbmMulticlassEstimatorCompareOva() Done(); } + /// + /// Test LightGBM's sigmoid parameter with a custom value. This test checks if ML.NET and LightGBM produce the same result. + /// + [LightGBMFact] + public void LightGbmMulticlassEstimatorCompareOvaUsingSigmoids() + { + var sigmoidScale = .790; + // Train ML.NET LightGBM and native LightGBM and apply the trained models to the training set. + LightGbmHelper(useSoftmax: false, sigmoid: sigmoidScale, out string modelString, out List mlnetPredictions, out double[] nativeResult1, out double[] nativeResult0); + + // The i-th predictor returned by LightGBM produces the raw score, denoted by z_i, of the i-th class. + // Assume that we have n classes in total. The i-th class probability can be computed via + // p_i = sigmoid(sigmoidScale * z_i) / (sigmoid(sigmoidScale * z_1) + ... + sigmoid(sigmoidScale * z_n)). + Assert.True(modelString != null); + + // Compare native LightGBM's and ML.NET's LightGBM results example by example + for (int i = 0; i < _rowNumber; ++i) + { + double sum = 0; + for (int j = 0; j < _classNumber; ++j) + { + Assert.Equal(nativeResult0[j + i * _classNumber], mlnetPredictions[i].Score[j], 6); + if (float.IsNaN((float)nativeResult1[j + i * _classNumber])) + continue; + sum += MathUtils.SigmoidSlow((float)sigmoidScale * (float)nativeResult1[j + i * _classNumber]); + } + for (int j = 0; j < _classNumber; ++j) + { + double prob = MathUtils.SigmoidSlow((float)sigmoidScale * (float)nativeResult1[j + i * _classNumber]); + Assert.Equal(prob / sum, mlnetPredictions[i].Score[j], 6); + } + } + + Done(); + } + + /// + /// Make sure different sigmoid parameters produce different scores. In this test, two LightGBM models are trained with two different sigmoid values. + /// + [LightGBMFact] + public void LightGbmMulticlassEstimatorCompareOvaUsingDifferentSigmoids() + { + // Run native implemenation twice, see that results are different with different sigmoid values. + var firstSigmoidScale = .790; + var secondSigmoidScale = .2; + + // Train native LightGBM with both sigmoid values and apply the trained models to the training set. + LightGbmHelper(useSoftmax: false, sigmoid: firstSigmoidScale, out string firstModelString, out List firstMlnetPredictions, out double[] firstNativeResult1, out double[] firstNativeResult0); + LightGbmHelper(useSoftmax: false, sigmoid: secondSigmoidScale, out string secondModelString, out List secondMlnetPredictions, out double[] secondNativeResult1, out double[] secondNativeResult0); + + // Compare native LightGBM's results when 2 different sigmoid values are used. + for (int i = 0; i < _rowNumber; ++i) + { + var areEqual = true; + for (int j = 0; j < _classNumber; ++j) + { + if (float.IsNaN((float)firstNativeResult1[j + i * _classNumber])) + continue; + if (float.IsNaN((float)secondNativeResult1[j + i * _classNumber])) + continue; + + // Testing to make sure that at least 1 value is different. This avoids false positives when values are 0 + // even for the same sigmoid value. + areEqual &= firstMlnetPredictions[i].Score[j].Equals(secondMlnetPredictions[i].Score[j]); + + // Testing that the native result is different before we apply the sigmoid. + Assert.NotEqual((float)firstNativeResult1[j + i * _classNumber], (float)secondNativeResult1[j + i * _classNumber], 6); + } + + // There should be at least 1 value that is different in the row. + Assert.False(areEqual); + } + + Done(); + } + [LightGBMFact] public void LightGbmMulticlassEstimatorCompareSoftMax() { // Train ML.NET LightGBM and native LightGBM and apply the trained models to the training set. - LightGbmHelper(useSoftmax: true, out string modelString, out List mlnetPredictions, out double[] nativeResult1, out double[] nativeResult0); + LightGbmHelper(useSoftmax: true, sigmoid: .5, out string modelString, out List mlnetPredictions, out double[] nativeResult1, out double[] nativeResult0); // The i-th predictor returned by LightGBM produces the raw score, denoted by z_i, of the i-th class. // Assume that we have n classes in total. The i-th class probability can be computed via