From 6de2abf871e10829f9fe14b7a4e2f72192edfb0f Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Wed, 5 Jun 2019 12:29:11 -0700 Subject: [PATCH 01/11] fixed Hardcoded sigmoid value. Let Microsoft.ML.Tests see internals of Microsoft.ML.StandardTrainers --- .../LightGbmBinaryTrainer.cs | 2 +- .../LightGbmMulticlassTrainer.cs | 2 +- .../Properties/AssemblyInfo.cs | 1 + .../OneVersusAllTrainer.cs | 32 +++++------ .../TrainerEstimators/TreeEstimators.cs | 53 +++++++++++++++++++ 5 files changed, 72 insertions(+), 18 deletions(-) diff --git a/src/Microsoft.ML.LightGbm/LightGbmBinaryTrainer.cs b/src/Microsoft.ML.LightGbm/LightGbmBinaryTrainer.cs index 861bbd1528..2efc892ae2 100644 --- a/src/Microsoft.ML.LightGbm/LightGbmBinaryTrainer.cs +++ b/src/Microsoft.ML.LightGbm/LightGbmBinaryTrainer.cs @@ -232,7 +232,7 @@ private protected override CalibratedModelParametersBase(Host, pred, cali); } diff --git a/src/Microsoft.ML.LightGbm/LightGbmMulticlassTrainer.cs b/src/Microsoft.ML.LightGbm/LightGbmMulticlassTrainer.cs index 2c0791fab5..40a0082fde 100644 --- a/src/Microsoft.ML.LightGbm/LightGbmMulticlassTrainer.cs +++ b/src/Microsoft.ML.LightGbm/LightGbmMulticlassTrainer.cs @@ -185,7 +185,7 @@ private protected override OneVersusAllModelParameters CreatePredictor() for (int i = 0; i < _tlcNumClass; ++i) { var pred = CreateBinaryPredictor(i, innerArgs); - var cali = new PlattCalibrator(Host, -0.5, 0); + var cali = new PlattCalibrator(Host, -LightGbmTrainerOptions.Sigmoid, 0); predictors[i] = new FeatureWeightsCalibratedModelParameters(Host, pred, cali); } string obj = (string)GetGbmParameters()["objective"]; diff --git a/src/Microsoft.ML.StandardTrainers/Properties/AssemblyInfo.cs b/src/Microsoft.ML.StandardTrainers/Properties/AssemblyInfo.cs index 8b2172b4f4..ecb343b55b 100644 --- a/src/Microsoft.ML.StandardTrainers/Properties/AssemblyInfo.cs +++ b/src/Microsoft.ML.StandardTrainers/Properties/AssemblyInfo.cs @@ -8,6 +8,7 @@ [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Ensemble" + PublicKey.Value)] [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.StaticPipe" + PublicKey.Value)] [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Core.Tests" + PublicKey.TestValue)] +[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Tests" + PublicKey.TestValue)] [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Predictor.Tests" + PublicKey.TestValue)] [assembly: InternalsVisibleTo(assemblyName: "RunTests" + InternalPublicKey.Value)] diff --git a/src/Microsoft.ML.StandardTrainers/Standard/MulticlassClassification/OneVersusAllTrainer.cs b/src/Microsoft.ML.StandardTrainers/Standard/MulticlassClassification/OneVersusAllTrainer.cs index 602497f630..5121f3c648 100644 --- a/src/Microsoft.ML.StandardTrainers/Standard/MulticlassClassification/OneVersusAllTrainer.cs +++ b/src/Microsoft.ML.StandardTrainers/Standard/MulticlassClassification/OneVersusAllTrainer.cs @@ -262,12 +262,12 @@ private static VersionInfo GetVersionInfo() private const string SubPredictorFmt = "SubPredictor_{0:000}"; - private readonly ImplBase _impl; + internal readonly ImplBase Impl; /// /// Retrieves the model parameters. /// - private ImmutableArray SubModelParameters => _impl.Predictors.Cast().ToImmutableArray(); + private ImmutableArray SubModelParameters => Impl.Predictors.Cast().ToImmutableArray(); /// /// The type of the prediction task. @@ -289,7 +289,7 @@ internal enum OutputFormula { Raw = 0, ProbabilityNormalization = 1, Softmax = 2 private DataViewType DistType { get; } - bool ICanSavePfa.CanSavePfa => _impl.CanSavePfa; + bool ICanSavePfa.CanSavePfa => Impl.CanSavePfa; [BestFriend] internal static OneVersusAllModelParameters Create(IHost host, OutputFormula outputFormula, TScalarPredictor[] predictors) @@ -356,8 +356,8 @@ private OneVersusAllModelParameters(IHostEnvironment env, ImplBase impl) Host.AssertValue(impl, nameof(impl)); Host.Assert(Utils.Size(impl.Predictors) > 0); - _impl = impl; - DistType = new VectorDataViewType(NumberDataViewType.Single, _impl.Predictors.Length); + Impl = impl; + DistType = new VectorDataViewType(NumberDataViewType.Single, Impl.Predictors.Length); } private OneVersusAllModelParameters(IHostEnvironment env, ModelLoadContext ctx) @@ -374,16 +374,16 @@ private OneVersusAllModelParameters(IHostEnvironment env, ModelLoadContext ctx) { var predictors = new IValueMapperDist[len]; LoadPredictors(Host, predictors, ctx); - _impl = new ImplDist(predictors); + Impl = new ImplDist(predictors); } else { var predictors = new TScalarPredictor[len]; LoadPredictors(Host, predictors, ctx); - _impl = new ImplRaw(predictors); + Impl = new ImplRaw(predictors); } - DistType = new VectorDataViewType(NumberDataViewType.Single, _impl.Predictors.Length); + DistType = new VectorDataViewType(NumberDataViewType.Single, Impl.Predictors.Length); } private static OneVersusAllModelParameters Create(IHostEnvironment env, ModelLoadContext ctx) @@ -406,12 +406,12 @@ private protected override void SaveCore(ModelSaveContext ctx) base.SaveCore(ctx); ctx.SetVersionInfo(GetVersionInfo()); - var preds = _impl.Predictors; + var preds = Impl.Predictors; // *** Binary format *** // bool: useDist // int: predictor count - ctx.Writer.WriteBoolByte(_impl is ImplDist); + ctx.Writer.WriteBoolByte(Impl is ImplDist); ctx.Writer.Write(preds.Length); // Save other streams. @@ -423,12 +423,12 @@ JToken ISingleCanSavePfa.SaveAsPfa(BoundPfaContext ctx, JToken input) { Host.CheckValue(ctx, nameof(ctx)); Host.CheckValue(input, nameof(input)); - return _impl.SaveAsPfa(ctx, input); + return Impl.SaveAsPfa(ctx, input); } DataViewType IValueMapper.InputType { - get { return _impl.InputType; } + get { return Impl.InputType; } } DataViewType IValueMapper.OutputType @@ -440,7 +440,7 @@ ValueMapper IValueMapper.GetMapper() Host.Check(typeof(TIn) == typeof(VBuffer)); Host.Check(typeof(TOut) == typeof(VBuffer)); - return (ValueMapper)(Delegate)_impl.GetMapper(); + return (ValueMapper)(Delegate)Impl.GetMapper(); } void ICanSaveInSourceCode.SaveAsCode(TextWriter writer, RoleMappedSchema schema) @@ -448,7 +448,7 @@ void ICanSaveInSourceCode.SaveAsCode(TextWriter writer, RoleMappedSchema schema) Host.CheckValue(writer, nameof(writer)); Host.CheckValue(schema, nameof(schema)); - var preds = _impl.Predictors; + var preds = Impl.Predictors; writer.WriteLine("double[] outputs = new double[{0}];", preds.Length); for (int i = 0; i < preds.Length; i++) @@ -468,7 +468,7 @@ void ICanSaveInTextFormat.SaveAsText(TextWriter writer, RoleMappedSchema schema) Host.CheckValue(writer, nameof(writer)); Host.CheckValue(schema, nameof(schema)); - var preds = _impl.Predictors; + var preds = Impl.Predictors; for (int i = 0; i < preds.Length; i++) { @@ -483,7 +483,7 @@ void ICanSaveInTextFormat.SaveAsText(TextWriter writer, RoleMappedSchema schema) } } - private abstract class ImplBase : ISingleCanSavePfa + internal abstract class ImplBase : ISingleCanSavePfa { public abstract DataViewType InputType { get; } public abstract IValueMapper[] Predictors { get; } diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs index beaec1a3a3..5e8ae2aeb4 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs @@ -15,6 +15,7 @@ using Microsoft.ML.Trainers.FastTree; using Microsoft.ML.Transforms; using Xunit; +using Microsoft.ML.Calibrators; namespace Microsoft.ML.Tests.TrainerEstimators { @@ -64,6 +65,31 @@ public void LightGBMBinaryEstimator() Done(); } + /// + /// LightGBMBinaryTrainer CorrectSigmoid test + /// + [LightGBMFact] + public void LightGBMBinaryEstimatorCorrectSigmoid() + { + var (pipe, dataView) = GetBinaryClassificationPipeline(); + var sigmoid = .789; + + var trainer = ML.BinaryClassification.Trainers.LightGbm(new LightGbmBinaryTrainer.Options + { + NumberOfLeaves = 10, + NumberOfThreads = 1, + MinimumExampleCountPerLeaf = 2, + Sigmoid = sigmoid + }); + + var transformedDataView = pipe.Fit(dataView).Transform(dataView); + var model = trainer.Fit(transformedDataView, transformedDataView); + + //the slope in the model calibrator should be equal to the negative of the sigmoid passed into the trainer + Assert.Equal(sigmoid, -model.Model.Calibrator.Slope); + Done(); + } + [Fact] public void GAMClassificationEstimator() @@ -251,6 +277,33 @@ public void LightGbmMulticlassEstimator() Done(); } + /// + /// LightGbmMulticlass CorrectSigmoid test + /// + [LightGBMFact] + public void LightGbmMulticlassEstimatorCorrectSigmoid() + { + var (pipeline, dataView) = GetMulticlassPipeline(); + var sigmoid = .789; + + var trainer = ML.MulticlassClassification.Trainers.LightGbm(new LightGbmMulticlassTrainer.Options + { + Sigmoid = sigmoid + }); + + var pipe = pipeline.Append(trainer) + .Append(new KeyToValueMappingEstimator(Env, "PredictedLabel")); + + var transformedDataView = pipe.Fit(dataView).Transform(dataView); + var model = trainer.Fit(transformedDataView, transformedDataView); + + //the slope in the all the calibrators should be equal to the negative of the sigmoid passed into the trainer + + Assert.True(model.Model.Impl.Predictors.All(predictor => + ((FeatureWeightsCalibratedModelParameters)predictor).Calibrator.Slope == -sigmoid)); + Done(); + } + // Number of examples private const int _rowNumber = 1000; // Number of features From d21eecf0f042a63a016055e6190cb0fd289af778 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Mon, 10 Jun 2019 10:42:03 -0700 Subject: [PATCH 02/11] Removed extra whitespace between comment and code --- test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs index 5e8ae2aeb4..a758f9bd46 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs @@ -298,7 +298,6 @@ public void LightGbmMulticlassEstimatorCorrectSigmoid() var model = trainer.Fit(transformedDataView, transformedDataView); //the slope in the all the calibrators should be equal to the negative of the sigmoid passed into the trainer - Assert.True(model.Model.Impl.Predictors.All(predictor => ((FeatureWeightsCalibratedModelParameters)predictor).Calibrator.Slope == -sigmoid)); Done(); From 7537b38610d823ab4c677b209df79729e533b411 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Mon, 10 Jun 2019 11:43:43 -0700 Subject: [PATCH 03/11] changed which parameter was internal --- .../OneVersusAllTrainer.cs | 30 +++++++++---------- .../TrainerEstimators/TreeEstimators.cs | 2 +- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/Microsoft.ML.StandardTrainers/Standard/MulticlassClassification/OneVersusAllTrainer.cs b/src/Microsoft.ML.StandardTrainers/Standard/MulticlassClassification/OneVersusAllTrainer.cs index 5121f3c648..a3dcbc350c 100644 --- a/src/Microsoft.ML.StandardTrainers/Standard/MulticlassClassification/OneVersusAllTrainer.cs +++ b/src/Microsoft.ML.StandardTrainers/Standard/MulticlassClassification/OneVersusAllTrainer.cs @@ -262,12 +262,12 @@ private static VersionInfo GetVersionInfo() private const string SubPredictorFmt = "SubPredictor_{0:000}"; - internal readonly ImplBase Impl; + private readonly ImplBase _impl; /// /// Retrieves the model parameters. /// - private ImmutableArray SubModelParameters => Impl.Predictors.Cast().ToImmutableArray(); + internal ImmutableArray SubModelParameters => _impl.Predictors.Cast().ToImmutableArray(); /// /// The type of the prediction task. @@ -289,7 +289,7 @@ internal enum OutputFormula { Raw = 0, ProbabilityNormalization = 1, Softmax = 2 private DataViewType DistType { get; } - bool ICanSavePfa.CanSavePfa => Impl.CanSavePfa; + bool ICanSavePfa.CanSavePfa => _impl.CanSavePfa; [BestFriend] internal static OneVersusAllModelParameters Create(IHost host, OutputFormula outputFormula, TScalarPredictor[] predictors) @@ -356,8 +356,8 @@ private OneVersusAllModelParameters(IHostEnvironment env, ImplBase impl) Host.AssertValue(impl, nameof(impl)); Host.Assert(Utils.Size(impl.Predictors) > 0); - Impl = impl; - DistType = new VectorDataViewType(NumberDataViewType.Single, Impl.Predictors.Length); + _impl = impl; + DistType = new VectorDataViewType(NumberDataViewType.Single, _impl.Predictors.Length); } private OneVersusAllModelParameters(IHostEnvironment env, ModelLoadContext ctx) @@ -374,16 +374,16 @@ private OneVersusAllModelParameters(IHostEnvironment env, ModelLoadContext ctx) { var predictors = new IValueMapperDist[len]; LoadPredictors(Host, predictors, ctx); - Impl = new ImplDist(predictors); + _impl = new ImplDist(predictors); } else { var predictors = new TScalarPredictor[len]; LoadPredictors(Host, predictors, ctx); - Impl = new ImplRaw(predictors); + _impl = new ImplRaw(predictors); } - DistType = new VectorDataViewType(NumberDataViewType.Single, Impl.Predictors.Length); + DistType = new VectorDataViewType(NumberDataViewType.Single, _impl.Predictors.Length); } private static OneVersusAllModelParameters Create(IHostEnvironment env, ModelLoadContext ctx) @@ -406,12 +406,12 @@ private protected override void SaveCore(ModelSaveContext ctx) base.SaveCore(ctx); ctx.SetVersionInfo(GetVersionInfo()); - var preds = Impl.Predictors; + var preds = _impl.Predictors; // *** Binary format *** // bool: useDist // int: predictor count - ctx.Writer.WriteBoolByte(Impl is ImplDist); + ctx.Writer.WriteBoolByte(_impl is ImplDist); ctx.Writer.Write(preds.Length); // Save other streams. @@ -423,12 +423,12 @@ JToken ISingleCanSavePfa.SaveAsPfa(BoundPfaContext ctx, JToken input) { Host.CheckValue(ctx, nameof(ctx)); Host.CheckValue(input, nameof(input)); - return Impl.SaveAsPfa(ctx, input); + return _impl.SaveAsPfa(ctx, input); } DataViewType IValueMapper.InputType { - get { return Impl.InputType; } + get { return _impl.InputType; } } DataViewType IValueMapper.OutputType @@ -440,7 +440,7 @@ ValueMapper IValueMapper.GetMapper() Host.Check(typeof(TIn) == typeof(VBuffer)); Host.Check(typeof(TOut) == typeof(VBuffer)); - return (ValueMapper)(Delegate)Impl.GetMapper(); + return (ValueMapper)(Delegate)_impl.GetMapper(); } void ICanSaveInSourceCode.SaveAsCode(TextWriter writer, RoleMappedSchema schema) @@ -448,7 +448,7 @@ void ICanSaveInSourceCode.SaveAsCode(TextWriter writer, RoleMappedSchema schema) Host.CheckValue(writer, nameof(writer)); Host.CheckValue(schema, nameof(schema)); - var preds = Impl.Predictors; + var preds = _impl.Predictors; writer.WriteLine("double[] outputs = new double[{0}];", preds.Length); for (int i = 0; i < preds.Length; i++) @@ -468,7 +468,7 @@ void ICanSaveInTextFormat.SaveAsText(TextWriter writer, RoleMappedSchema schema) Host.CheckValue(writer, nameof(writer)); Host.CheckValue(schema, nameof(schema)); - var preds = Impl.Predictors; + var preds = _impl.Predictors; for (int i = 0; i < preds.Length; i++) { diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs index a758f9bd46..f0b4b293cb 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs @@ -298,7 +298,7 @@ public void LightGbmMulticlassEstimatorCorrectSigmoid() var model = trainer.Fit(transformedDataView, transformedDataView); //the slope in the all the calibrators should be equal to the negative of the sigmoid passed into the trainer - Assert.True(model.Model.Impl.Predictors.All(predictor => + Assert.True(model.Model.SubModelParameters.All(predictor => ((FeatureWeightsCalibratedModelParameters)predictor).Calibrator.Slope == -sigmoid)); Done(); } From bdba6ea1b2cdbde03fbbdd26c2d24ccae7292541 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Mon, 10 Jun 2019 11:46:40 -0700 Subject: [PATCH 04/11] changed unneeded internal parameter back to private --- .../Standard/MulticlassClassification/OneVersusAllTrainer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ML.StandardTrainers/Standard/MulticlassClassification/OneVersusAllTrainer.cs b/src/Microsoft.ML.StandardTrainers/Standard/MulticlassClassification/OneVersusAllTrainer.cs index a3dcbc350c..2ae05af908 100644 --- a/src/Microsoft.ML.StandardTrainers/Standard/MulticlassClassification/OneVersusAllTrainer.cs +++ b/src/Microsoft.ML.StandardTrainers/Standard/MulticlassClassification/OneVersusAllTrainer.cs @@ -483,7 +483,7 @@ void ICanSaveInTextFormat.SaveAsText(TextWriter writer, RoleMappedSchema schema) } } - internal abstract class ImplBase : ISingleCanSavePfa + private abstract class ImplBase : ISingleCanSavePfa { public abstract DataViewType InputType { get; } public abstract IValueMapper[] Predictors { get; } From bbb076b9e425f5525e6df07dc19b35f76807cba9 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Mon, 10 Jun 2019 12:18:49 -0700 Subject: [PATCH 05/11] Put comments in correct format, sorted the using list --- test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs index f0b4b293cb..87d0ed9b9a 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs @@ -6,6 +6,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading; +using Microsoft.ML.Calibrators; using Microsoft.ML.Data; using Microsoft.ML.Internal.Utilities; using Microsoft.ML.Trainers.LightGbm; @@ -15,7 +16,6 @@ using Microsoft.ML.Trainers.FastTree; using Microsoft.ML.Transforms; using Xunit; -using Microsoft.ML.Calibrators; namespace Microsoft.ML.Tests.TrainerEstimators { @@ -85,7 +85,7 @@ public void LightGBMBinaryEstimatorCorrectSigmoid() var transformedDataView = pipe.Fit(dataView).Transform(dataView); var model = trainer.Fit(transformedDataView, transformedDataView); - //the slope in the model calibrator should be equal to the negative of the sigmoid passed into the trainer + // The slope in the model calibrator should be equal to the negative of the sigmoid passed into the trainer. Assert.Equal(sigmoid, -model.Model.Calibrator.Slope); Done(); } @@ -297,7 +297,7 @@ public void LightGbmMulticlassEstimatorCorrectSigmoid() var transformedDataView = pipe.Fit(dataView).Transform(dataView); var model = trainer.Fit(transformedDataView, transformedDataView); - //the slope in the all the calibrators should be equal to the negative of the sigmoid passed into the trainer + // The slope in the all the calibrators should be equal to the negative of the sigmoid passed into the trainer. Assert.True(model.Model.SubModelParameters.All(predictor => ((FeatureWeightsCalibratedModelParameters)predictor).Calibrator.Slope == -sigmoid)); Done(); From 7c48cf59c57c595fb89764fe4a8e794e2aca00a8 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Mon, 10 Jun 2019 16:54:15 -0700 Subject: [PATCH 06/11] Added direct tests between ML.NET and LightGBM with non-default sigmoid. --- .../TrainerEstimators/TreeEstimators.cs | 83 ++++++++++++++++++- 1 file changed, 80 insertions(+), 3 deletions(-) diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs index 87d0ed9b9a..013c06f8ff 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs @@ -9,11 +9,11 @@ using Microsoft.ML.Calibrators; using Microsoft.ML.Data; using Microsoft.ML.Internal.Utilities; -using Microsoft.ML.Trainers.LightGbm; using Microsoft.ML.RunTests; using Microsoft.ML.Runtime; using Microsoft.ML.TestFramework.Attributes; using Microsoft.ML.Trainers.FastTree; +using Microsoft.ML.Trainers.LightGbm; using Microsoft.ML.Transforms; using Xunit; @@ -319,7 +319,7 @@ private class GbmExample public float[] Score; } - private void LightGbmHelper(bool useSoftmax, out string modelString, out List mlnetPredictions, out double[] lgbmRawScores, out double[] lgbmProbabilities) + private void LightGbmHelper(bool useSoftmax, out string modelString, out List mlnetPredictions, out double[] lgbmRawScores, out double[] lgbmProbabilities, double sigmoid = .5) { // Prepare data and train LightGBM model via ML.NET // Training matrix. It contains all feature vectors. @@ -352,7 +352,8 @@ private void LightGbmHelper(bool useSoftmax, out string modelString, out List + /// LightGbmMulticlass UsingSigmoids test + /// + [LightGBMFact] + public void LightGbmMulticlassEstimatorCompareOvaUsingSigmoids() + { + var sigmoid = .790; + // Train ML.NET LightGBM and native LightGBM and apply the trained models to the training set. + LightGbmHelper(useSoftmax: false, out string modelString, out List mlnetPredictions, out double[] nativeResult1, out double[] nativeResult0, sigmoid: sigmoid); + + // The i-th predictor returned by LightGBM produces the raw score, denoted by z_i, of the i-th class. + // Assume that we have n classes in total. The i-th class probability can be computed via + // p_i = sigmoid(sigmoidScale * z_i) / (sigmoid(sigmoidScale * z_1) + ... + sigmoid(sigmoidScale * z_n)). + Assert.True(modelString != null); + + // Compare native LightGBM's and ML.NET's LightGBM results example by example + for (int i = 0; i < _rowNumber; ++i) + { + double sum = 0; + for (int j = 0; j < _classNumber; ++j) + { + Assert.Equal(nativeResult0[j + i * _classNumber], mlnetPredictions[i].Score[j], 6); + if (float.IsNaN((float)nativeResult1[j + i * _classNumber])) + continue; + sum += MathUtils.SigmoidSlow((float)sigmoid * (float)nativeResult1[j + i * _classNumber]); + } + for (int j = 0; j < _classNumber; ++j) + { + double prob = MathUtils.SigmoidSlow((float)sigmoid * (float)nativeResult1[j + i * _classNumber]); + Assert.Equal(prob / sum, mlnetPredictions[i].Score[j], 6); + } + } + + Done(); + } + + /// + /// LightGbmMulticlass UsingDifferentSigmoids test + /// + [LightGBMFact] + public void LightGbmMulticlassEstimatorCompareOvaUsingDifferentSigmoids() + { + // Run native implemenation twice, see that results are different with different sigmoid values. + var firstSigmoid = .790; + var secondSigmoid = .2; + + // Train native LightGBM with both sigmoid values and apply the trained models to the training set. + LightGbmHelper(useSoftmax: false, out string firstModelString, out List firstMlnetPredictions, out double[] firstNativeResult1, out double[] firstNativeResult0, sigmoid: firstSigmoid); + LightGbmHelper(useSoftmax: false, out string secondModelString, out List secondMlnetPredictions, out double[] secondNativeResult1, out double[] secondNativeResult0, sigmoid: secondSigmoid); + + // Compare native LightGBM's results when 2 different sigmoid values are used. + for (int i = 0; i < _rowNumber; ++i) + { + var areEqual = true; + for (int j = 0; j < _classNumber; ++j) + { + if (float.IsNaN((float)firstNativeResult1[j + i * _classNumber])) + continue; + if (float.IsNaN((float)secondNativeResult1[j + i * _classNumber])) + continue; + + // Testing to make sure that at least 1 value is different. This avoids false positives when values are 0 + // even for the same sigmoid value. + areEqual &= firstMlnetPredictions[i].Score[j].Equals(secondMlnetPredictions[i].Score[j]); + + // Testing that the native result is different before we apply the sigmoid. + Assert.NotEqual((float)firstNativeResult1[j + i * _classNumber], (float)secondNativeResult1[j + i * _classNumber], 6); + } + + // There should be at least 1 value that is different in the row. + Assert.False(areEqual); + } + + Done(); + } + [LightGBMFact] public void LightGbmMulticlassEstimatorCompareSoftMax() { From 56d674b38d0c2c318065befa270cabaa3550c3b2 Mon Sep 17 00:00:00 2001 From: Michael Sharp <51342856+michaelgsharp@users.noreply.github.com> Date: Tue, 11 Jun 2019 08:21:05 -0700 Subject: [PATCH 07/11] Update test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs Co-Authored-By: Wei-Sheng Chin --- test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs index 013c06f8ff..0a63e0a087 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs @@ -459,7 +459,7 @@ public void LightGbmMulticlassEstimatorCompareOva() } /// - /// LightGbmMulticlass UsingSigmoids test + /// Test LightGBM's sigmoid parameter with a custom value. This test checks if ML.NET and LightGBM produce the same result. /// [LightGBMFact] public void LightGbmMulticlassEstimatorCompareOvaUsingSigmoids() From e989921477ecd1e0d23409bb1176a4c4ad3d8621 Mon Sep 17 00:00:00 2001 From: Michael Sharp <51342856+michaelgsharp@users.noreply.github.com> Date: Tue, 11 Jun 2019 08:21:44 -0700 Subject: [PATCH 08/11] Update test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs Co-Authored-By: Wei-Sheng Chin --- test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs index 0a63e0a087..6ebdc38db3 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs @@ -353,7 +353,7 @@ private void LightGbmHelper(bool useSoftmax, out string modelString, out List Date: Tue, 11 Jun 2019 08:21:56 -0700 Subject: [PATCH 09/11] Update test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs Co-Authored-By: Wei-Sheng Chin --- test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs index 6ebdc38db3..f1ed191cf7 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs @@ -495,7 +495,7 @@ public void LightGbmMulticlassEstimatorCompareOvaUsingSigmoids() } /// - /// LightGbmMulticlass UsingDifferentSigmoids test + /// Make sure different sigmoid parameters produce different scores. In this test, two LightGBM models are trained with two different sigmoid values. /// [LightGBMFact] public void LightGbmMulticlassEstimatorCompareOvaUsingDifferentSigmoids() From cb43df3473ad29e93779c99b76feaf86a9c92721 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Tue, 11 Jun 2019 08:35:16 -0700 Subject: [PATCH 10/11] renamed sigmoid to sigmoidScale --- .../TrainerEstimators/TreeEstimators.cs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs index f1ed191cf7..6b2c8d1966 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs @@ -464,9 +464,9 @@ public void LightGbmMulticlassEstimatorCompareOva() [LightGBMFact] public void LightGbmMulticlassEstimatorCompareOvaUsingSigmoids() { - var sigmoid = .790; + var sigmoidScale = .790; // Train ML.NET LightGBM and native LightGBM and apply the trained models to the training set. - LightGbmHelper(useSoftmax: false, out string modelString, out List mlnetPredictions, out double[] nativeResult1, out double[] nativeResult0, sigmoid: sigmoid); + LightGbmHelper(useSoftmax: false, out string modelString, out List mlnetPredictions, out double[] nativeResult1, out double[] nativeResult0, sigmoid: sigmoidScale); // The i-th predictor returned by LightGBM produces the raw score, denoted by z_i, of the i-th class. // Assume that we have n classes in total. The i-th class probability can be computed via @@ -482,11 +482,11 @@ public void LightGbmMulticlassEstimatorCompareOvaUsingSigmoids() Assert.Equal(nativeResult0[j + i * _classNumber], mlnetPredictions[i].Score[j], 6); if (float.IsNaN((float)nativeResult1[j + i * _classNumber])) continue; - sum += MathUtils.SigmoidSlow((float)sigmoid * (float)nativeResult1[j + i * _classNumber]); + sum += MathUtils.SigmoidSlow((float)sigmoidScale * (float)nativeResult1[j + i * _classNumber]); } for (int j = 0; j < _classNumber; ++j) { - double prob = MathUtils.SigmoidSlow((float)sigmoid * (float)nativeResult1[j + i * _classNumber]); + double prob = MathUtils.SigmoidSlow((float)sigmoidScale * (float)nativeResult1[j + i * _classNumber]); Assert.Equal(prob / sum, mlnetPredictions[i].Score[j], 6); } } @@ -501,12 +501,12 @@ public void LightGbmMulticlassEstimatorCompareOvaUsingSigmoids() public void LightGbmMulticlassEstimatorCompareOvaUsingDifferentSigmoids() { // Run native implemenation twice, see that results are different with different sigmoid values. - var firstSigmoid = .790; - var secondSigmoid = .2; + var firstSigmoidScale = .790; + var secondSigmoidScale = .2; // Train native LightGBM with both sigmoid values and apply the trained models to the training set. - LightGbmHelper(useSoftmax: false, out string firstModelString, out List firstMlnetPredictions, out double[] firstNativeResult1, out double[] firstNativeResult0, sigmoid: firstSigmoid); - LightGbmHelper(useSoftmax: false, out string secondModelString, out List secondMlnetPredictions, out double[] secondNativeResult1, out double[] secondNativeResult0, sigmoid: secondSigmoid); + LightGbmHelper(useSoftmax: false, out string firstModelString, out List firstMlnetPredictions, out double[] firstNativeResult1, out double[] firstNativeResult0, sigmoid: firstSigmoidScale); + LightGbmHelper(useSoftmax: false, out string secondModelString, out List secondMlnetPredictions, out double[] secondNativeResult1, out double[] secondNativeResult0, sigmoid: secondSigmoidScale); // Compare native LightGBM's results when 2 different sigmoid values are used. for (int i = 0; i < _rowNumber; ++i) From 7461e4b67991252d557176eba9ad4ba5a65fe2b0 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Tue, 11 Jun 2019 10:04:55 -0700 Subject: [PATCH 11/11] changed sigmoid from a default parameter so it can come before the out params --- .../TrainerEstimators/TreeEstimators.cs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs index 6b2c8d1966..bf8eba4ce7 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs @@ -319,7 +319,7 @@ private class GbmExample public float[] Score; } - private void LightGbmHelper(bool useSoftmax, out string modelString, out List mlnetPredictions, out double[] lgbmRawScores, out double[] lgbmProbabilities, double sigmoid = .5) + private void LightGbmHelper(bool useSoftmax, double sigmoid, out string modelString, out List mlnetPredictions, out double[] lgbmRawScores, out double[] lgbmProbabilities) { // Prepare data and train LightGBM model via ML.NET // Training matrix. It contains all feature vectors. @@ -429,14 +429,15 @@ private void LightGbmHelper(bool useSoftmax, out string modelString, out List mlnetPredictions, out double[] nativeResult1, out double[] nativeResult0); + LightGbmHelper(useSoftmax: false, sigmoid: sigmoidScale, out string modelString, out List mlnetPredictions, out double[] nativeResult1, out double[] nativeResult0); // The i-th predictor returned by LightGBM produces the raw score, denoted by z_i, of the i-th class. // Assume that we have n classes in total. The i-th class probability can be computed via // p_i = sigmoid(sigmoidScale * z_i) / (sigmoid(sigmoidScale * z_1) + ... + sigmoid(sigmoidScale * z_n)). Assert.True(modelString != null); - float sigmoidScale = 0.5f; // Constant used train LightGBM. See gbmParams["sigmoid"] in the helper function. // Compare native LightGBM's and ML.NET's LightGBM results example by example for (int i = 0; i < _rowNumber; ++i) { @@ -466,7 +467,7 @@ public void LightGbmMulticlassEstimatorCompareOvaUsingSigmoids() { var sigmoidScale = .790; // Train ML.NET LightGBM and native LightGBM and apply the trained models to the training set. - LightGbmHelper(useSoftmax: false, out string modelString, out List mlnetPredictions, out double[] nativeResult1, out double[] nativeResult0, sigmoid: sigmoidScale); + LightGbmHelper(useSoftmax: false, sigmoid: sigmoidScale, out string modelString, out List mlnetPredictions, out double[] nativeResult1, out double[] nativeResult0); // The i-th predictor returned by LightGBM produces the raw score, denoted by z_i, of the i-th class. // Assume that we have n classes in total. The i-th class probability can be computed via @@ -505,8 +506,8 @@ public void LightGbmMulticlassEstimatorCompareOvaUsingDifferentSigmoids() var secondSigmoidScale = .2; // Train native LightGBM with both sigmoid values and apply the trained models to the training set. - LightGbmHelper(useSoftmax: false, out string firstModelString, out List firstMlnetPredictions, out double[] firstNativeResult1, out double[] firstNativeResult0, sigmoid: firstSigmoidScale); - LightGbmHelper(useSoftmax: false, out string secondModelString, out List secondMlnetPredictions, out double[] secondNativeResult1, out double[] secondNativeResult0, sigmoid: secondSigmoidScale); + LightGbmHelper(useSoftmax: false, sigmoid: firstSigmoidScale, out string firstModelString, out List firstMlnetPredictions, out double[] firstNativeResult1, out double[] firstNativeResult0); + LightGbmHelper(useSoftmax: false, sigmoid: secondSigmoidScale, out string secondModelString, out List secondMlnetPredictions, out double[] secondNativeResult1, out double[] secondNativeResult0); // Compare native LightGBM's results when 2 different sigmoid values are used. for (int i = 0; i < _rowNumber; ++i) @@ -538,7 +539,7 @@ public void LightGbmMulticlassEstimatorCompareOvaUsingDifferentSigmoids() public void LightGbmMulticlassEstimatorCompareSoftMax() { // Train ML.NET LightGBM and native LightGBM and apply the trained models to the training set. - LightGbmHelper(useSoftmax: true, out string modelString, out List mlnetPredictions, out double[] nativeResult1, out double[] nativeResult0); + LightGbmHelper(useSoftmax: true, sigmoid: .5, out string modelString, out List mlnetPredictions, out double[] nativeResult1, out double[] nativeResult0); // The i-th predictor returned by LightGBM produces the raw score, denoted by z_i, of the i-th class. // Assume that we have n classes in total. The i-th class probability can be computed via