Add test for linear svm and polish it's code (#1998)

Ivanidzo4ka · web-flow · commit 8a951c5b405e · 2019-01-02T16:01:01.000-08:00
diff --git a/src/Microsoft.ML.Ensemble/Trainer/Binary/EnsembleTrainer.cs b/src/Microsoft.ML.Ensemble/Trainer/Binary/EnsembleTrainer.cs
@@ -58,7 +58,7 @@ public Arguments()
                 BasePredictors = new[]
                 {
                     ComponentFactoryUtils.CreateFromFunction(
-                        env => new LinearSvm(env))
+                        env => new LinearSvmTrainer(env))
                 };
             }
         }
diff --git a/src/Microsoft.ML.StandardLearners/Standard/MultiClass/MetaMulticlassTrainer.cs b/src/Microsoft.ML.StandardLearners/Standard/MultiClass/MetaMulticlassTrainer.cs
@@ -87,7 +87,7 @@ private TScalarTrainer CreateTrainer()
         {
             return Args.PredictorType != null ?
                 Args.PredictorType.CreateComponent(Host) :
-                new LinearSvm(Host, new LinearSvm.Arguments());
+                new LinearSvmTrainer(Host, new LinearSvmTrainer.Arguments());
         }
 
         private protected IDataView MapLabelsCore<T>(ColumnType type, InPredicate<T> equalsTarget, RoleMappedData data)
diff --git a/src/Microsoft.ML.StandardLearners/Standard/Online/LinearSvm.cs b/src/Microsoft.ML.StandardLearners/Standard/Online/LinearSvm.cs
@@ -15,22 +15,21 @@
 using Microsoft.ML.Numeric;
 using Microsoft.ML.Trainers.Online;
 using Microsoft.ML.Training;
-using Float = System.Single;
 
-[assembly: LoadableClass(LinearSvm.Summary, typeof(LinearSvm), typeof(LinearSvm.Arguments),
+[assembly: LoadableClass(LinearSvmTrainer.Summary, typeof(LinearSvmTrainer), typeof(LinearSvmTrainer.Arguments),
     new[] { typeof(SignatureBinaryClassifierTrainer), typeof(SignatureTrainer), typeof(SignatureFeatureScorerTrainer) },
-    LinearSvm.UserNameValue,
-    LinearSvm.LoadNameValue,
-    LinearSvm.ShortName)]
+    LinearSvmTrainer.UserNameValue,
+    LinearSvmTrainer.LoadNameValue,
+    LinearSvmTrainer.ShortName)]
 
-[assembly: LoadableClass(typeof(void), typeof(LinearSvm), null, typeof(SignatureEntryPointModule), "LinearSvm")]
+[assembly: LoadableClass(typeof(void), typeof(LinearSvmTrainer), null, typeof(SignatureEntryPointModule), "LinearSvm")]
 
 namespace Microsoft.ML.Trainers.Online
 {
     /// <summary>
     /// Linear SVM that implements PEGASOS for training. See: http://ttic.uchicago.edu/~shai/papers/ShalevSiSr07.pdf
     /// </summary>
-    public sealed class LinearSvm : OnlineLinearTrainer<BinaryPredictionTransformer<LinearBinaryModelParameters>, LinearBinaryModelParameters>
+    public sealed class LinearSvmTrainer : OnlineLinearTrainer<BinaryPredictionTransformer<LinearBinaryModelParameters>, LinearBinaryModelParameters>
     {
         internal const string LoadNameValue = "LinearSVM";
         internal const string ShortName = "svm";
@@ -47,7 +46,7 @@ public sealed class Arguments : OnlineLinearArguments
             [Argument(ArgumentType.AtMostOnce, HelpText = "Regularizer constant", ShortName = "lambda", SortOrder = 50)]
             [TGUI(SuggestedSweeps = "0.00001-0.1;log;inc:10")]
             [TlcModule.SweepableFloatParamAttribute("Lambda", 0.00001f, 0.1f, 10, isLogScale: true)]
-            public Float Lambda = (Float)0.001;
+            public float Lambda = 0.001f;
 
             [Argument(ArgumentType.AtMostOnce, HelpText = "Batch size", ShortName = "batch", SortOrder = 190)]
             [TGUI(Label = "Batch Size")]
@@ -78,16 +77,16 @@ private sealed class TrainState : TrainStateBase
             // weightsUpdate/weightsUpdateScale/biasUpdate are similar to weights/weightsScale/bias, in that
             // all elements of weightsUpdate are considered to be multiplied by weightsUpdateScale, and the
             // bias update term is not considered to be multiplied by the scale.
-            private VBuffer<Float> _weightsUpdate;
-            private Float _weightsUpdateScale;
-            private Float _biasUpdate;
+            private VBuffer<float> _weightsUpdate;
+            private float _weightsUpdateScale;
+            private float _biasUpdate;
 
             private readonly int _batchSize;
             private readonly bool _noBias;
             private readonly bool _performProjection;
             private readonly float _lambda;
 
-            public TrainState(IChannel ch, int numFeatures, LinearModelParameters predictor, LinearSvm parent)
+            public TrainState(IChannel ch, int numFeatures, LinearModelParameters predictor, LinearSvmTrainer parent)
                 : base(ch, numFeatures, predictor, parent)
             {
                 _batchSize = parent.Args.BatchSize;
@@ -101,7 +100,7 @@ public TrainState(IChannel ch, int numFeatures, LinearModelParameters predictor,
                 if (predictor == null)
                     VBufferUtils.Densify(ref Weights);
 
-                _weightsUpdate = VBufferUtils.CreateEmpty<Float>(numFeatures);
+                _weightsUpdate = VBufferUtils.CreateEmpty<float>(numFeatures);
 
             }
 
@@ -119,7 +118,7 @@ private void BeginBatch()
                 VBufferUtils.Resize(ref _weightsUpdate, _weightsUpdate.Length, 0);
             }
 
-            private void FinishBatch(in VBuffer<Float> weightsUpdate, Float weightsUpdateScale)
+            private void FinishBatch(in VBuffer<float> weightsUpdate, float weightsUpdateScale)
             {
                 if (_numBatchExamples > 0)
                     UpdateWeights(in weightsUpdate, weightsUpdateScale);
@@ -129,19 +128,19 @@ private void FinishBatch(in VBuffer<Float> weightsUpdate, Float weightsUpdateSca
             /// <summary>
             /// Observe an example and update weights if necesary.
             /// </summary>
-            public override void ProcessDataInstance(IChannel ch, in VBuffer<Float> feat, Float label, Float weight)
+            public override void ProcessDataInstance(IChannel ch, in VBuffer<float> feat, float label, float weight)
             {
                 base.ProcessDataInstance(ch, in feat, label, weight);
 
                 // compute the update and update if needed
-                Float output = Margin(in feat);
-                Float trueOutput = (label > 0 ? 1 : -1);
-                Float loss = output * trueOutput - 1;
+                float output = Margin(in feat);
+                float trueOutput = (label > 0 ? 1 : -1);
+                float loss = output * trueOutput - 1;
 
                 // Accumulate the update if there is a loss and we have larger batches.
                 if (_batchSize > 1 && loss < 0)
                 {
-                    Float currentBiasUpdate = trueOutput * weight;
+                    float currentBiasUpdate = trueOutput * weight;
                     _biasUpdate += currentBiasUpdate;
                     // Only aggregate in the case where we're handling multiple instances.
                     if (_weightsUpdate.GetValues().Length == 0)
@@ -160,7 +159,7 @@ public override void ProcessDataInstance(IChannel ch, in VBuffer<Float> feat, Fl
                         Contracts.Assert(_weightsUpdate.GetValues().Length == 0);
                         // If we aren't aggregating multiple instances, just use the instance's
                         // vector directly.
-                        Float currentBiasUpdate = trueOutput * weight;
+                        float currentBiasUpdate = trueOutput * weight;
                         _biasUpdate += currentBiasUpdate;
                         FinishBatch(in feat, currentBiasUpdate);
                     }
@@ -174,13 +173,13 @@ public override void ProcessDataInstance(IChannel ch, in VBuffer<Float> feat, Fl
             /// Updates the weights at the end of the batch. Since weightsUpdate can be an instance
             /// feature vector, this function should not change the contents of weightsUpdate.
             /// </summary>
-            private void UpdateWeights(in VBuffer<Float> weightsUpdate, Float weightsUpdateScale)
+            private void UpdateWeights(in VBuffer<float> weightsUpdate, float weightsUpdateScale)
             {
                 Contracts.Assert(_batch > 0);
 
                 // REVIEW: This is really odd - normally lambda is small, so the learning rate is initially huge!?!?!
                 // Changed from the paper's recommended rate = 1 / (lambda * t) to rate = 1 / (1 + lambda * t).
-                Float rate = 1 / (1 + _lambda * _batch);
+                float rate = 1 / (1 + _lambda * _batch);
 
                 // w_{t+1/2} = (1 - eta*lambda) w_t + eta/k * totalUpdate
                 WeightsScale *= 1 - rate * _lambda;
@@ -194,7 +193,7 @@ private void UpdateWeights(in VBuffer<Float> weightsUpdate, Float weightsUpdateS
                 // w_{t+1} = min{1, 1/sqrt(lambda)/|w_{t+1/2}|} * w_{t+1/2}
                 if (_performProjection)
                 {
-                    Float normalizer = 1 / (MathUtils.Sqrt(_lambda) * VectorUtils.Norm(Weights) * Math.Abs(WeightsScale));
+                    float normalizer = 1 / (MathUtils.Sqrt(_lambda) * VectorUtils.Norm(Weights) * Math.Abs(WeightsScale));
                     if (normalizer < 1)
                     {
                         // REVIEW: Why would we not scale _bias if we're scaling the weights?
@@ -208,7 +207,7 @@ private void UpdateWeights(in VBuffer<Float> weightsUpdate, Float weightsUpdateS
             /// <summary>
             /// Return the raw margin from the decision hyperplane.
             /// </summary>
-            public override Float Margin(in VBuffer<Float> feat)
+            public override float Margin(in VBuffer<float> feat)
                 => Bias + VectorUtils.DotProduct(in feat, in Weights) * WeightsScale;
 
             public override LinearBinaryModelParameters CreatePredictor()
@@ -222,21 +221,21 @@ public override LinearBinaryModelParameters CreatePredictor()
         protected override bool NeedCalibration => true;
 
         /// <summary>
-        /// Initializes a new instance of <see cref="LinearSvm"/>.
+        /// Initializes a new instance of <see cref="LinearSvmTrainer"/>.
         /// </summary>
         /// <param name="env">The environment to use.</param>
         /// <param name="labelColumn">The name of the label column. </param>
         /// <param name="featureColumn">The name of the feature column.</param>
         /// <param name="weightsColumn">The optional name of the weights column.</param>
         /// <param name="numIterations">The number of training iteraitons.</param>
         /// <param name="advancedSettings">A delegate to supply more advanced arguments to the algorithm.</param>
-        public LinearSvm(IHostEnvironment env,
+        public LinearSvmTrainer(IHostEnvironment env,
             string labelColumn = DefaultColumnNames.Label,
             string featureColumn = DefaultColumnNames.Features,
             string weightsColumn = null,
             int numIterations = Arguments.OnlineDefaultArgs.NumIterations,
             Action<Arguments> advancedSettings = null)
-            :this(env, InvokeAdvanced(advancedSettings, new Arguments
+            : this(env, InvokeAdvanced(advancedSettings, new Arguments
             {
                 LabelColumn = labelColumn,
                 FeatureColumn = featureColumn,
@@ -246,8 +245,8 @@ public LinearSvm(IHostEnvironment env,
         {
         }
 
-        internal LinearSvm(IHostEnvironment env, Arguments args)
-            : base(args, env, UserNameValue, MakeLabelColumn(args.LabelColumn))
+        internal LinearSvmTrainer(IHostEnvironment env, Arguments args)
+            : base(args, env, UserNameValue, TrainerUtils.MakeBoolScalarLabel(args.LabelColumn))
         {
             Contracts.CheckUserArg(args.Lambda > 0, nameof(args.Lambda), UserErrorPositive);
             Contracts.CheckUserArg(args.BatchSize > 0, nameof(args.BatchSize), UserErrorPositive);
@@ -261,9 +260,8 @@ protected override SchemaShape.Column[] GetOutputColumnsCore(SchemaShape inputSc
         {
             return new[]
             {
-                new SchemaShape.Column(DefaultColumnNames.Score, SchemaShape.Column.VectorKind.Scalar, NumberType.R4, false),
-                new SchemaShape.Column(DefaultColumnNames.Probability, SchemaShape.Column.VectorKind.Scalar, NumberType.R4, false),
-                new SchemaShape.Column(DefaultColumnNames.PredictedLabel, SchemaShape.Column.VectorKind.Scalar, BoolType.Instance, false)
+                new SchemaShape.Column(DefaultColumnNames.Score, SchemaShape.Column.VectorKind.Scalar, NumberType.R4, false, new SchemaShape(MetadataUtils.GetTrainerOutputMetadata())),
+                new SchemaShape.Column(DefaultColumnNames.PredictedLabel, SchemaShape.Column.VectorKind.Scalar, BoolType.Instance, false, new SchemaShape(MetadataUtils.GetTrainerOutputMetadata()))
             };
         }
 
@@ -274,14 +272,7 @@ private protected override void CheckLabels(RoleMappedData data)
         }
 
         private protected override TrainStateBase MakeState(IChannel ch, int numFeatures, LinearModelParameters predictor)
-        {
-            return new TrainState(ch, numFeatures, predictor, this);
-        }
-
-        private static SchemaShape.Column MakeLabelColumn(string labelColumn)
-        {
-            return new SchemaShape.Column(labelColumn, SchemaShape.Column.VectorKind.Scalar, BoolType.Instance, false);
-        }
+            => new TrainState(ch, numFeatures, predictor, this);
 
         [TlcModule.EntryPoint(Name = "Trainers.LinearSvmBinaryClassifier", Desc = "Train a linear SVM.", UserName = UserNameValue, ShortName = ShortName)]
         public static CommonOutputs.BinaryClassificationOutput TrainLinearSvm(IHostEnvironment env, Arguments input)
@@ -292,12 +283,15 @@ public static CommonOutputs.BinaryClassificationOutput TrainLinearSvm(IHostEnvir
             EntryPointUtils.CheckInputArgs(host, input);
 
             return LearnerEntryPointsUtils.Train<Arguments, CommonOutputs.BinaryClassificationOutput>(host, input,
-                () => new LinearSvm(host, input),
+                () => new LinearSvmTrainer(host, input),
                 () => LearnerEntryPointsUtils.FindColumn(host, input.TrainingData.Schema, input.LabelColumn),
                 calibrator: input.Calibrator, maxCalibrationExamples: input.MaxCalibrationExamples);
         }
 
         protected override BinaryPredictionTransformer<LinearBinaryModelParameters> MakeTransformer(LinearBinaryModelParameters model, Schema trainSchema)
-        => new BinaryPredictionTransformer<LinearBinaryModelParameters>(Host, model, trainSchema, FeatureColumn.Name);
+            => new BinaryPredictionTransformer<LinearBinaryModelParameters>(Host, model, trainSchema, FeatureColumn.Name);
+
+        public BinaryPredictionTransformer<LinearBinaryModelParameters> Train(IDataView trainData, IPredictor initialPredictor = null)
+            => TrainTransformer(trainData, initPredictor: initialPredictor);
     }
 }
diff --git a/src/Microsoft.ML.StandardLearners/StandardLearnersCatalog.cs b/src/Microsoft.ML.StandardLearners/StandardLearnersCatalog.cs
@@ -384,7 +384,7 @@ public static Pkpd PairwiseCoupling(this MulticlassClassificationContext.Multicl
         }
 
         /// <summary>
-        /// Predict a target using a linear binary classification model trained with the <see cref="LinearSvm"/> trainer.
+        /// Predict a target using a linear binary classification model trained with the <see cref="LinearSvmTrainer"/> trainer.
         /// </summary>
         /// <remarks>
         /// <para>
@@ -403,15 +403,15 @@ public static Pkpd PairwiseCoupling(this MulticlassClassificationContext.Multicl
         /// <param name="weightsColumn">The optional name of the weights column.</param>
         /// <param name="numIterations">The number of training iteraitons.</param>
         /// <param name="advancedSettings">A delegate to supply more advanced arguments to the algorithm.</param>
-        public static LinearSvm LinearSupportVectorMachines(this BinaryClassificationContext.BinaryClassificationTrainers ctx,
+        public static LinearSvmTrainer LinearSupportVectorMachines(this BinaryClassificationContext.BinaryClassificationTrainers ctx,
             string labelColumn = DefaultColumnNames.Label,
             string featureColumn = DefaultColumnNames.Features,
             string weightsColumn = null,
             int numIterations = OnlineLinearArguments.OnlineDefaultArgs.NumIterations,
-            Action<LinearSvm.Arguments> advancedSettings = null)
+            Action<LinearSvmTrainer.Arguments> advancedSettings = null)
         {
             Contracts.CheckValue(ctx, nameof(ctx));
-            return new LinearSvm(CatalogUtils.GetEnvironment(ctx), labelColumn, featureColumn, weightsColumn, numIterations, advancedSettings);
+            return new LinearSvmTrainer(CatalogUtils.GetEnvironment(ctx), labelColumn, featureColumn, weightsColumn, numIterations, advancedSettings);
         }
     }
 }
diff --git a/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv b/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv
@@ -57,7 +57,7 @@ Trainers.LightGbmBinaryClassifier	Train a LightGBM binary classification model.
 Trainers.LightGbmClassifier	Train a LightGBM multi class model.	Microsoft.ML.LightGBM.LightGbm	TrainMultiClass	Microsoft.ML.LightGBM.LightGbmArguments	Microsoft.ML.EntryPoints.CommonOutputs+MulticlassClassificationOutput
 Trainers.LightGbmRanker	Train a LightGBM ranking model.	Microsoft.ML.LightGBM.LightGbm	TrainRanking	Microsoft.ML.LightGBM.LightGbmArguments	Microsoft.ML.EntryPoints.CommonOutputs+RankingOutput
 Trainers.LightGbmRegressor	LightGBM Regression	Microsoft.ML.LightGBM.LightGbm	TrainRegression	Microsoft.ML.LightGBM.LightGbmArguments	Microsoft.ML.EntryPoints.CommonOutputs+RegressionOutput
-Trainers.LinearSvmBinaryClassifier	Train a linear SVM.	Microsoft.ML.Trainers.Online.LinearSvm	TrainLinearSvm	Microsoft.ML.Trainers.Online.LinearSvm+Arguments	Microsoft.ML.EntryPoints.CommonOutputs+BinaryClassificationOutput
+Trainers.LinearSvmBinaryClassifier	Train a linear SVM.	Microsoft.ML.Trainers.Online.LinearSvmTrainer	TrainLinearSvm	Microsoft.ML.Trainers.Online.LinearSvmTrainer+Arguments	Microsoft.ML.EntryPoints.CommonOutputs+BinaryClassificationOutput
 Trainers.LogisticRegressionBinaryClassifier	Logistic Regression is a method in statistics used to predict the probability of occurrence of an event and can be used as a classification algorithm. The algorithm predicts the probability of occurrence of an event by fitting data to a logistical function.	Microsoft.ML.Learners.LogisticRegression	TrainBinary	Microsoft.ML.Learners.LogisticRegression+Arguments	Microsoft.ML.EntryPoints.CommonOutputs+BinaryClassificationOutput
 Trainers.LogisticRegressionClassifier	Logistic Regression is a method in statistics used to predict the probability of occurrence of an event and can be used as a classification algorithm. The algorithm predicts the probability of occurrence of an event by fitting data to a logistical function.	Microsoft.ML.Learners.LogisticRegression	TrainMultiClass	Microsoft.ML.Learners.MulticlassLogisticRegression+Arguments	Microsoft.ML.EntryPoints.CommonOutputs+MulticlassClassificationOutput
 Trainers.NaiveBayesClassifier	Train a MultiClassNaiveBayesTrainer.	Microsoft.ML.Trainers.MultiClassNaiveBayesTrainer	TrainMultiClassNaiveBayesTrainer	Microsoft.ML.Trainers.MultiClassNaiveBayesTrainer+Arguments	Microsoft.ML.EntryPoints.CommonOutputs+MulticlassClassificationOutput
diff --git a/test/Microsoft.ML.Tests/Scenarios/OvaTest.cs b/test/Microsoft.ML.Tests/Scenarios/OvaTest.cs
@@ -136,7 +136,7 @@ public void OvaLinearSvm()
             var data = mlContext.Data.Cache(reader.Read(GetDataPath(dataPath)));
 
             // Pipeline
-            var pipeline = new Ova(mlContext, new LinearSvm(mlContext, numIterations: 100),  useProbabilities: false);
+            var pipeline = new Ova(mlContext, new LinearSvmTrainer(mlContext, numIterations: 100),  useProbabilities: false);
 
             var model = pipeline.Fit(data);
             var predictions = model.Transform(data);
diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/OnlineLinearTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/OnlineLinearTests.cs

Original file line number	Diff line number	Diff line change
`@@ -58,7 +58,7 @@ public Arguments()`
`58`	`58`	`BasePredictors = new[]`
`59`	`59`	`{`
`60`	`60`	`ComponentFactoryUtils.CreateFromFunction(`
`61`		`- env => new LinearSvm(env))`
	`61`	`+ env => new LinearSvmTrainer(env))`
`62`	`62`	`};`
`63`	`63`	`}`
`64`	`64`	`}`
Original file line number	Diff line number	Diff line change
`@@ -87,7 +87,7 @@ private TScalarTrainer CreateTrainer()`
`87`	`87`	`{`
`88`	`88`	`return Args.PredictorType != null ?`
`89`	`89`	`Args.PredictorType.CreateComponent(Host) :`
`90`		`- new LinearSvm(Host, new LinearSvm.Arguments());`
	`90`	`+ new LinearSvmTrainer(Host, new LinearSvmTrainer.Arguments());`
`91`	`91`	`}`
`92`	`92`
`93`	`93`	`private protected IDataView MapLabelsCore<T>(ColumnType type, InPredicate<T> equalsTarget, RoleMappedData data)`
Original file line number	Diff line number	Diff line change
`@@ -384,7 +384,7 @@ public static Pkpd PairwiseCoupling(this MulticlassClassificationContext.Multicl`
`384`	`384`	`}`
`385`	`385`
`386`	`386`	`/// <summary>`
`387`		`- /// Predict a target using a linear binary classification model trained with the <see cref="LinearSvm"/> trainer.`
	`387`	`+ /// Predict a target using a linear binary classification model trained with the <see cref="LinearSvmTrainer"/> trainer.`
`388`	`388`	`/// </summary>`
`389`	`389`	`/// <remarks>`
`390`	`390`	`/// <para>`
`@@ -403,15 +403,15 @@ public static Pkpd PairwiseCoupling(this MulticlassClassificationContext.Multicl`
`403`	`403`	`/// <param name="weightsColumn">The optional name of the weights column.</param>`
`404`	`404`	`/// <param name="numIterations">The number of training iteraitons.</param>`
`405`	`405`	`/// <param name="advancedSettings">A delegate to supply more advanced arguments to the algorithm.</param>`
`406`		`- public static LinearSvm LinearSupportVectorMachines(this BinaryClassificationContext.BinaryClassificationTrainers ctx,`
	`406`	`+ public static LinearSvmTrainer LinearSupportVectorMachines(this BinaryClassificationContext.BinaryClassificationTrainers ctx,`
`407`	`407`	`string labelColumn = DefaultColumnNames.Label,`
`408`	`408`	`string featureColumn = DefaultColumnNames.Features,`
`409`	`409`	`string weightsColumn = null,`
`410`	`410`	`int numIterations = OnlineLinearArguments.OnlineDefaultArgs.NumIterations,`
`411`		`- Action<LinearSvm.Arguments> advancedSettings = null)`
	`411`	`+ Action<LinearSvmTrainer.Arguments> advancedSettings = null)`
`412`	`412`	`{`
`413`	`413`	`Contracts.CheckValue(ctx, nameof(ctx));`
`414`		`- return new LinearSvm(CatalogUtils.GetEnvironment(ctx), labelColumn, featureColumn, weightsColumn, numIterations, advancedSettings);`
	`414`	`+ return new LinearSvmTrainer(CatalogUtils.GetEnvironment(ctx), labelColumn, featureColumn, weightsColumn, numIterations, advancedSettings);`
`415`	`415`	`}`
`416`	`416`	`}`
`417`	`417`	`}`