Commeeting the code

Breaking down the main file into 3 parts
Expanding the range for the numerical value checks, if that range is smaller than 0.0001, to help with fluctuation of tests across OS

Author: sfilipi

src/Microsoft.ML.StandardLearners/FactorizationMachine/FactorizationMachine.cs renamed to src/Microsoft.ML.StandardLearners/FactorizationMachine/FactorizationMachineTrainer.cs

@@ -20,60 +20,16 @@
     new[] { typeof(SignatureBinaryClassifierTrainer), typeof(SignatureTrainer) }, FieldAwareFactorizationMachineTrainer.UserName, FieldAwareFactorizationMachineTrainer.LoadName,
     FieldAwareFactorizationMachineTrainer.ShortName, DocName = "trainer/FactorizationMachine.md")]
 
-[assembly: LoadableClass(typeof(FieldAwareFactorizationMachinePredictor), null, typeof(SignatureLoadModel), "Field Aware Factorization Machine", FieldAwareFactorizationMachinePredictor.LoaderSignature)]
-
 [assembly: LoadableClass(typeof(void), typeof(FieldAwareFactorizationMachineTrainer), null, typeof(SignatureEntryPointModule), FieldAwareFactorizationMachineTrainer.LoadName)]
 
 namespace Microsoft.ML.Runtime.FactorizationMachine
 {
-    internal sealed class FieldAwareFactorizationMachineUtils
-    {
-        internal static int GetAlignedVectorLength(int length)
-        {
-            int res = length % 4;
-            if (res == 0)
-                return length;
-            else
-                return length + (4 - res);
-        }
-
-        internal static bool LoadOneExampleIntoBuffer(ValueGetter<VBuffer<float>>[] getters, VBuffer<float> featureBuffer, bool norm, ref int count,
-            int[] fieldIndexBuffer, int[] featureIndexBuffer, float[] featureValueBuffer)
-        {
-            count = 0;
-            float featureNorm = 0;
-            int bias = 0;
-            float annihilation = 0;
-            for (int f = 0; f < getters.Length; f++)
-            {
-                getters[f](ref featureBuffer);
-                foreach (var pair in featureBuffer.Items())
-                {
-                    fieldIndexBuffer[count] = f;
-                    featureIndexBuffer[count] = bias + pair.Key;
-                    featureValueBuffer[count] = pair.Value;
-                    featureNorm += pair.Value * pair.Value;
-                    annihilation += pair.Value - pair.Value;
-                    count++;
-                }
-                bias += featureBuffer.Length;
-            }
-            featureNorm = MathUtils.Sqrt(featureNorm);
-            if (norm)
-            {
-                for (int i = 0; i < count; i++)
-                    featureValueBuffer[i] /= featureNorm;
-            }
-            return FloatUtils.IsFinite(annihilation);
-        }
-    }
-
     /// <summary>
     /// Train a field-aware factorization machine using ADAGRAD (an advanced stochastic gradient method). See references below
     /// for details. This trainer is essentially faster the one introduced in [2] because of some implemtation tricks[3].
     /// [1] http://jmlr.org/papers/volume12/duchi11a/duchi11a.pdf
     /// [2] http://www.csie.ntu.edu.tw/~cjlin/papers/ffm.pdf
-    /// [3] fast-ffm.tex in FactorizationMachine project folder
+    /// [3] https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf
     /// </summary>
     public sealed class FieldAwareFactorizationMachineTrainer : TrainerBase<RoleMappedData, FieldAwareFactorizationMachinePredictor>,
         IIncrementalTrainer<RoleMappedData, FieldAwareFactorizationMachinePredictor>, IValidatingTrainer<RoleMappedData>,
@@ -327,6 +283,8 @@ private void TrainCore(IChannel ch, IProgressChannel pch, RoleMappedData data, R
             Func<int, bool> pred = c => fieldColumnIndexes.Contains(c) || c == data.Schema.Label.Index || (data.Schema.Weight != null && c == data.Schema.Weight.Index);
             InitializeTrainingState(fieldCount, totalFeatureCount, predictor, out float[] linearWeights,
                 out AlignedArray latentWeightsAligned, out float[] linearAccSqGrads, out AlignedArray latentAccSqGradsAligned);
+
+            // refer to Algorithm 3 in https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf
             while (iter++ < _numIterations)
             {
                 using (var cursor = data.Data.GetRowCursor(pred, rng))
@@ -358,9 +316,13 @@ private void TrainCore(IChannel ch, IProgressChannel pch, RoleMappedData data, R
                             badExampleCount++;
                             continue;
                         }
+
+                        // refer to Algorithm 1 in [3] https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf
                         FieldAwareFactorizationMachineInterface.CalculateIntermediateVariables(fieldCount, _latentDimAligned, count,
                             featureFieldBuffer, featureIndexBuffer, featureValueBuffer, linearWeights, latentWeightsAligned, latentSum, ref modelResponse);
                         var slope = CalculateLossSlope(label, modelResponse);
+
+                        // refer to Algorithm 2 in [3] https://github.com/wschin/fast-ffm/blob/master/fast-ffm.pdf
                         FieldAwareFactorizationMachineInterface.CalculateGradientAndUpdate(_lambdaLinear, _lambdaLatent, _learningRate, fieldCount, _latentDimAligned, weight, count,
                             featureFieldBuffer, featureIndexBuffer, featureValueBuffer, latentSum, slope, linearWeights, latentWeightsAligned, linearAccSqGrads, latentAccSqGradsAligned);
                         loss += weight * CalculateLoss(label, modelResponse);
@@ -453,259 +415,4 @@ public static CommonOutputs.BinaryClassificationOutput TrainBinary(IHostEnvironm
                 () => LearnerEntryPointsUtils.FindColumn(host, input.TrainingData.Schema, input.LabelColumn));
         }
     }
-
-    public sealed class FieldAwareFactorizationMachinePredictor : PredictorBase<float>, ISchemaBindableMapper, ICanSaveModel
-    {
-        public const string LoaderSignature = "FieldAwareFactMacPredict";
-        public override PredictionKind PredictionKind => PredictionKind.BinaryClassification;
-        private bool _norm;
-        internal int FieldCount { get; }
-        internal int FeatureCount { get; }
-        internal int LatentDim { get; }
-        internal int LatentDimAligned { get; }
-        private readonly float[] _linearWeights;
-        private readonly AlignedArray _latentWeightsAligned;
-
-        private static VersionInfo GetVersionInfo()
-        {
-            return new VersionInfo(
-                modelSignature: "FAFAMAPD",
-                verWrittenCur: 0x00010001,
-                verReadableCur: 0x00010001,
-                verWeCanReadBack: 0x00010001,
-                loaderSignature: LoaderSignature);
-        }
-
-        internal FieldAwareFactorizationMachinePredictor(IHostEnvironment env, bool norm, int fieldCount, int featureCount, int latentDim,
-            float[] linearWeights, AlignedArray latentWeightsAligned) : base(env, LoaderSignature)
-        {
-            Host.Assert(fieldCount > 0);
-            Host.Assert(featureCount > 0);
-            Host.Assert(latentDim > 0);
-            Host.Assert(Utils.Size(linearWeights) == featureCount);
-            LatentDimAligned = FieldAwareFactorizationMachineUtils.GetAlignedVectorLength(latentDim);
-            Host.Assert(latentWeightsAligned.Size == checked(featureCount * fieldCount * LatentDimAligned));
-
-            _norm = norm;
-            FieldCount = fieldCount;
-            FeatureCount = featureCount;
-            LatentDim = latentDim;
-            _linearWeights = linearWeights;
-            _latentWeightsAligned = latentWeightsAligned;
-        }
-
-        private FieldAwareFactorizationMachinePredictor(IHostEnvironment env, ModelLoadContext ctx) : base(env, LoaderSignature)
-        {
-            Host.AssertValue(ctx);
-
-            // *** Binary format ***
-            // bool: whether to normalize feature vectors
-            // int: number of fields
-            // int: number of features
-            // int: latent dimension
-            // float[]: linear coefficients
-            // float[]: latent representation of features
-
-            var norm = ctx.Reader.ReadBoolean();
-            var fieldCount = ctx.Reader.ReadInt32();
-            Host.CheckDecode(fieldCount > 0);
-            var featureCount = ctx.Reader.ReadInt32();
-            Host.CheckDecode(featureCount > 0);
-            var latentDim = ctx.Reader.ReadInt32();
-            Host.CheckDecode(latentDim > 0);
-            LatentDimAligned = FieldAwareFactorizationMachineUtils.GetAlignedVectorLength(latentDim);
-            Host.Check(checked(featureCount * fieldCount * LatentDimAligned) <= Utils.ArrayMaxSize, "Latent dimension too large");
-            var linearWeights = ctx.Reader.ReadFloatArray();
-            Host.CheckDecode(Utils.Size(linearWeights) == featureCount);
-            var latentWeights = ctx.Reader.ReadFloatArray();
-            Host.CheckDecode(Utils.Size(latentWeights) == featureCount * fieldCount * latentDim);
-
-            _norm = norm;
-            FieldCount = fieldCount;
-            FeatureCount = featureCount;
-            LatentDim = latentDim;
-            _linearWeights = linearWeights;
-            _latentWeightsAligned = new AlignedArray(FeatureCount * FieldCount * LatentDimAligned, 16);
-            for (int j = 0; j < FeatureCount; j++)
-            {
-                for (int f = 0; f < FieldCount; f++)
-                {
-                    int vBias = j * FieldCount * LatentDim + f * LatentDim;
-                    int vBiasAligned = j * FieldCount * LatentDimAligned + f * LatentDimAligned;
-                    for (int k = 0; k < LatentDimAligned; k++)
-                    {
-                        if (k < LatentDim)
-                            _latentWeightsAligned[vBiasAligned + k] = latentWeights[vBias + k];
-                        else
-                            _latentWeightsAligned[vBiasAligned + k] = 0;
-                    }
-                }
-            }
-        }
-
-        public static FieldAwareFactorizationMachinePredictor Create(IHostEnvironment env, ModelLoadContext ctx)
-        {
-            Contracts.CheckValue(env, nameof(env));
-            env.CheckValue(ctx, nameof(ctx));
-            ctx.CheckAtModel(GetVersionInfo());
-            return new FieldAwareFactorizationMachinePredictor(env, ctx);
-        }
-
-        protected override void SaveCore(ModelSaveContext ctx)
-        {
-            Host.AssertValue(ctx);
-            ctx.SetVersionInfo(GetVersionInfo());
-
-            // *** Binary format ***
-            // bool: whether to normalize feature vectors
-            // int: number of fields
-            // int: number of features
-            // int: latent dimension
-            // float[]: linear coefficients
-            // float[]: latent representation of features
-
-            Host.Assert(FieldCount > 0);
-            Host.Assert(FeatureCount > 0);
-            Host.Assert(LatentDim > 0);
-            Host.Assert(Utils.Size(_linearWeights) == FeatureCount);
-            Host.Assert(_latentWeightsAligned.Size == FeatureCount * FieldCount * LatentDimAligned);
-
-            ctx.Writer.Write(_norm);
-            ctx.Writer.Write(FieldCount);
-            ctx.Writer.Write(FeatureCount);
-            ctx.Writer.Write(LatentDim);
-            ctx.Writer.WriteFloatArray(_linearWeights);
-            float[] latentWeights = new float[FeatureCount * FieldCount * LatentDim];
-            for (int j = 0; j < FeatureCount; j++)
-            {
-                for (int f = 0; f < FieldCount; f++)
-                {
-                    int vBias = j * FieldCount * LatentDim + f * LatentDim;
-                    int vBiasAligned = j * FieldCount * LatentDimAligned + f * LatentDimAligned;
-                    for (int k = 0; k < LatentDim; k++)
-                        latentWeights[vBias + k] = _latentWeightsAligned[vBiasAligned + k];
-                }
-            }
-            ctx.Writer.WriteFloatArray(latentWeights);
-        }
-
-        internal float CalculateResponse(ValueGetter<VBuffer<float>>[] getters, VBuffer<float> featureBuffer,
-            int[] featureFieldBuffer, int[] featureIndexBuffer, float[] featureValueBuffer, AlignedArray latentSum)
-        {
-            int count = 0;
-            float modelResponse = 0;
-            FieldAwareFactorizationMachineUtils.LoadOneExampleIntoBuffer(getters, featureBuffer, _norm, ref count,
-                featureFieldBuffer, featureIndexBuffer, featureValueBuffer);
-            FieldAwareFactorizationMachineInterface.CalculateIntermediateVariables(FieldCount, LatentDimAligned, count,
-                featureFieldBuffer, featureIndexBuffer, featureValueBuffer, _linearWeights, _latentWeightsAligned, latentSum, ref modelResponse);
-            return modelResponse;
-        }
-
-        public ISchemaBoundMapper Bind(IHostEnvironment env, RoleMappedSchema schema)
-        {
-            return new FieldAwareFactorizationMachineScalarRowMapper(env, schema, new BinaryClassifierSchema(), this);
-        }
-
-        internal void CopyLinearWeightsTo(float[] linearWeights)
-        {
-            Host.AssertValue(_linearWeights);
-            Host.AssertValue(linearWeights);
-            Array.Copy(_linearWeights, linearWeights, _linearWeights.Length);
-        }
-
-        internal void CopyLatentWeightsTo(AlignedArray latentWeights)
-        {
-            Host.AssertValue(_latentWeightsAligned);
-            Host.AssertValue(latentWeights);
-            latentWeights.CopyFrom(_latentWeightsAligned);
-        }
-    }
-
-    internal sealed class FieldAwareFactorizationMachineScalarRowMapper : ISchemaBoundRowMapper
-    {
-        private readonly FieldAwareFactorizationMachinePredictor _pred;
-
-        public RoleMappedSchema InputSchema { get; }
-
-        public ISchema OutputSchema { get; }
-
-        public ISchemaBindableMapper Bindable => _pred;
-
-        private readonly ColumnInfo[] _columns;
-        private readonly List<int> _inputColumnIndexes;
-        private readonly IHostEnvironment _env;
-
-        public FieldAwareFactorizationMachineScalarRowMapper(IHostEnvironment env, RoleMappedSchema schema,
-            ISchema outputSchema, FieldAwareFactorizationMachinePredictor pred)
-        {
-            Contracts.AssertValue(env);
-            Contracts.AssertValue(schema);
-            Contracts.CheckParam(outputSchema.ColumnCount == 2, nameof(outputSchema));
-            Contracts.CheckParam(outputSchema.GetColumnType(0).IsNumber, nameof(outputSchema));
-            Contracts.CheckParam(outputSchema.GetColumnType(1).IsNumber, nameof(outputSchema));
-            Contracts.AssertValue(pred);
-
-            _env = env;
-            _columns = schema.GetColumns(RoleMappedSchema.ColumnRole.Feature).ToArray();
-            _pred = pred;
-
-            var inputFeatureColumns = _columns.Select(c => new KeyValuePair<RoleMappedSchema.ColumnRole, string>(RoleMappedSchema.ColumnRole.Feature, c.Name)).ToList();
-            InputSchema = RoleMappedSchema.Create(schema.Schema, inputFeatureColumns);
-            OutputSchema = outputSchema;
-
-            _inputColumnIndexes = new List<int>();
-            foreach (var kvp in inputFeatureColumns)
-            {
-                if (schema.Schema.TryGetColumnIndex(kvp.Value, out int index))
-                    _inputColumnIndexes.Add(index);
-            }
-        }
-
-        public IRow GetOutputRow(IRow input, Func<int, bool> predicate, out Action action)
-        {
-            var latentSum = new AlignedArray(_pred.FieldCount * _pred.FieldCount * _pred.LatentDimAligned, 16);
-            var featureBuffer = new VBuffer<float>();
-            var featureFieldBuffer = new int[_pred.FeatureCount];
-            var featureIndexBuffer = new int[_pred.FeatureCount];
-            var featureValueBuffer = new float[_pred.FeatureCount];
-            var inputGetters = new ValueGetter<VBuffer<float>>[_pred.FieldCount];
-            for (int f = 0; f < _pred.FieldCount; f++)
-                inputGetters[f] = input.GetGetter<VBuffer<float>>(_inputColumnIndexes[f]);
-
-            action = null;
-            var getters = new Delegate[2];
-            if (predicate(0))
-            {
-                ValueGetter<float> responseGetter = (ref float value) =>
-                {
-                    value = _pred.CalculateResponse(inputGetters, featureBuffer, featureFieldBuffer, featureIndexBuffer, featureValueBuffer, latentSum);
-                };
-                getters[0] = responseGetter;
-            }
-            if (predicate(1))
-            {
-                ValueGetter<float> probGetter = (ref float value) =>
-                {
-                    value = _pred.CalculateResponse(inputGetters, featureBuffer, featureFieldBuffer, featureIndexBuffer, featureValueBuffer, latentSum);
-                    value = MathUtils.SigmoidSlow(value);
-                };
-                getters[1] = probGetter;
-            }
-
-            return new SimpleRow(OutputSchema, input, getters);
-        }
-
-        public Func<int, bool> GetDependencies(Func<int, bool> predicate)
-        {
-            if (Enumerable.Range(0, OutputSchema.ColumnCount).Any(predicate))
-                return index => _inputColumnIndexes.Any(c => c == index);
-            else
-                return index => false;
-        }
-
-        public IEnumerable<KeyValuePair<RoleMappedSchema.ColumnRole, string>> GetInputColumnRoles()
-        {
-            return InputSchema.GetColumnRoles().Select(kvp => new KeyValuePair<RoleMappedSchema.ColumnRole, string>(kvp.Key, kvp.Value.Name));
-        }
-    }
 }