Remove duplicate NormalizeFeatures from FFM trainer (#2964)

* Remove duplicate NormalizeFeatures from FFM trainer

* add comment

* set FFM feature normalization default to false

* by default normalization is off for FFM

* Hide base class NormalizeFeatures field

* hide base option class NormalizeField for FFM Options

Author: ganik

src/Microsoft.ML.EntryPoints/JsonUtils/JsonManifestUtils.cs

@@ -153,15 +153,24 @@ private static JArray BuildInputManifest(IExceptionContext ectx, Type inputType,
 
             // Instantiate a value of the input, to pull defaults out of.
             var defaults = Activator.CreateInstance(inputType);
-
+            var collectedFields = new HashSet<string>();
             var inputs = new List<KeyValuePair<Double, JObject>>();
             foreach (var fieldInfo in inputType.GetFields(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance))
             {
                 var inputAttr = fieldInfo.GetCustomAttributes(typeof(ArgumentAttribute), false).FirstOrDefault() as ArgumentAttribute;
                 if (inputAttr == null || inputAttr.Visibility == ArgumentAttribute.VisibilityType.CmdLineOnly)
                     continue;
+                var name = inputAttr.Name ?? fieldInfo.Name;
+                // The order of GetFields is stable, meaning that
+                // fields are always returned in the same order and for this reason
+                // unit tests to compare manifest are passing. For the same reason
+                // duplicate name skipped are always in the same correct order.
+                // Same name field can bubble up from base class even though
+                // its overidden / hidden, skip it.
+                if (collectedFields.Contains(name))
+                    continue;
                 var jo = new JObject();
-                jo[FieldNames.Name] = inputAttr.Name ?? fieldInfo.Name;
+                jo[FieldNames.Name] = name;
                 jo[FieldNames.Type] = BuildTypeToken(ectx, fieldInfo, fieldInfo.FieldType, catalog);
                 jo[FieldNames.Desc] = inputAttr.HelpText;
                 if (inputAttr.Aliases != null)
@@ -262,6 +271,7 @@ private static JArray BuildInputManifest(IExceptionContext ectx, Type inputType,
                 }
 
                 inputs.Add(new KeyValuePair<Double, JObject>(inputAttr.SortOrder, jo));
+                collectedFields.Add(name);
             }
             return new JArray(inputs.OrderBy(x => x.Key).Select(x => x.Value).ToArray());
         }

src/Microsoft.ML.StandardTrainers/FactorizationMachine/FactorizationMachineTrainer.cs

@@ -80,7 +80,7 @@ public sealed class Options : TrainerInputBaseWithWeight
             /// Whether to normalize the input vectors so that the concatenation of all fields' feature vectors is unit-length.
             /// </summary>
             [Argument(ArgumentType.AtMostOnce, HelpText = "Whether to normalize the input vectors so that the concatenation of all fields' feature vectors is unit-length", ShortName = "norm", SortOrder = 6)]
-            public bool Normalize = true;
+            public new bool NormalizeFeatures = true;
 
             /// <summary>
             /// Extra feature column names. The column named <see cref="TrainerInputBase.FeatureColumnName"/> stores features from the first field.
@@ -136,7 +136,7 @@ public sealed class Options : TrainerInputBaseWithWeight
         /// The <see cref="TrainerInfo"/> containing at least the training data for this trainer.
         /// </summary>
         TrainerInfo ITrainer.Info => _info;
-        private static readonly TrainerInfo _info = new TrainerInfo(supportValid: true, supportIncrementalTrain: true);
+        private static readonly TrainerInfo _info = new TrainerInfo(normalization: false, supportValid: true, supportIncrementalTrain: true);
 
         private int _latentDim;
         private int _latentDimAligned;
@@ -224,7 +224,7 @@ private void Initialize(IHostEnvironment env, Options options)
             _lambdaLatent = options.LambdaLatent;
             _learningRate = options.LearningRate;
             _numIterations = options.NumberOfIterations;
-            _norm = options.Normalize;
+            _norm = options.NormalizeFeatures;
             _shuffle = options.Shuffle;
             _verbose = options.Verbose;
             _radius = options.Radius;

test/BaselineOutput/Common/EntryPoints/core_manifest.json

@@ -10155,26 +10155,6 @@
         },
         {
           "Name": "NormalizeFeatures",
-          "Type": {
-            "Kind": "Enum",
-            "Values": [
-              "No",
-              "Warn",
-              "Auto",
-              "Yes"
-            ]
-          },
-          "Desc": "Normalize option for the feature column",
-          "Aliases": [
-            "norm"
-          ],
-          "Required": false,
-          "SortOrder": 5.0,
-          "IsNullable": false,
-          "Default": "Auto"
-        },
-        {
-          "Name": "Normalize",
           "Type": "Bool",
           "Desc": "Whether to normalize the input vectors so that the concatenation of all fields' feature vectors is unit-length",
           "Aliases": [