Pass fold index to cross validation metrics. (#575)

RowTag in metrics

Author: Ivanidzo4ka

src/Microsoft.ML/Models/BinaryClassificationMetrics.cs

@@ -7,6 +7,7 @@
 using Microsoft.ML.Runtime.Data;
 using System;
 using System.Collections.Generic;
+using static Microsoft.ML.Runtime.Data.MetricKinds;
 
 namespace Microsoft.ML.Models
 {
@@ -35,7 +36,7 @@ internal static List<BinaryClassificationMetrics> FromMetrics(IHostEnvironment e
             var confusionMatrices = ConfusionMatrix.Create(env, confusionMatrix).GetEnumerator();
 
             int index = 0;
-            foreach(var metric in metricsEnumerable)
+            foreach (var metric in metricsEnumerable)
             {
 
                 if (index++ >= confusionMatriceStartIndex && !confusionMatrices.MoveNext())
@@ -57,6 +58,7 @@ internal static List<BinaryClassificationMetrics> FromMetrics(IHostEnvironment e
                         Entropy = metric.Entropy,
                         F1Score = metric.F1Score,
                         Auprc = metric.Auprc,
+                        RowTag = metric.RowTag,
                         ConfusionMatrix = confusionMatrices.Current,
                     });
 
@@ -162,6 +164,12 @@ internal static List<BinaryClassificationMetrics> FromMetrics(IHostEnvironment e
         /// </summary>
         public ConfusionMatrix ConfusionMatrix { get; private set; }
 
+        /// <summary>
+        /// For cross-validation, this is equal to "Fold N" for per-fold metric rows, "Overall" for the average metrics and "STD" for standard deviation.
+        /// For non-CV scenarios, this is equal to null
+        /// </summary>
+        public string RowTag { get; private set; }
+
         /// <summary>
         /// This class contains the public fields necessary to deserialize from IDataView.
         /// </summary>
@@ -200,6 +208,9 @@ private sealed class SerializationClass
 
             [ColumnName(BinaryClassifierEvaluator.AuPrc)]
             public Double Auprc;
+
+            [ColumnName(ColumnNames.FoldIndex)]
+            public string RowTag;
 #pragma warning restore 649 // never assigned
         }
     }

src/Microsoft.ML/Models/ClassificationMetrics.cs

@@ -6,6 +6,7 @@
 using Microsoft.ML.Runtime.Api;
 using Microsoft.ML.Runtime.Data;
 using System.Collections.Generic;
+using static Microsoft.ML.Runtime.Data.MetricKinds;
 
 namespace Microsoft.ML.Models
 {
@@ -51,7 +52,8 @@ internal static List<ClassificationMetrics> FromMetrics(IHostEnvironment env, ID
                         LogLossReduction = metric.LogLossReduction,
                         TopKAccuracy = metric.TopKAccuracy,
                         PerClassLogLoss = metric.PerClassLogLoss,
-                        ConfusionMatrix = confusionMatrices.Current
+                        ConfusionMatrix = confusionMatrices.Current,
+                        RowTag = metric.RowTag,
                     });
 
             }
@@ -127,6 +129,12 @@ internal static List<ClassificationMetrics> FromMetrics(IHostEnvironment env, ID
         /// </remarks>
         public double[] PerClassLogLoss { get; private set; }
 
+        /// <summary>
+        /// For cross-validation, this is equal to "Fold N" for per-fold metric rows, "Overall" for the average metrics and "STD" for standard deviation.
+        /// For non-CV scenarios, this is equal to null
+        /// </summary>
+        public string RowTag { get; private set; }
+
         /// <summary>
         /// Gets the confusion matrix, or error matrix, of the classifier.
         /// </summary>
@@ -155,6 +163,9 @@ private sealed class SerializationClass
 
             [ColumnName(MultiClassClassifierEvaluator.PerClassLogLoss)]
             public double[] PerClassLogLoss;
+
+            [ColumnName(ColumnNames.FoldIndex)]
+            public string RowTag;
 #pragma warning restore 649 // never assigned
         }
     }

src/Microsoft.ML/Models/ClusterMetrics.cs

@@ -7,6 +7,7 @@
 using Microsoft.ML.Runtime.Data;
 using System;
 using System.Collections.Generic;
+using static Microsoft.ML.Runtime.Data.MetricKinds;
 
 namespace Microsoft.ML.Models
 {
@@ -38,6 +39,7 @@ internal static List<ClusterMetrics> FromOverallMetrics(IHostEnvironment env, ID
                     AvgMinScore = metric.AvgMinScore,
                     Nmi = metric.Nmi,
                     Dbi = metric.Dbi,
+                    RowTag = metric.RowTag,
                 });
             }
 
@@ -73,6 +75,12 @@ internal static List<ClusterMetrics> FromOverallMetrics(IHostEnvironment env, ID
         /// </remarks>
         public double AvgMinScore { get; private set; }
 
+        /// <summary>
+        /// For cross-validation, this is equal to "Fold N" for per-fold metric rows, "Overall" for the average metrics and "STD" for standard deviation.
+        /// For non-CV scenarios, this is equal to null
+        /// </summary>
+        public string RowTag { get; private set; }
+
         /// <summary>
         /// This class contains the public fields necessary to deserialize from IDataView.
         /// </summary>
@@ -88,6 +96,8 @@ private sealed class SerializationClass
             [ColumnName(Runtime.Data.ClusteringEvaluator.AvgMinScore)]
             public Double AvgMinScore;
 
+            [ColumnName(ColumnNames.FoldIndex)]
+            public string RowTag;
 #pragma warning restore 649 // never assigned
         }
     }

src/Microsoft.ML/Models/RegressionMetrics.cs

@@ -7,6 +7,7 @@
 using Microsoft.ML.Runtime.Data;
 using System;
 using System.Collections.Generic;
+using static Microsoft.ML.Runtime.Data.MetricKinds;
 
 namespace Microsoft.ML.Models
 {
@@ -40,6 +41,7 @@ internal static List<RegressionMetrics> FromOverallMetrics(IHostEnvironment env,
                     Rms = metric.Rms,
                     LossFn = metric.LossFn,
                     RSquared = metric.RSquared,
+                    RowTag = metric.RowTag,
                 });
             }
 
@@ -90,6 +92,12 @@ internal static List<RegressionMetrics> FromOverallMetrics(IHostEnvironment env,
         /// </summary>
         public double RSquared { get; private set; }
 
+        /// <summary>
+        /// For cross-validation, this is equal to "Fold N" for per-fold metric rows, "Overall" for the average metrics and "STD" for standard deviation.
+        /// For non-CV scenarios, this is equal to null
+        /// </summary>
+        public string RowTag { get; private set; }
+
         /// <summary>
         /// This class contains the public fields necessary to deserialize from IDataView.
         /// </summary>
@@ -110,6 +118,9 @@ private sealed class SerializationClass
 
             [ColumnName(Runtime.Data.RegressionEvaluator.RSquared)]
             public Double RSquared;
+
+            [ColumnName(ColumnNames.FoldIndex)]
+            public string RowTag;
 #pragma warning restore 649 // never assigned
         }
     }