Changes based on PR comments.

Author: michaelgsharp

src/Microsoft.ML.AutoML/API/RankingExperiment.cs

@@ -33,7 +33,7 @@ public sealed class RankingExperimentSettings : ExperimentSettings
         /// <value>
         /// The default value is 10.
         /// </value>
-        public int OptimizationMetricTruncationLevel { get; set; }
+        public uint OptimizationMetricTruncationLevel { get; set; }
 
         public RankingExperimentSettings()
         {
@@ -80,7 +80,7 @@ public static class RankingExperimentResultExtensions
         /// <param name="metric">Metric to consider when selecting the best run.</param>
         /// <param name="optimizationMetricTruncationLevel">Maximum truncation level for computing (N)DCG. Defaults to 10.</param>
         /// <returns>The best experiment run.</returns>
-        public static RunDetail<RankingMetrics> Best(this IEnumerable<RunDetail<RankingMetrics>> results, RankingMetric metric = RankingMetric.Ndcg, int optimizationMetricTruncationLevel = 10)
+        public static RunDetail<RankingMetrics> Best(this IEnumerable<RunDetail<RankingMetrics>> results, RankingMetric metric = RankingMetric.Ndcg, uint optimizationMetricTruncationLevel = 10)
         {
             var metricsAgent = new RankingMetricsAgent(null, metric, optimizationMetricTruncationLevel);
             var isMetricMaximizing = new OptimizingMetricInfo(metric).IsMaximizing;
@@ -94,7 +94,7 @@ public static RunDetail<RankingMetrics> Best(this IEnumerable<RunDetail<RankingM
         /// <param name="metric">Metric to consider when selecting the best run.</param>
         /// <param name="optimizationMetricTruncationLevel">Maximum truncation level for computing (N)DCG. Defaults to 10.</param>
         /// <returns>The best experiment run.</returns>
-        public static CrossValidationRunDetail<RankingMetrics> Best(this IEnumerable<CrossValidationRunDetail<RankingMetrics>> results, RankingMetric metric = RankingMetric.Ndcg, int optimizationMetricTruncationLevel = 10)
+        public static CrossValidationRunDetail<RankingMetrics> Best(this IEnumerable<CrossValidationRunDetail<RankingMetrics>> results, RankingMetric metric = RankingMetric.Ndcg, uint optimizationMetricTruncationLevel = 10)
         {
             var metricsAgent = new RankingMetricsAgent(null, metric, optimizationMetricTruncationLevel);
             var isMetricMaximizing = new OptimizingMetricInfo(metric).IsMaximizing;

src/Microsoft.ML.AutoML/Experiment/MetricsAgents/RankingMetricsAgent.cs

@@ -2,24 +2,29 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
+using System;
 using Microsoft.ML.Data;
+using Microsoft.ML.Runtime;
 
 namespace Microsoft.ML.AutoML
 {
     internal class RankingMetricsAgent : IMetricsAgent<RankingMetrics>
     {
         private readonly MLContext _mlContext;
         private readonly RankingMetric _optimizingMetric;
-        private readonly int _dcgTruncationLevel;
+        private readonly uint _dcgTruncationLevel;
 
-        public RankingMetricsAgent(MLContext mlContext, RankingMetric metric, int optimizationMetricTruncationLevel)
+        public RankingMetricsAgent(MLContext mlContext, RankingMetric metric, uint optimizationMetricTruncationLevel)
         {
             _mlContext = mlContext;
             _optimizingMetric = metric;
 
+            if (optimizationMetricTruncationLevel <= 0)
+                throw _mlContext.ExceptUserArg(nameof(optimizationMetricTruncationLevel), "DCG Truncation Level must be greater than 0");
+
             // We want to make sure we always have at least 10 results. Getting extra results adds no measurable performance
             // impact, so err on the side of more.
-            _dcgTruncationLevel = System.Math.Max(10, 2 * optimizationMetricTruncationLevel);
+            _dcgTruncationLevel = optimizationMetricTruncationLevel;
         }
 
         // Optimizing metric used: NDCG@10 and DCG@10
@@ -33,11 +38,9 @@ public double GetScore(RankingMetrics metrics)
             switch (_optimizingMetric)
             {
                 case RankingMetric.Ndcg:
-                    return (metrics.NormalizedDiscountedCumulativeGains.Count >= 10) ? metrics.NormalizedDiscountedCumulativeGains[9] :
-                        metrics.NormalizedDiscountedCumulativeGains[metrics.NormalizedDiscountedCumulativeGains.Count - 1];
+                    return metrics.NormalizedDiscountedCumulativeGains[Math.Min(metrics.NormalizedDiscountedCumulativeGains.Count, (int)_dcgTruncationLevel) - 1];
                 case RankingMetric.Dcg:
-                    return (metrics.DiscountedCumulativeGains.Count >= 10) ? metrics.DiscountedCumulativeGains[9] :
-                        metrics.DiscountedCumulativeGains[metrics.DiscountedCumulativeGains.Count-1];
+                    return metrics.DiscountedCumulativeGains[Math.Min(metrics.DiscountedCumulativeGains.Count, (int)_dcgTruncationLevel) - 1];
                 default:
                     throw MetricsAgentUtil.BuildMetricNotSupportedException(_optimizingMetric);
             }
@@ -66,7 +69,7 @@ public RankingMetrics EvaluateMetrics(IDataView data, string labelColumn, string
         {
             var rankingEvalOptions = new RankingEvaluatorOptions
             {
-                DcgTruncationLevel = _dcgTruncationLevel
+                DcgTruncationLevel = Math.Max(10, 2 * (int)_dcgTruncationLevel)
             };
 
             return _mlContext.Ranking.Evaluate(data, rankingEvalOptions, labelColumn, groupIdColumn);

src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs

@@ -35,7 +35,7 @@ public static RunDetail<MulticlassClassificationMetrics> GetBestRun(IEnumerable<
         }
 
         public static RunDetail<RankingMetrics> GetBestRun(IEnumerable<RunDetail<RankingMetrics>> results,
-            RankingMetric metric, int dcgTruncationLevel)
+            RankingMetric metric, uint dcgTruncationLevel)
         {
             var metricsAgent = new RankingMetricsAgent(null, metric, dcgTruncationLevel);
 

test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs

@@ -176,7 +176,7 @@ public void AutoFitRankingTest()
             var settings = new RankingExperimentSettings()
             {
                 MaxExperimentTimeInSeconds = 5,
-                OptimizationMetricTruncationLevel = 5
+                OptimizationMetricTruncationLevel = 3
             };
             var experiment = mlContext.Auto()
                 .CreateRankingExperiment(settings);
@@ -203,6 +203,7 @@ public void AutoFitRankingTest()
             for (int i = 0; i < experimentResults.Length; i++)
             {
                 RunDetail<RankingMetrics> bestRun = experimentResults[i].BestRun;
+                // The user requested 3, but we always return at least 10.
                 Assert.Equal(10, bestRun.ValidationMetrics.DiscountedCumulativeGains.Count);
                 Assert.Equal(10, bestRun.ValidationMetrics.NormalizedDiscountedCumulativeGains.Count);
                 Assert.True(experimentResults[i].RunDetails.Count() > 0);

test/Microsoft.ML.AutoML.Tests/MetricsAgentsTests.cs

@@ -179,7 +179,7 @@ private static double GetScore(RegressionMetrics metrics, RegressionMetric metri
             return new RegressionMetricsAgent(null, metric).GetScore(metrics);
         }
 
-        private static double GetScore(RankingMetrics metrics, RankingMetric metric, int dcgTruncationLevel)
+        private static double GetScore(RankingMetrics metrics, RankingMetric metric, uint dcgTruncationLevel)
         {
             return new RankingMetricsAgent(null, metric, dcgTruncationLevel).GetScore(metrics);
         }
@@ -202,7 +202,7 @@ private static bool IsPerfectModel(RegressionMetrics metrics, RegressionMetric m
             return IsPerfectModel(metricsAgent, metrics);
         }
 
-        private static bool IsPerfectModel(RankingMetrics metrics, RankingMetric metric, int dcgTruncationLevel)
+        private static bool IsPerfectModel(RankingMetrics metrics, RankingMetric metric, uint dcgTruncationLevel)
         {
             var metricsAgent = new RankingMetricsAgent(null, metric, dcgTruncationLevel);
             return IsPerfectModel(metricsAgent, metrics);