fix cancellation bug in SweepablePipelineRunner && fix object null exception in AutoML v1.0 regression API (#6560)

LittleLittleCloud · web-flow · commit fe2a4428c111 · 2023-02-15T21:33:11.000-08:00
* update * update * blocking run task * revert tests * add more logging * fix #6558 * update * fix tests * fix build * fix tests * fix build error * add logger * increase time * fix tests
diff --git a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs
@@ -447,7 +447,7 @@ public Task<TrialResult> RunAsync(TrialSettings settings, CancellationToken ct)
                     _context?.CancelExecution();
                 }))
                 {
-                    return Task.Run(() => Run(settings));
+                    return Task.FromResult(Run(settings));
                 }
             }
             catch (Exception ex) when (ct.IsCancellationRequested)
diff --git a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs
@@ -445,7 +445,7 @@ public Task<TrialResult> RunAsync(TrialSettings settings, CancellationToken ct)
                     _context?.CancelExecution();
                 }))
                 {
-                    return Task.Run(() => Run(settings));
+                    return Task.FromResult(Run(settings));
                 }
             }
             catch (Exception ex) when (ct.IsCancellationRequested)
diff --git a/src/Microsoft.ML.AutoML/API/RegressionExperiment.cs b/src/Microsoft.ML.AutoML/API/RegressionExperiment.cs
@@ -159,7 +159,7 @@ public override ExperimentResult<RegressionMetrics> Execute(IDataView trainData,
                 int numCrossValFolds = 10;
                 _experiment.SetDataset(trainData, numCrossValFolds);
                 _pipeline = CreateRegressionPipeline(trainData, columnInformation, preFeaturizer);
-
+                _experiment.SetPipeline(_pipeline);
                 TrialResultMonitor<RegressionMetrics> monitor = null;
                 _experiment.SetMonitor((provider) =>
                 {
diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs
@@ -51,15 +51,7 @@ private void InitializeServiceCollection()
             _serviceCollection.TryAddTransient((provider) =>
             {
                 var contextManager = provider.GetRequiredService<IMLContextManager>();
-                var trainingStopManager = provider.GetRequiredService<AggregateTrainingStopManager>();
                 var context = contextManager.CreateMLContext();
-                trainingStopManager.OnStopTraining += (s, e) =>
-                {
-                    // only force-canceling running trials when there's completed trials.
-                    // otherwise, wait for the current running trial to be completed.
-                    if (_bestTrialResult != null)
-                        context.CancelExecution();
-                };
 
                 return context;
             });
diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs
@@ -96,7 +96,7 @@ public Task<TrialResult> RunAsync(TrialSettings settings, CancellationToken ct)
                     _mLContext?.CancelExecution();
                 }))
                 {
-                    return Task.Run(() => Run(settings));
+                    return Task.FromResult(Run(settings));
                 }
             }
             catch (Exception ex) when (ct.IsCancellationRequested)
diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs
@@ -48,7 +48,7 @@ public void AutoFit_UCI_Adult_Test()
             var trainData = textLoader.Load(dataPath);
             var settings = new BinaryExperimentSettings
             {
-                MaxExperimentTimeInSeconds = 1,
+                MaxModels = 1,
             };
 
             settings.Trainers.Remove(BinaryClassificationTrainer.LightGbm);
@@ -75,7 +75,7 @@ public void AutoFit_UCI_Adult_Train_Test_Split_Test()
             var dataTrainTest = context.Data.TrainTestSplit(trainData);
             var settings = new BinaryExperimentSettings
             {
-                MaxExperimentTimeInSeconds = 1,
+                MaxModels = 1,
             };
 
             settings.Trainers.Remove(BinaryClassificationTrainer.LightGbm);
@@ -101,7 +101,7 @@ public void AutoFit_UCI_Adult_CrossValidation_10_Test()
             var trainData = textLoader.Load(dataPath);
             var settings = new BinaryExperimentSettings
             {
-                MaxExperimentTimeInSeconds = 1,
+                MaxModels = 1,
             };
 
             settings.Trainers.Remove(BinaryClassificationTrainer.LightGbm);
@@ -197,13 +197,23 @@ public void AutoFit_Taxi_Fare_Test()
             settings.Trainers.Remove(RegressionTrainer.StochasticDualCoordinateAscent);
             settings.Trainers.Remove(RegressionTrainer.LbfgsPoissonRegression);
 
+            // verify for dataset > 15000L
             var result = context.Auto()
                 .CreateRegressionExperiment(settings)
                 .Execute(dataset, label);
 
             Assert.True(result.BestRun.ValidationMetrics.RSquared > 0.70);
             Assert.NotNull(result.BestRun.Estimator);
             Assert.NotNull(result.BestRun.TrainerName);
+
+            // verify for dataset < 15000L
+            result = context.Auto()
+                .CreateRegressionExperiment(settings)
+                .Execute(context.Data.TakeRows(dataset, 1000), label);
+
+            Assert.True(result.BestRun.ValidationMetrics.RSquared > 0.70);
+            Assert.NotNull(result.BestRun.Estimator);
+            Assert.NotNull(result.BestRun.TrainerName);
         }
 
         [Theory]
@@ -229,7 +239,7 @@ public void AutoFitMultiTest(bool useNumberOfCVFolds)
                 uint numberOfCVFolds = 5;
                 var settings = new MulticlassExperimentSettings
                 {
-                    MaxExperimentTimeInSeconds = 1,
+                    MaxModels = 1,
                 };
 
                 settings.Trainers.Remove(MulticlassClassificationTrainer.LightGbm);
@@ -257,7 +267,7 @@ public void AutoFitMultiTest(bool useNumberOfCVFolds)
                 trainData = context.Data.TakeRows(trainData, crossValRowCountThreshold - 1);
                 var settings = new MulticlassExperimentSettings
                 {
-                    MaxExperimentTimeInSeconds = 1,
+                    MaxModels = 1,
                 };
 
                 settings.Trainers.Remove(MulticlassClassificationTrainer.LightGbm);
@@ -286,8 +296,13 @@ public void AutoFitMultiClassification_Image_TrainTest()
             TrainTestData trainTestData = context.Data.TrainTestSplit(trainData, testFraction: 0.2, seed: 1);
             IDataView trainDataset = SplitUtil.DropAllColumnsExcept(context, trainTestData.TrainSet, originalColumnNames);
             IDataView testDataset = SplitUtil.DropAllColumnsExcept(context, trainTestData.TestSet, originalColumnNames);
+            var settings = new MulticlassExperimentSettings
+            {
+                MaxModels = 1,
+            };
+
             var result = context.Auto()
-                            .CreateMulticlassClassificationExperiment(20)
+                            .CreateMulticlassClassificationExperiment(settings)
                             .Execute(trainDataset, testDataset, columnInference.ColumnInformation);
 
             result.BestRun.ValidationMetrics.MicroAccuracy.Should().BeGreaterThan(0.1);
@@ -305,8 +320,12 @@ public void AutoFitMultiClassification_Image_CV()
             var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions);
             var trainData = context.Data.ShuffleRows(textLoader.Load(datasetPath), seed: 1);
             var originalColumnNames = trainData.Schema.Select(c => c.Name);
+            var settings = new MulticlassExperimentSettings
+            {
+                MaxModels = 1,
+            };
             var result = context.Auto()
-                            .CreateMulticlassClassificationExperiment(100)
+                            .CreateMulticlassClassificationExperiment(settings)
                             .Execute(trainData, 5, columnInference.ColumnInformation);
 
             result.BestRun.Results.Select(x => x.ValidationMetrics.MicroAccuracy).Max().Should().BeGreaterThan(0.1);
@@ -330,8 +349,12 @@ public void AutoFitMultiClassification_Image()
             var columnInference = context.Auto().InferColumns(datasetPath, "Label");
             var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions);
             var trainData = textLoader.Load(datasetPath);
+            var settings = new MulticlassExperimentSettings
+            {
+                MaxModels = 1,
+            };
             var result = context.Auto()
-                            .CreateMulticlassClassificationExperiment(100)
+                            .CreateMulticlassClassificationExperiment(settings)
                             .Execute(trainData, columnInference.ColumnInformation);
 
             Assert.InRange(result.BestRun.ValidationMetrics.MicroAccuracy, 0.1, 0.9);
@@ -358,7 +381,7 @@ public void AutoFitRankingTest()
             // STEP 2: Run AutoML experiment
             var settings = new RankingExperimentSettings()
             {
-                MaxExperimentTimeInSeconds = 5,
+                MaxModels = 5,
                 OptimizationMetricTruncationLevel = 3
             };
             var experiment = mlContext.Auto()
diff --git a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs
@@ -151,14 +151,21 @@ public async Task AutoMLExperiment_return_current_best_trial_when_ct_is_canceled
         public async Task AutoMLExperiment_finish_training_when_time_is_up_Async()
         {
             var context = new MLContext(1);
+            context.Log += (o, e) =>
+            {
+                if (e.Source.StartsWith("AutoMLExperiment"))
+                {
+                    this.Output.WriteLine(e.RawMessage);
+                }
+            };
 
             var experiment = context.Auto().CreateExperiment();
             experiment.SetTrainingTimeInSeconds(5)
                       .SetTrialRunner((serviceProvider) =>
                       {
                           var channel = serviceProvider.GetService<IChannel>();
                           var settings = serviceProvider.GetService<AutoMLExperiment.AutoMLExperimentSettings>();
-                          return new DummyTrialRunner(settings, 1, channel);
+                          return new DummyTrialRunner(settings, 0, channel);
                       })
                       .SetTuner<RandomSearchTuner>();
 

Original file line number	Diff line number	Diff line change
`@@ -447,7 +447,7 @@ public Task<TrialResult> RunAsync(TrialSettings settings, CancellationToken ct)`
`447`	`447`	`_context?.CancelExecution();`
`448`	`448`	`}))`
`449`	`449`	`{`
`450`		`- return Task.Run(() => Run(settings));`
	`450`	`+ return Task.FromResult(Run(settings));`
`451`	`451`	`}`
`452`	`452`	`}`
`453`	`453`	`catch (Exception ex) when (ct.IsCancellationRequested)`
Original file line number	Diff line number	Diff line change
`@@ -445,7 +445,7 @@ public Task<TrialResult> RunAsync(TrialSettings settings, CancellationToken ct)`
`445`	`445`	`_context?.CancelExecution();`
`446`	`446`	`}))`
`447`	`447`	`{`
`448`		`- return Task.Run(() => Run(settings));`
	`448`	`+ return Task.FromResult(Run(settings));`
`449`	`449`	`}`
`450`	`450`	`}`
`451`	`451`	`catch (Exception ex) when (ct.IsCancellationRequested)`
Original file line number	Diff line number	Diff line change
`@@ -159,7 +159,7 @@ public override ExperimentResult<RegressionMetrics> Execute(IDataView trainData,`
`159`	`159`	`int numCrossValFolds = 10;`
`160`	`160`	`_experiment.SetDataset(trainData, numCrossValFolds);`
`161`	`161`	`_pipeline = CreateRegressionPipeline(trainData, columnInformation, preFeaturizer);`
`162`		`-`
	`162`	`+ _experiment.SetPipeline(_pipeline);`
`163`	`163`	`TrialResultMonitor<RegressionMetrics> monitor = null;`
`164`	`164`	`_experiment.SetMonitor((provider) =>`
`165`	`165`	`{`
Original file line number	Diff line number	Diff line change
`@@ -96,7 +96,7 @@ public Task<TrialResult> RunAsync(TrialSettings settings, CancellationToken ct)`
`96`	`96`	`_mLContext?.CancelExecution();`
`97`	`97`	`}))`
`98`	`98`	`{`
`99`		`- return Task.Run(() => Run(settings));`
	`99`	`+ return Task.FromResult(Run(settings));`
`100`	`100`	`}`
`101`	`101`	`}`
`102`	`102`	`catch (Exception ex) when (ct.IsCancellationRequested)`