From 7e0359258a4fbf73d73237b5a749aa61f0d7f873 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Thu, 9 Mar 2023 14:58:14 -0800 Subject: [PATCH 1/3] update max_model when trial fails --- .../AutoMLExperiment/AutoMLExperiment.cs | 27 +++++++------------ 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index 5836fc2c8b..da30f1cbce 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -282,9 +282,9 @@ void handler(object o, EventArgs e) } } } - catch (OperationCanceledException ex) when (aggregateTrainingStopManager.IsStopTrainingRequested() == false) + catch (Exception ex) when (aggregateTrainingStopManager.IsStopTrainingRequested() == false) { - logger.Trace($"trial cancelled - {JsonSerializer.Serialize(trialSettings)}, continue training"); + logger.Trace($"catch exception when training trial {trialSettings.TrialId} - {JsonSerializer.Serialize(trialSettings)}, continue training"); trialSettings.EndedAtUtc = DateTime.UtcNow; monitor?.ReportFailTrial(trialSettings, ex); var trialResult = new TrialResult @@ -296,22 +296,8 @@ void handler(object o, EventArgs e) tuner.Update(trialResult); trialResultManager?.AddOrUpdateTrialResult(trialResult); aggregateTrainingStopManager.Update(trialResult); - continue; - } - catch (OperationCanceledException) when (aggregateTrainingStopManager.IsStopTrainingRequested()) - { - logger.Trace($"trial cancelled - {JsonSerializer.Serialize(trialSettings)}, stop training"); - break; - } - catch (Exception ex) - { - logger.Trace($"trial failed - {JsonSerializer.Serialize(trialSettings)}, stop training"); - - trialSettings.EndedAtUtc = DateTime.UtcNow; - monitor?.ReportFailTrial(trialSettings, ex); - - if (!aggregateTrainingStopManager.IsStopTrainingRequested() && _bestTrialResult == null) + if (ex is not OperationCanceledException && _bestTrialResult == null) { logger.Trace($"trial fatal error - {JsonSerializer.Serialize(trialSettings)}, stop training"); @@ -321,6 +307,13 @@ void handler(object o, EventArgs e) // when error is fatal (like schema mismatch). throw; } + continue; + } + catch (Exception) when (aggregateTrainingStopManager.IsStopTrainingRequested()) + { + logger.Trace($"trial cancelled - {JsonSerializer.Serialize(trialSettings)}, stop training"); + + break; } finally { From b546c94c06c99aa53b544af8befbd8a45f73a4b7 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Tue, 21 Mar 2023 14:33:28 -0700 Subject: [PATCH 2/3] add nullable mark to cfo tuner --- src/Microsoft.ML.AutoML/Tuner/CostFrugalTuner.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ML.AutoML/Tuner/CostFrugalTuner.cs b/src/Microsoft.ML.AutoML/Tuner/CostFrugalTuner.cs index a7a7f855a8..e674949477 100644 --- a/src/Microsoft.ML.AutoML/Tuner/CostFrugalTuner.cs +++ b/src/Microsoft.ML.AutoML/Tuner/CostFrugalTuner.cs @@ -24,7 +24,7 @@ internal class CostFrugalTuner : ITuner private double _bestLoss; public CostFrugalTuner(AutoMLExperiment.AutoMLExperimentSettings settings, ITrialResultManager trialResultManager = null) - : this(settings.SearchSpace, settings.SearchSpace.SampleFromFeatureSpace(settings.SearchSpace.Default), trialResultManager.GetAllTrialResults(), settings.Seed) + : this(settings.SearchSpace, settings.SearchSpace.SampleFromFeatureSpace(settings.SearchSpace.Default), trialResultManager?.GetAllTrialResults(), settings.Seed) { } From ddf7953d9b98cacb6a6ecbcc270e7b2ea3de8029 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Thu, 23 Mar 2023 15:10:36 -0700 Subject: [PATCH 3/3] update exception message --- .../AutoMLExperiment/AutoMLExperiment.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index da30f1cbce..f31c832711 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -284,7 +284,14 @@ void handler(object o, EventArgs e) } catch (Exception ex) when (aggregateTrainingStopManager.IsStopTrainingRequested() == false) { - logger.Trace($"catch exception when training trial {trialSettings.TrialId} - {JsonSerializer.Serialize(trialSettings)}, continue training"); + var exceptionMessage = $@" +Exception thrown during Trial {trialSettings.TrialId} with configuration {JsonSerializer.Serialize(trialSettings)} + +Exception Details: ex.Message + +Abandoning Trial {trialSettings.TrialId} and continue training. +"; + logger.Trace(exceptionMessage); trialSettings.EndedAtUtc = DateTime.UtcNow; monitor?.ReportFailTrial(trialSettings, ex); var trialResult = new TrialResult