From 7fbc060423b624a2187d01a65215be6bd8ec6dc6 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Thu, 9 Feb 2023 17:13:16 -0800 Subject: [PATCH 01/14] update --- .../AutoMLExperimentTests.cs | 87 ++++++++++++++++++- .../Microsoft.ML.AutoML.Tests.csproj | 8 ++ 2 files changed, 92 insertions(+), 3 deletions(-) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs index 8e073394c6..bf0bdfc8d3 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs @@ -14,9 +14,11 @@ using Microsoft.Data.Analysis; using Microsoft.Extensions.DependencyInjection; using Microsoft.ML.AutoML.CodeGen; +using Microsoft.ML.Data; using Microsoft.ML.Runtime; using Microsoft.ML.TestFramework; using Microsoft.ML.TestFramework.Attributes; +using Microsoft.ML.TorchSharp; using Xunit; using Xunit.Abstractions; @@ -237,13 +239,14 @@ public async Task AutoMLExperiment_UCI_Adult_CV_5_Test() var experiment = context.Auto().CreateExperiment(); var pipeline = context.Auto().Featurizer(data, "_Features_", excludeColumns: new[] { DatasetUtil.UciAdultLabel }) .Append(context.Auto().BinaryClassification(DatasetUtil.UciAdultLabel, "_Features_", useLgbm: false, useSdcaLogisticRegression: false, useLbfgsLogisticRegression: false)); - + var cts = new CancellationTokenSource(); + cts.CancelAfter(50000); experiment.SetDataset(data, 5) .SetBinaryClassificationMetric(BinaryClassificationMetric.AreaUnderRocCurve, DatasetUtil.UciAdultLabel) .SetPipeline(pipeline) - .SetTrainingTimeInSeconds(10); + .SetTrainingTimeInSeconds(100); - var result = await experiment.RunAsync(); + var result = await experiment.RunAsync(cts.Token); result.Metric.Should().BeGreaterThan(0.8); } @@ -360,6 +363,84 @@ public void AutoMLExperiment_should_use_seed_from_context_if_provided() settings = experiment.ServiceCollection.BuildServiceProvider().GetRequiredService(); settings.Seed.Should().Be(1); } + + [Fact] + public async Task AutoMLExperiment_should_cancel_text_classification() + { + var context = new MLContext(1); + context.Log += (o, e) => + { + if (e.Message.Contains("AutoMLExperiment") || e.Message.Contains("NasBertTrainer")) + { + this.Output.WriteLine(e.RawMessage); + } + }; + var dataView = context.Data.LoadFromEnumerable( + new List(new TestSingleSentenceData[] { + new TestSingleSentenceData() + { // Testing longer than 512 words. + Sentence1 = "ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community . ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community . ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community . ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .", + Sentiment = "Negative" + }, + new TestSingleSentenceData() + { + Sentence1 = "with a sharp script and strong performances", + Sentiment = "Positive" + }, + new TestSingleSentenceData() + { + Sentence1 = "that director m. night shyamalan can weave an eerie spell and", + Sentiment = "Positive" + }, + new TestSingleSentenceData() + { + Sentence1 = "comfortable", + Sentiment = "Positive" + }, + new TestSingleSentenceData() + { + Sentence1 = "does have its charms .", + Sentiment = "Positive" + }, + new TestSingleSentenceData() + { + Sentence1 = "banal as the telling", + Sentiment = "Negative" + }, + new TestSingleSentenceData() + { + Sentence1 = "faithful without being forceful , sad without being shrill , `` a walk to remember '' succeeds through sincerity .", + Sentiment = "Negative" + }, + new TestSingleSentenceData() + { + Sentence1 = "leguizamo 's best movie work so far", + Sentiment = "Negative" + } + })); + + var experiment = context.Auto().CreateExperiment(); + var chain = new EstimatorChain(); + var pipeline = chain.Append(context.Transforms.Conversion.MapValueToKey("Label", "Sentiment")) + .Append(context.MulticlassClassification.Trainers.TextClassification(outputColumnName: "PredictedLabel", maxEpochs: 100)) + .Append(SweepableEstimatorFactory.CreateMapKeyToValue(new MapKeyToValueOption + { + InputColumnName = "PredictedLabel", + OutputColumnName = "PredictedLabel", + })); + experiment.SetPipeline(pipeline) + .SetDataset(dataView, dataView) + .SetMulticlassClassificationMetric(MulticlassClassificationMetric.MacroAccuracy, "Label") + .SetMaxModelToExplore(1); + var cts = new CancellationTokenSource(); + cts.CancelAfter(15_000); + await experiment.RunAsync(cts.Token); + } + } + class TestSingleSentenceData + { + public string Sentence1; + public string Sentiment; } class DummyTrialRunner : ITrialRunner diff --git a/test/Microsoft.ML.AutoML.Tests/Microsoft.ML.AutoML.Tests.csproj b/test/Microsoft.ML.AutoML.Tests/Microsoft.ML.AutoML.Tests.csproj index d47d425192..32a0bf11a9 100644 --- a/test/Microsoft.ML.AutoML.Tests/Microsoft.ML.AutoML.Tests.csproj +++ b/test/Microsoft.ML.AutoML.Tests/Microsoft.ML.AutoML.Tests.csproj @@ -9,6 +9,7 @@ + @@ -51,4 +52,11 @@ + + + + + + + \ No newline at end of file From 5313e5ebd5778029ddc91ee4397be37d946f9f00 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Thu, 9 Feb 2023 17:13:32 -0800 Subject: [PATCH 02/14] update --- test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs index bf0bdfc8d3..cc985bdaa7 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs @@ -422,7 +422,11 @@ public async Task AutoMLExperiment_should_cancel_text_classification() var experiment = context.Auto().CreateExperiment(); var chain = new EstimatorChain(); var pipeline = chain.Append(context.Transforms.Conversion.MapValueToKey("Label", "Sentiment")) - .Append(context.MulticlassClassification.Trainers.TextClassification(outputColumnName: "PredictedLabel", maxEpochs: 100)) + .Append(SweepableEstimatorFactory.CreateTextClassificationMulti(new TextClassificationOption + { + MaxEpochs = 100, + LabelColumnName = "Label", + })) .Append(SweepableEstimatorFactory.CreateMapKeyToValue(new MapKeyToValueOption { InputColumnName = "PredictedLabel", From a35f7e2e0b6c1c7a13cf4452a640b0634bafb5ea Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Thu, 9 Feb 2023 17:23:34 -0800 Subject: [PATCH 03/14] blocking run task --- .../AutoMLExperiment/Runner/SweepablePipelineRunner.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs index d42363bde0..ff3298c291 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs @@ -96,7 +96,7 @@ public Task RunAsync(TrialSettings settings, CancellationToken ct) _mLContext?.CancelExecution(); })) { - return Task.Run(() => Run(settings)); + return Task.FromResult(Run(settings)); } } catch (Exception ex) when (ct.IsCancellationRequested) From e7982f891d56aca02e77badf87161dfc86ea7432 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Fri, 10 Feb 2023 10:23:21 -0800 Subject: [PATCH 04/14] revert tests --- .../AutoMLExperimentTests.cs | 91 +------------------ .../Microsoft.ML.AutoML.Tests.csproj | 8 -- 2 files changed, 3 insertions(+), 96 deletions(-) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs index cc985bdaa7..8e073394c6 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs @@ -14,11 +14,9 @@ using Microsoft.Data.Analysis; using Microsoft.Extensions.DependencyInjection; using Microsoft.ML.AutoML.CodeGen; -using Microsoft.ML.Data; using Microsoft.ML.Runtime; using Microsoft.ML.TestFramework; using Microsoft.ML.TestFramework.Attributes; -using Microsoft.ML.TorchSharp; using Xunit; using Xunit.Abstractions; @@ -239,14 +237,13 @@ public async Task AutoMLExperiment_UCI_Adult_CV_5_Test() var experiment = context.Auto().CreateExperiment(); var pipeline = context.Auto().Featurizer(data, "_Features_", excludeColumns: new[] { DatasetUtil.UciAdultLabel }) .Append(context.Auto().BinaryClassification(DatasetUtil.UciAdultLabel, "_Features_", useLgbm: false, useSdcaLogisticRegression: false, useLbfgsLogisticRegression: false)); - var cts = new CancellationTokenSource(); - cts.CancelAfter(50000); + experiment.SetDataset(data, 5) .SetBinaryClassificationMetric(BinaryClassificationMetric.AreaUnderRocCurve, DatasetUtil.UciAdultLabel) .SetPipeline(pipeline) - .SetTrainingTimeInSeconds(100); + .SetTrainingTimeInSeconds(10); - var result = await experiment.RunAsync(cts.Token); + var result = await experiment.RunAsync(); result.Metric.Should().BeGreaterThan(0.8); } @@ -363,88 +360,6 @@ public void AutoMLExperiment_should_use_seed_from_context_if_provided() settings = experiment.ServiceCollection.BuildServiceProvider().GetRequiredService(); settings.Seed.Should().Be(1); } - - [Fact] - public async Task AutoMLExperiment_should_cancel_text_classification() - { - var context = new MLContext(1); - context.Log += (o, e) => - { - if (e.Message.Contains("AutoMLExperiment") || e.Message.Contains("NasBertTrainer")) - { - this.Output.WriteLine(e.RawMessage); - } - }; - var dataView = context.Data.LoadFromEnumerable( - new List(new TestSingleSentenceData[] { - new TestSingleSentenceData() - { // Testing longer than 512 words. - Sentence1 = "ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community . ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community . ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community . ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .ultimately feels as flat as the scruffy sands of its titular community .", - Sentiment = "Negative" - }, - new TestSingleSentenceData() - { - Sentence1 = "with a sharp script and strong performances", - Sentiment = "Positive" - }, - new TestSingleSentenceData() - { - Sentence1 = "that director m. night shyamalan can weave an eerie spell and", - Sentiment = "Positive" - }, - new TestSingleSentenceData() - { - Sentence1 = "comfortable", - Sentiment = "Positive" - }, - new TestSingleSentenceData() - { - Sentence1 = "does have its charms .", - Sentiment = "Positive" - }, - new TestSingleSentenceData() - { - Sentence1 = "banal as the telling", - Sentiment = "Negative" - }, - new TestSingleSentenceData() - { - Sentence1 = "faithful without being forceful , sad without being shrill , `` a walk to remember '' succeeds through sincerity .", - Sentiment = "Negative" - }, - new TestSingleSentenceData() - { - Sentence1 = "leguizamo 's best movie work so far", - Sentiment = "Negative" - } - })); - - var experiment = context.Auto().CreateExperiment(); - var chain = new EstimatorChain(); - var pipeline = chain.Append(context.Transforms.Conversion.MapValueToKey("Label", "Sentiment")) - .Append(SweepableEstimatorFactory.CreateTextClassificationMulti(new TextClassificationOption - { - MaxEpochs = 100, - LabelColumnName = "Label", - })) - .Append(SweepableEstimatorFactory.CreateMapKeyToValue(new MapKeyToValueOption - { - InputColumnName = "PredictedLabel", - OutputColumnName = "PredictedLabel", - })); - experiment.SetPipeline(pipeline) - .SetDataset(dataView, dataView) - .SetMulticlassClassificationMetric(MulticlassClassificationMetric.MacroAccuracy, "Label") - .SetMaxModelToExplore(1); - var cts = new CancellationTokenSource(); - cts.CancelAfter(15_000); - await experiment.RunAsync(cts.Token); - } - } - class TestSingleSentenceData - { - public string Sentence1; - public string Sentiment; } class DummyTrialRunner : ITrialRunner diff --git a/test/Microsoft.ML.AutoML.Tests/Microsoft.ML.AutoML.Tests.csproj b/test/Microsoft.ML.AutoML.Tests/Microsoft.ML.AutoML.Tests.csproj index 32a0bf11a9..d47d425192 100644 --- a/test/Microsoft.ML.AutoML.Tests/Microsoft.ML.AutoML.Tests.csproj +++ b/test/Microsoft.ML.AutoML.Tests/Microsoft.ML.AutoML.Tests.csproj @@ -9,7 +9,6 @@ - @@ -52,11 +51,4 @@ - - - - - - - \ No newline at end of file From b352cc3b5005cddbbfe8a58caa49c5c6784157a3 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Fri, 10 Feb 2023 14:19:21 -0800 Subject: [PATCH 05/14] add more logging --- .../AutoMLExperimentTests.cs | 155 +++++++++--------- 1 file changed, 81 insertions(+), 74 deletions(-) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs index 8e073394c6..ec10c6aa6d 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs @@ -14,6 +14,7 @@ using Microsoft.Data.Analysis; using Microsoft.Extensions.DependencyInjection; using Microsoft.ML.AutoML.CodeGen; +using Microsoft.ML.Data; using Microsoft.ML.Runtime; using Microsoft.ML.TestFramework; using Microsoft.ML.TestFramework.Attributes; @@ -151,7 +152,13 @@ public async Task AutoMLExperiment_return_current_best_trial_when_ct_is_canceled public async Task AutoMLExperiment_finish_training_when_time_is_up_Async() { var context = new MLContext(1); - + context.Log += (o, e) => + { + if (e.Source.StartsWith("AutoMLExperiment")) + { + this.Output.WriteLine(e.RawMessage); + } + }; var experiment = context.Auto().CreateExperiment(); experiment.SetTrainingTimeInSeconds(5) .SetTrialRunner((serviceProvider) => @@ -237,13 +244,14 @@ public async Task AutoMLExperiment_UCI_Adult_CV_5_Test() var experiment = context.Auto().CreateExperiment(); var pipeline = context.Auto().Featurizer(data, "_Features_", excludeColumns: new[] { DatasetUtil.UciAdultLabel }) .Append(context.Auto().BinaryClassification(DatasetUtil.UciAdultLabel, "_Features_", useLgbm: false, useSdcaLogisticRegression: false, useLbfgsLogisticRegression: false)); - + var cts = new CancellationTokenSource(); + cts.CancelAfter(50000); experiment.SetDataset(data, 5) .SetBinaryClassificationMetric(BinaryClassificationMetric.AreaUnderRocCurve, DatasetUtil.UciAdultLabel) .SetPipeline(pipeline) - .SetTrainingTimeInSeconds(10); + .SetTrainingTimeInSeconds(100); - var result = await experiment.RunAsync(); + var result = await experiment.RunAsync(cts.Token); result.Metric.Should().BeGreaterThan(0.8); } @@ -360,95 +368,94 @@ public void AutoMLExperiment_should_use_seed_from_context_if_provided() settings = experiment.ServiceCollection.BuildServiceProvider().GetRequiredService(); settings.Seed.Should().Be(1); } - } - - class DummyTrialRunner : ITrialRunner - { - private readonly int _finishAfterNSeconds; - private readonly IChannel _logger; - - public DummyTrialRunner(AutoMLExperiment.AutoMLExperimentSettings automlSettings, int finishAfterNSeconds, IChannel logger) - { - _finishAfterNSeconds = finishAfterNSeconds; - _logger = logger; - } - public void Dispose() + class DummyTrialRunner : ITrialRunner { - } + private readonly int _finishAfterNSeconds; + private readonly IChannel _logger; - public async Task RunAsync(TrialSettings settings, CancellationToken ct) - { - _logger.Info("Update Running Trial"); - await Task.Delay(_finishAfterNSeconds * 1000, ct); - ct.ThrowIfCancellationRequested(); - _logger.Info("Update Completed Trial"); - var metric = 1.000 + 0.01 * settings.TrialId; - return new TrialResult + public DummyTrialRunner(AutoMLExperiment.AutoMLExperimentSettings automlSettings, int finishAfterNSeconds, IChannel logger) { - TrialSettings = settings, - DurationInMilliseconds = _finishAfterNSeconds * 1000, - Metric = metric, - Loss = - -metric, - }; - } - } + _finishAfterNSeconds = finishAfterNSeconds; + _logger = logger; + } - class DummyPeformanceMonitor : IPerformanceMonitor - { - private readonly int _checkIntervalInMilliseconds; - private System.Timers.Timer _timer; + public void Dispose() + { + } - public DummyPeformanceMonitor() - { - _checkIntervalInMilliseconds = 1000; + public async Task RunAsync(TrialSettings settings, CancellationToken ct) + { + _logger.Info("Update Running Trial"); + await Task.Delay(_finishAfterNSeconds * 1000, ct); + ct.ThrowIfCancellationRequested(); + _logger.Info("Update Completed Trial"); + var metric = 1.000 + 0.01 * settings.TrialId; + return new TrialResult + { + TrialSettings = settings, + DurationInMilliseconds = _finishAfterNSeconds * 1000, + Metric = metric, + Loss = - -metric, + }; + } } + class DummyPeformanceMonitor : IPerformanceMonitor + { + private readonly int _checkIntervalInMilliseconds; + private System.Timers.Timer _timer; - public event EventHandler PerformanceMetricsUpdated; + public DummyPeformanceMonitor() + { + _checkIntervalInMilliseconds = 1000; + } - public void Dispose() - { - } + public event EventHandler PerformanceMetricsUpdated; - public double? GetPeakCpuUsage() - { - return 100; - } + public void Dispose() + { + } - public double? GetPeakMemoryUsageInMegaByte() - { - return 1000; - } + public double? GetPeakCpuUsage() + { + return 100; + } - public void OnPerformanceMetricsUpdatedHandler(TrialSettings trialSettings, TrialPerformanceMetrics metrics, CancellationTokenSource trialCancellationTokenSource) - { - } + public double? GetPeakMemoryUsageInMegaByte() + { + return 1000; + } - public void Start() - { - if (_timer == null) + public void OnPerformanceMetricsUpdatedHandler(TrialSettings trialSettings, TrialPerformanceMetrics metrics, CancellationTokenSource trialCancellationTokenSource) + { + } + + public void Start() { - _timer = new System.Timers.Timer(_checkIntervalInMilliseconds); - _timer.Elapsed += (o, e) => + if (_timer == null) { - PerformanceMetricsUpdated?.Invoke(this, new TrialPerformanceMetrics() { PeakCpuUsage = 100, PeakMemoryUsage = 1000 }); - }; + _timer = new System.Timers.Timer(_checkIntervalInMilliseconds); + _timer.Elapsed += (o, e) => + { + PerformanceMetricsUpdated?.Invoke(this, new TrialPerformanceMetrics() { PeakCpuUsage = 100, PeakMemoryUsage = 1000 }); + }; - _timer.AutoReset = true; + _timer.AutoReset = true; + } + _timer.Enabled = true; } - _timer.Enabled = true; - } - public void Pause() - { - _timer.Enabled = false; - } + public void Pause() + { + _timer.Enabled = false; + } - public void Stop() - { - _timer?.Stop(); - _timer?.Dispose(); - _timer = null; + public void Stop() + { + _timer?.Stop(); + _timer?.Dispose(); + _timer = null; + } } } } From a296e27ac50a7c9f31cee020eb6c5bbfc2370b6a Mon Sep 17 00:00:00 2001 From: LittleLittleCloud Date: Mon, 13 Feb 2023 11:10:13 -0800 Subject: [PATCH 06/14] fix #6558 --- src/Microsoft.ML.AutoML/API/RegressionExperiment.cs | 2 +- test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ML.AutoML/API/RegressionExperiment.cs b/src/Microsoft.ML.AutoML/API/RegressionExperiment.cs index 3d93a63b0a..e8ac5e405e 100644 --- a/src/Microsoft.ML.AutoML/API/RegressionExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/RegressionExperiment.cs @@ -159,7 +159,7 @@ public override ExperimentResult Execute(IDataView trainData, int numCrossValFolds = 10; _experiment.SetDataset(trainData, numCrossValFolds); _pipeline = CreateRegressionPipeline(trainData, columnInformation, preFeaturizer); - + _experiment.SetPipeline(_pipeline); TrialResultMonitor monitor = null; _experiment.SetMonitor((provider) => { diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index 432895c441..8b6219d9c0 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -197,6 +197,7 @@ public void AutoFit_Taxi_Fare_Test() settings.Trainers.Remove(RegressionTrainer.StochasticDualCoordinateAscent); settings.Trainers.Remove(RegressionTrainer.LbfgsPoissonRegression); + // verify for dataset > 15000L var result = context.Auto() .CreateRegressionExperiment(settings) .Execute(dataset, label); @@ -204,6 +205,15 @@ public void AutoFit_Taxi_Fare_Test() Assert.True(result.BestRun.ValidationMetrics.RSquared > 0.70); Assert.NotNull(result.BestRun.Estimator); Assert.NotNull(result.BestRun.TrainerName); + + // verify for dataset < 15000L + result = context.Auto() + .CreateRegressionExperiment(settings) + .Execute(context.Data.TakeRows(dataset, 1000), label); + + Assert.True(result.BestRun.ValidationMetrics.RSquared > 0.70); + Assert.NotNull(result.BestRun.Estimator); + Assert.NotNull(result.BestRun.TrainerName); } [Theory] From a9001299dead2ea6faa63604e41ffd4147c419b8 Mon Sep 17 00:00:00 2001 From: LittleLittleCloud Date: Mon, 13 Feb 2023 11:11:20 -0800 Subject: [PATCH 07/14] update --- src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs | 2 +- .../API/MulticlassClassificationExperiment.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs index 761aac2f4c..70db9035e5 100644 --- a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs @@ -447,7 +447,7 @@ public Task RunAsync(TrialSettings settings, CancellationToken ct) _context?.CancelExecution(); })) { - return Task.Run(() => Run(settings)); + return Task.FromResult(Run(settings)); } } catch (Exception ex) when (ct.IsCancellationRequested) diff --git a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs index b7344e7b0f..df96c28873 100644 --- a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs @@ -445,7 +445,7 @@ public Task RunAsync(TrialSettings settings, CancellationToken ct) _context?.CancelExecution(); })) { - return Task.Run(() => Run(settings)); + return Task.FromResult(Run(settings)); } } catch (Exception ex) when (ct.IsCancellationRequested) From f7744a4f73173aa562403cec7205666fa237a9d3 Mon Sep 17 00:00:00 2001 From: LittleLittleCloud Date: Mon, 13 Feb 2023 12:52:41 -0800 Subject: [PATCH 08/14] fix tests --- .../Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 27 ++- .../AutoMLExperimentTests.cs | 156 ++++++++---------- 2 files changed, 93 insertions(+), 90 deletions(-) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index 8b6219d9c0..576bf0335b 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -48,7 +48,7 @@ public void AutoFit_UCI_Adult_Test() var trainData = textLoader.Load(dataPath); var settings = new BinaryExperimentSettings { - MaxExperimentTimeInSeconds = 1, + MaxModels = 1, }; settings.Trainers.Remove(BinaryClassificationTrainer.LightGbm); @@ -101,7 +101,7 @@ public void AutoFit_UCI_Adult_CrossValidation_10_Test() var trainData = textLoader.Load(dataPath); var settings = new BinaryExperimentSettings { - MaxExperimentTimeInSeconds = 1, + MaxModels = 1, }; settings.Trainers.Remove(BinaryClassificationTrainer.LightGbm); @@ -239,7 +239,7 @@ public void AutoFitMultiTest(bool useNumberOfCVFolds) uint numberOfCVFolds = 5; var settings = new MulticlassExperimentSettings { - MaxExperimentTimeInSeconds = 1, + MaxModels = 1, }; settings.Trainers.Remove(MulticlassClassificationTrainer.LightGbm); @@ -296,8 +296,13 @@ public void AutoFitMultiClassification_Image_TrainTest() TrainTestData trainTestData = context.Data.TrainTestSplit(trainData, testFraction: 0.2, seed: 1); IDataView trainDataset = SplitUtil.DropAllColumnsExcept(context, trainTestData.TrainSet, originalColumnNames); IDataView testDataset = SplitUtil.DropAllColumnsExcept(context, trainTestData.TestSet, originalColumnNames); + var settings = new MulticlassExperimentSettings + { + MaxModels = 1, + }; + var result = context.Auto() - .CreateMulticlassClassificationExperiment(20) + .CreateMulticlassClassificationExperiment(settings) .Execute(trainDataset, testDataset, columnInference.ColumnInformation); result.BestRun.ValidationMetrics.MicroAccuracy.Should().BeGreaterThan(0.1); @@ -315,8 +320,12 @@ public void AutoFitMultiClassification_Image_CV() var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions); var trainData = context.Data.ShuffleRows(textLoader.Load(datasetPath), seed: 1); var originalColumnNames = trainData.Schema.Select(c => c.Name); + var settings = new MulticlassExperimentSettings + { + MaxModels = 1, + }; var result = context.Auto() - .CreateMulticlassClassificationExperiment(100) + .CreateMulticlassClassificationExperiment(settings) .Execute(trainData, 5, columnInference.ColumnInformation); result.BestRun.Results.Select(x => x.ValidationMetrics.MicroAccuracy).Max().Should().BeGreaterThan(0.1); @@ -340,8 +349,12 @@ public void AutoFitMultiClassification_Image() var columnInference = context.Auto().InferColumns(datasetPath, "Label"); var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions); var trainData = textLoader.Load(datasetPath); + var settings = new MulticlassExperimentSettings + { + MaxModels = 1, + }; var result = context.Auto() - .CreateMulticlassClassificationExperiment(100) + .CreateMulticlassClassificationExperiment(settings) .Execute(trainData, columnInference.ColumnInformation); Assert.InRange(result.BestRun.ValidationMetrics.MicroAccuracy, 0.1, 0.9); @@ -368,7 +381,7 @@ public void AutoFitRankingTest() // STEP 2: Run AutoML experiment var settings = new RankingExperimentSettings() { - MaxExperimentTimeInSeconds = 5, + MaxModels = 5, OptimizationMetricTruncationLevel = 3 }; var experiment = mlContext.Auto() diff --git a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs index ec10c6aa6d..18a0245718 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs @@ -14,7 +14,6 @@ using Microsoft.Data.Analysis; using Microsoft.Extensions.DependencyInjection; using Microsoft.ML.AutoML.CodeGen; -using Microsoft.ML.Data; using Microsoft.ML.Runtime; using Microsoft.ML.TestFramework; using Microsoft.ML.TestFramework.Attributes; @@ -74,7 +73,7 @@ public async Task AutoMLExperiment_cancel_trial_when_exceeds_memory_limit_Async( // the following experiment set memory usage limit to 0.01mb // so all trials should be canceled and there should be no successful trials. // therefore when experiment finishes, it should throw timeout exception with no model trained message. - experiment.SetMaxModelToExplore(10) + experiment.SetTrainingTimeInSeconds(10) .SetTrialRunner((serviceProvider) => { var channel = serviceProvider.GetService(); @@ -82,7 +81,8 @@ public async Task AutoMLExperiment_cancel_trial_when_exceeds_memory_limit_Async( return new DummyTrialRunner(settings, 5, channel); }) .SetTuner() - .SetMaximumMemoryUsageInMegaByte(0.01); + .SetMaximumMemoryUsageInMegaByte(0.01) + .SetPerformanceMonitor(); var runExperimentAction = async () => await experiment.RunAsync(); await runExperimentAction.Should().ThrowExactlyAsync(); @@ -152,13 +152,7 @@ public async Task AutoMLExperiment_return_current_best_trial_when_ct_is_canceled public async Task AutoMLExperiment_finish_training_when_time_is_up_Async() { var context = new MLContext(1); - context.Log += (o, e) => - { - if (e.Source.StartsWith("AutoMLExperiment")) - { - this.Output.WriteLine(e.RawMessage); - } - }; + var experiment = context.Auto().CreateExperiment(); experiment.SetTrainingTimeInSeconds(5) .SetTrialRunner((serviceProvider) => @@ -244,14 +238,13 @@ public async Task AutoMLExperiment_UCI_Adult_CV_5_Test() var experiment = context.Auto().CreateExperiment(); var pipeline = context.Auto().Featurizer(data, "_Features_", excludeColumns: new[] { DatasetUtil.UciAdultLabel }) .Append(context.Auto().BinaryClassification(DatasetUtil.UciAdultLabel, "_Features_", useLgbm: false, useSdcaLogisticRegression: false, useLbfgsLogisticRegression: false)); - var cts = new CancellationTokenSource(); - cts.CancelAfter(50000); + experiment.SetDataset(data, 5) .SetBinaryClassificationMetric(BinaryClassificationMetric.AreaUnderRocCurve, DatasetUtil.UciAdultLabel) .SetPipeline(pipeline) - .SetTrainingTimeInSeconds(100); + .SetTrainingTimeInSeconds(10); - var result = await experiment.RunAsync(cts.Token); + var result = await experiment.RunAsync(); result.Metric.Should().BeGreaterThan(0.8); } @@ -368,94 +361,91 @@ public void AutoMLExperiment_should_use_seed_from_context_if_provided() settings = experiment.ServiceCollection.BuildServiceProvider().GetRequiredService(); settings.Seed.Should().Be(1); } + } - class DummyTrialRunner : ITrialRunner - { - private readonly int _finishAfterNSeconds; - private readonly IChannel _logger; + class DummyTrialRunner : ITrialRunner + { + private readonly int _finishAfterNSeconds; + private readonly CancellationToken _ct; + private readonly IChannel _logger; - public DummyTrialRunner(AutoMLExperiment.AutoMLExperimentSettings automlSettings, int finishAfterNSeconds, IChannel logger) - { - _finishAfterNSeconds = finishAfterNSeconds; - _logger = logger; - } + public DummyTrialRunner(AutoMLExperiment.AutoMLExperimentSettings automlSettings, int finishAfterNSeconds, IChannel logger) + { + _finishAfterNSeconds = finishAfterNSeconds; + _ct = automlSettings.CancellationToken; + _logger = logger; + } - public void Dispose() - { - } + public void Dispose() + { + } - public async Task RunAsync(TrialSettings settings, CancellationToken ct) + public async Task RunAsync(TrialSettings settings, CancellationToken ct) + { + _logger.Info("Update Running Trial"); + await Task.Delay(_finishAfterNSeconds * 1000, ct); + _ct.ThrowIfCancellationRequested(); + _logger.Info("Update Completed Trial"); + var metric = 1.000 + 0.01 * settings.TrialId; + return new TrialResult { - _logger.Info("Update Running Trial"); - await Task.Delay(_finishAfterNSeconds * 1000, ct); - ct.ThrowIfCancellationRequested(); - _logger.Info("Update Completed Trial"); - var metric = 1.000 + 0.01 * settings.TrialId; - return new TrialResult - { - TrialSettings = settings, - DurationInMilliseconds = _finishAfterNSeconds * 1000, - Metric = metric, - Loss = - -metric, - }; - } + TrialSettings = settings, + DurationInMilliseconds = _finishAfterNSeconds * 1000, + Metric = metric, + Loss = - -metric, + }; } - class DummyPeformanceMonitor : IPerformanceMonitor - { - private readonly int _checkIntervalInMilliseconds; - private System.Timers.Timer _timer; + } - public DummyPeformanceMonitor() - { - _checkIntervalInMilliseconds = 1000; - } + class DummyPeformanceMonitor : IPerformanceMonitor + { + private readonly int _checkIntervalInMilliseconds; + private System.Timers.Timer _timer; - public event EventHandler PerformanceMetricsUpdated; + public DummyPeformanceMonitor() + { + _checkIntervalInMilliseconds = 1000; + } - public void Dispose() - { - } + public event EventHandler CpuUsage; - public double? GetPeakCpuUsage() - { - return 100; - } + public event EventHandler MemoryUsageInMegaByte; - public double? GetPeakMemoryUsageInMegaByte() - { - return 1000; - } + public void Dispose() + { + } - public void OnPerformanceMetricsUpdatedHandler(TrialSettings trialSettings, TrialPerformanceMetrics metrics, CancellationTokenSource trialCancellationTokenSource) - { - } + public double? GetPeakCpuUsage() + { + return 100; + } + + public double? GetPeakMemoryUsageInMegaByte() + { + return 1000; + } - public void Start() + public void Start() + { + if (_timer == null) { - if (_timer == null) + _timer = new System.Timers.Timer(_checkIntervalInMilliseconds); + _timer.Elapsed += (o, e) => { - _timer = new System.Timers.Timer(_checkIntervalInMilliseconds); - _timer.Elapsed += (o, e) => - { - PerformanceMetricsUpdated?.Invoke(this, new TrialPerformanceMetrics() { PeakCpuUsage = 100, PeakMemoryUsage = 1000 }); - }; + CpuUsage?.Invoke(this, 100); + MemoryUsageInMegaByte?.Invoke(this, 1000); + }; - _timer.AutoReset = true; - } + _timer.AutoReset = true; _timer.Enabled = true; } + } - public void Pause() - { - _timer.Enabled = false; - } - - public void Stop() - { - _timer?.Stop(); - _timer?.Dispose(); - _timer = null; - } + public void Stop() + { + _timer?.Stop(); + _timer?.Dispose(); + _timer = null; } } } From 038d27033cd8156221d9c6b263bf68aee941d29a Mon Sep 17 00:00:00 2001 From: LittleLittleCloud Date: Mon, 13 Feb 2023 13:33:41 -0800 Subject: [PATCH 09/14] fix build --- .../AutoMLExperimentTests.cs | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs index 18a0245718..8e073394c6 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs @@ -73,7 +73,7 @@ public async Task AutoMLExperiment_cancel_trial_when_exceeds_memory_limit_Async( // the following experiment set memory usage limit to 0.01mb // so all trials should be canceled and there should be no successful trials. // therefore when experiment finishes, it should throw timeout exception with no model trained message. - experiment.SetTrainingTimeInSeconds(10) + experiment.SetMaxModelToExplore(10) .SetTrialRunner((serviceProvider) => { var channel = serviceProvider.GetService(); @@ -81,8 +81,7 @@ public async Task AutoMLExperiment_cancel_trial_when_exceeds_memory_limit_Async( return new DummyTrialRunner(settings, 5, channel); }) .SetTuner() - .SetMaximumMemoryUsageInMegaByte(0.01) - .SetPerformanceMonitor(); + .SetMaximumMemoryUsageInMegaByte(0.01); var runExperimentAction = async () => await experiment.RunAsync(); await runExperimentAction.Should().ThrowExactlyAsync(); @@ -366,13 +365,11 @@ public void AutoMLExperiment_should_use_seed_from_context_if_provided() class DummyTrialRunner : ITrialRunner { private readonly int _finishAfterNSeconds; - private readonly CancellationToken _ct; private readonly IChannel _logger; public DummyTrialRunner(AutoMLExperiment.AutoMLExperimentSettings automlSettings, int finishAfterNSeconds, IChannel logger) { _finishAfterNSeconds = finishAfterNSeconds; - _ct = automlSettings.CancellationToken; _logger = logger; } @@ -384,7 +381,7 @@ public async Task RunAsync(TrialSettings settings, CancellationToke { _logger.Info("Update Running Trial"); await Task.Delay(_finishAfterNSeconds * 1000, ct); - _ct.ThrowIfCancellationRequested(); + ct.ThrowIfCancellationRequested(); _logger.Info("Update Completed Trial"); var metric = 1.000 + 0.01 * settings.TrialId; return new TrialResult @@ -407,9 +404,7 @@ public DummyPeformanceMonitor() _checkIntervalInMilliseconds = 1000; } - public event EventHandler CpuUsage; - - public event EventHandler MemoryUsageInMegaByte; + public event EventHandler PerformanceMetricsUpdated; public void Dispose() { @@ -425,6 +420,10 @@ public void Dispose() return 1000; } + public void OnPerformanceMetricsUpdatedHandler(TrialSettings trialSettings, TrialPerformanceMetrics metrics, CancellationTokenSource trialCancellationTokenSource) + { + } + public void Start() { if (_timer == null) @@ -432,13 +431,17 @@ public void Start() _timer = new System.Timers.Timer(_checkIntervalInMilliseconds); _timer.Elapsed += (o, e) => { - CpuUsage?.Invoke(this, 100); - MemoryUsageInMegaByte?.Invoke(this, 1000); + PerformanceMetricsUpdated?.Invoke(this, new TrialPerformanceMetrics() { PeakCpuUsage = 100, PeakMemoryUsage = 1000 }); }; _timer.AutoReset = true; - _timer.Enabled = true; } + _timer.Enabled = true; + } + + public void Pause() + { + _timer.Enabled = false; } public void Stop() From a63a31fabd922bee4932144fc36b9b8751be8d2f Mon Sep 17 00:00:00 2001 From: LittleLittleCloud Date: Mon, 13 Feb 2023 15:47:21 -0800 Subject: [PATCH 10/14] fix tests --- test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index 576bf0335b..83d52792c1 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -267,7 +267,7 @@ public void AutoFitMultiTest(bool useNumberOfCVFolds) trainData = context.Data.TakeRows(trainData, crossValRowCountThreshold - 1); var settings = new MulticlassExperimentSettings { - MaxExperimentTimeInSeconds = 1, + MaxModels = 1, }; settings.Trainers.Remove(MulticlassClassificationTrainer.LightGbm); From f68f634576c7933ba9d59d3f8aaf0033d7863c8f Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Tue, 14 Feb 2023 10:23:46 -0800 Subject: [PATCH 11/14] fix build error --- test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index 83d52792c1..a62b2dfc4b 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -75,7 +75,7 @@ public void AutoFit_UCI_Adult_Train_Test_Split_Test() var dataTrainTest = context.Data.TrainTestSplit(trainData); var settings = new BinaryExperimentSettings { - MaxExperimentTimeInSeconds = 1, + MaxModels = 1, }; settings.Trainers.Remove(BinaryClassificationTrainer.LightGbm); From 6e0e5df00e89d76dc5cfbe0f645e6f91e3e14b53 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Tue, 14 Feb 2023 12:03:35 -0800 Subject: [PATCH 12/14] add logger --- test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs index 8e073394c6..6094d9fae7 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs @@ -151,6 +151,13 @@ public async Task AutoMLExperiment_return_current_best_trial_when_ct_is_canceled public async Task AutoMLExperiment_finish_training_when_time_is_up_Async() { var context = new MLContext(1); + context.Log += (o, e) => + { + if (e.Source.StartsWith("AutoMLExperiment")) + { + this.Output.WriteLine(e.RawMessage); + } + }; var experiment = context.Auto().CreateExperiment(); experiment.SetTrainingTimeInSeconds(5) From 0c84487daabe70bd76dc65c72147b7111767338d Mon Sep 17 00:00:00 2001 From: LittleLittleCloud Date: Tue, 14 Feb 2023 17:23:12 -0800 Subject: [PATCH 13/14] increase time --- test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs index 8e073394c6..4dfdb1fe43 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs @@ -153,7 +153,7 @@ public async Task AutoMLExperiment_finish_training_when_time_is_up_Async() var context = new MLContext(1); var experiment = context.Auto().CreateExperiment(); - experiment.SetTrainingTimeInSeconds(5) + experiment.SetTrainingTimeInSeconds(10) .SetTrialRunner((serviceProvider) => { var channel = serviceProvider.GetService(); @@ -163,7 +163,7 @@ public async Task AutoMLExperiment_finish_training_when_time_is_up_Async() .SetTuner(); var cts = new CancellationTokenSource(); - cts.CancelAfter(10 * 1000); + cts.CancelAfter(20 * 1000); var res = await experiment.RunAsync(cts.Token); res.Metric.Should().BeGreaterThan(0); From 052d09f478d26e3dabbd5aced8703bb2478bb30b Mon Sep 17 00:00:00 2001 From: LittleLittleCloud Date: Wed, 15 Feb 2023 14:28:57 -0800 Subject: [PATCH 14/14] fix tests --- .../AutoMLExperiment/AutoMLExperiment.cs | 8 -------- test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs | 6 +++--- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index ab89bfcf13..5836fc2c8b 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -51,15 +51,7 @@ private void InitializeServiceCollection() _serviceCollection.TryAddTransient((provider) => { var contextManager = provider.GetRequiredService(); - var trainingStopManager = provider.GetRequiredService(); var context = contextManager.CreateMLContext(); - trainingStopManager.OnStopTraining += (s, e) => - { - // only force-canceling running trials when there's completed trials. - // otherwise, wait for the current running trial to be completed. - if (_bestTrialResult != null) - context.CancelExecution(); - }; return context; }); diff --git a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs index da8f4e05b0..415528ba9c 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs @@ -160,17 +160,17 @@ public async Task AutoMLExperiment_finish_training_when_time_is_up_Async() }; var experiment = context.Auto().CreateExperiment(); - experiment.SetTrainingTimeInSeconds(10) + experiment.SetTrainingTimeInSeconds(5) .SetTrialRunner((serviceProvider) => { var channel = serviceProvider.GetService(); var settings = serviceProvider.GetService(); - return new DummyTrialRunner(settings, 1, channel); + return new DummyTrialRunner(settings, 0, channel); }) .SetTuner(); var cts = new CancellationTokenSource(); - cts.CancelAfter(20 * 1000); + cts.CancelAfter(10 * 1000); var res = await experiment.RunAsync(cts.Token); res.Metric.Should().BeGreaterThan(0);