From 1985cce2ee7a5ad695ba84c69f95a55a6869c977 Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Wed, 16 Dec 2020 03:55:01 -0800 Subject: [PATCH 1/2] Forward logs of Experiment's sub MLContexts to main MLContext --- .../Experiment/Experiment.cs | 8 +++++ .../Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 32 ++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ML.AutoML/Experiment/Experiment.cs b/src/Microsoft.ML.AutoML/Experiment/Experiment.cs index 1e5232a1d6..0f0da519a8 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Experiment.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Experiment.cs @@ -85,6 +85,13 @@ private void MainContextCanceledEvent(object state) } } + private void RelayCurrentContextLogsToLogger(object sender, LoggingEventArgs e) + { + // Relay logs that are generated by the current MLContext to the Experiment class's + // _logger. + _logger.Info(e.Message); + } + public IList Execute() { var iterationResults = new List(); @@ -129,6 +136,7 @@ public IList Execute() // context is canceled to stop further model training. The cancellation of the main MLContext // a user has instantiated is not desirable, thus additional MLContexts are used. _currentModelMLContext = _newContextSeedGenerator == null ? new MLContext() : new MLContext(_newContextSeedGenerator.Next()); + _currentModelMLContext.Log += RelayCurrentContextLogsToLogger; var pipeline = PipelineSuggester.GetNextInferredPipeline(_currentModelMLContext, _history, _datasetColumnInfo, _task, _optimizingMetricInfo.IsMaximizing, _experimentSettings.CacheBeforeTrainer, _logger, _trainerAllowList); // break if no candidates returned, means no valid pipeline available diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index bbfcd62d6d..516bd9efa8 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -21,10 +21,40 @@ namespace Microsoft.ML.AutoML.Test { public class AutoFitTests : BaseTestClass { + // Marker necessary for AutoFitContextLogTest to ensure that the wanted logs + // from Experiment's sub MLContexts were relayed to the main calling MLContext. + bool _markerAutoFitContextLogTest; public AutoFitTests(ITestOutputHelper output) : base(output) { } + private void MlContextLog(object sender, LoggingEventArgs e) + { + // Log containing ImageClassificationTrainer will only come from AutoML's sub + // contexts. + if (!_markerAutoFitContextLogTest && e.Message.Contains("[Source=ImageClassificationTrainer;")) + _markerAutoFitContextLogTest = true; + } + + [TensorFlowFact] + public void AutoFitContextLogTest() + { + // This test confirms that logs produced from contexts made during AutoML experiment + // runs are correctly relayed to the main Experiment MLContext. + _markerAutoFitContextLogTest = false; + var context = new MLContext(1); + context.Log += MlContextLog; + var datasetPath = DatasetUtil.GetFlowersDataset(); + var columnInference = context.Auto().InferColumns(datasetPath, "Label"); + var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions); + var trainData = textLoader.Load(datasetPath); + var result = context.Auto() + .CreateMulticlassClassificationExperiment(15) + .Execute(trainData, columnInference.ColumnInformation); + Assert.True(_markerAutoFitContextLogTest, "Image classification trainer logs from Experiment's sub contexts" + + "were not relayed to the main MLContext."); + } + [Fact] public void AutoFitBinaryTest() { @@ -85,7 +115,7 @@ public void AutoFitMultiTest(bool useNumberOfCVFolds) Assert.Equal(NumberDataViewType.Single, scoredData.Schema[DefaultColumnNames.PredictedLabel].Type); } } - + [TensorFlowFact] //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined [Trait("Category", "SkipInCI")] From 370e9b7aefc1c2feddb7d73f28e8f8d9e0326f59 Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Wed, 16 Dec 2020 17:25:43 +0300 Subject: [PATCH 2/2] Adressed reviews --- .../Experiment/Experiment.cs | 18 +++++++++++++++++- test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 4 ++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.ML.AutoML/Experiment/Experiment.cs b/src/Microsoft.ML.AutoML/Experiment/Experiment.cs index 0f0da519a8..7425666486 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Experiment.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Experiment.cs @@ -89,7 +89,23 @@ private void RelayCurrentContextLogsToLogger(object sender, LoggingEventArgs e) { // Relay logs that are generated by the current MLContext to the Experiment class's // _logger. - _logger.Info(e.Message); + switch (e.Kind) + { + case ChannelMessageKind.Trace: + _logger.Trace(e.Message); + break; + case ChannelMessageKind.Info: + _logger.Info(e.Message); + break; + case ChannelMessageKind.Warning: + _logger.Warning(e.Message); + break; + case ChannelMessageKind.Error: + _logger.Error(e.Message); + break; + default: + throw new NotImplementedException($"{nameof(ChannelMessageKind)}.{e.Kind} is not yet implemented."); + } } public IList Execute() diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index 516bd9efa8..d5af1f1be5 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -38,7 +38,7 @@ private void MlContextLog(object sender, LoggingEventArgs e) [TensorFlowFact] public void AutoFitContextLogTest() - { + { // This test confirms that logs produced from contexts made during AutoML experiment // runs are correctly relayed to the main Experiment MLContext. _markerAutoFitContextLogTest = false; @@ -115,7 +115,7 @@ public void AutoFitMultiTest(bool useNumberOfCVFolds) Assert.Equal(NumberDataViewType.Single, scoredData.Schema[DefaultColumnNames.PredictedLabel].Type); } } - + [TensorFlowFact] //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined [Trait("Category", "SkipInCI")]