From 96cb24df6fa740c2bccef0d06306a82592f9bb89 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Mon, 9 Jan 2023 11:36:00 -0800 Subject: [PATCH 1/2] add _lock and improve CPU usage calc formula --- .../AutoMLExperiment/IPerformanceMonitor.cs | 51 +++++++++++-------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs index 1be581c49a..6f8a85dbb3 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs @@ -46,6 +46,9 @@ public class DefaultPerformanceMonitor : IPerformanceMonitor private double? _peakMemoryUsage; private readonly int _checkIntervalInMilliseconds; private TimeSpan _totalCpuProcessorTime; + private DateTime _previousSamplingUtcTime; + private static readonly object _lock = new object(); + public DefaultPerformanceMonitor(AutoMLExperiment.AutoMLExperimentSettings settings, IChannel logger, int checkIntervalInMilliseconds) { @@ -79,6 +82,7 @@ public void Start() { _timer = new Timer(_checkIntervalInMilliseconds); _totalCpuProcessorTime = Process.GetCurrentProcess().TotalProcessorTime; + _previousSamplingUtcTime = DateTime.UtcNow; _timer.Elapsed += OnCheckCpuAndMemoryUsage; _timer.AutoReset = true; _logger?.Trace($"{typeof(DefaultPerformanceMonitor)} has been started"); @@ -119,28 +123,33 @@ private void SampleCpuAndMemoryUsage() // the % of CPU usage by current process is simply currentCpuProcessorTime / total CPU time. using (var process = Process.GetCurrentProcess()) { - var currentCpuProcessorTime = Process.GetCurrentProcess().TotalProcessorTime; - var elapseCpuProcessorTime = currentCpuProcessorTime - _totalCpuProcessorTime; - var cpuUsedMs = elapseCpuProcessorTime.TotalMilliseconds; - var cpuUsageInTotal = cpuUsedMs / (Environment.ProcessorCount * _checkIntervalInMilliseconds); - _totalCpuProcessorTime = currentCpuProcessorTime; - _peakCpuUsage = Math.Max(cpuUsageInTotal, _peakCpuUsage ?? 0); - - // calculate Memory Usage in MB - var memoryUsage = process.PrivateMemorySize64 * 1.0 / (1024 * 1024); - _peakMemoryUsage = Math.Max(memoryUsage, _peakMemoryUsage ?? 0); - - var metrics = new TrialPerformanceMetrics() + lock (_lock) { - CpuUsage = cpuUsageInTotal, - MemoryUsage = memoryUsage, - PeakCpuUsage = _peakCpuUsage, - PeakMemoryUsage = _peakMemoryUsage - }; - - _logger?.Trace($"current CPU: {cpuUsageInTotal}, current Memory(mb): {memoryUsage}"); - - PerformanceMetricsUpdated?.Invoke(this, metrics); + var currentUtc = DateTime.UtcNow; + var currentCpuProcessorTime = process.TotalProcessorTime; + var elapseCpuProcessorTime = currentCpuProcessorTime - _totalCpuProcessorTime; + var cpuUsedMs = elapseCpuProcessorTime.TotalMilliseconds; + var cpuUsageInTotal = cpuUsedMs / (Environment.ProcessorCount * (currentUtc - _previousSamplingUtcTime).TotalMilliseconds); + _totalCpuProcessorTime = currentCpuProcessorTime; + _previousSamplingUtcTime = currentUtc; + _peakCpuUsage = Math.Max(cpuUsageInTotal, _peakCpuUsage ?? 0); + + // calculate Memory Usage in MB + var memoryUsage = process.PrivateMemorySize64 * 1.0 / (1024 * 1024); + _peakMemoryUsage = Math.Max(memoryUsage, _peakMemoryUsage ?? 0); + + var metrics = new TrialPerformanceMetrics() + { + CpuUsage = cpuUsageInTotal, + MemoryUsage = memoryUsage, + PeakCpuUsage = _peakCpuUsage, + PeakMemoryUsage = _peakMemoryUsage + }; + + _logger?.Trace($"current CPU: {cpuUsageInTotal}, current Memory(mb): {memoryUsage}"); + + PerformanceMetricsUpdated?.Invoke(this, metrics); + } } } From ae3ff87125a668edbf6c533b4e543f38f42cf4de Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Mon, 9 Jan 2023 11:44:49 -0800 Subject: [PATCH 2/2] sample CPU && Memory usage while monitor pausing. --- src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs | 3 +-- .../AutoMLExperiment/IPerformanceMonitor.cs | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index 0d80f74176..5b43ac8f8f 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -273,14 +273,13 @@ void handler(object o, EventArgs e) performanceMonitor.Start(); logger.Trace($"trial setting - {JsonSerializer.Serialize(trialSettings)}"); var trialResult = await trialTask; - + performanceMonitor.Pause(); var peakCpu = performanceMonitor?.GetPeakCpuUsage(); var peakMemoryInMB = performanceMonitor?.GetPeakMemoryUsageInMegaByte(); trialResult.PeakCpu = peakCpu; trialResult.PeakMemoryInMegaByte = peakMemoryInMB; trialResult.TrialSettings.EndedAtUtc = DateTime.UtcNow; - performanceMonitor.Pause(); monitor?.ReportCompletedTrial(trialResult); tuner.Update(trialResult); trialResultManager?.AddOrUpdateTrialResult(trialResult); diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs index 6f8a85dbb3..6c86f52724 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs @@ -87,8 +87,6 @@ public void Start() _timer.AutoReset = true; _logger?.Trace($"{typeof(DefaultPerformanceMonitor)} has been started"); } - - // trigger the PerformanceMetricsUpdated event and (re)start the timer _timer.Enabled = false; SampleCpuAndMemoryUsage(); _timer.Enabled = true; @@ -97,6 +95,7 @@ public void Start() public void Pause() { _timer.Enabled = false; + SampleCpuAndMemoryUsage(); } public void Stop()