| File: AutoMLExperiment\IPerformanceMonitor.cs | Web Access |
| Project: src\src\Microsoft.ML.AutoML\Microsoft.ML.AutoML.csproj (Microsoft.ML.AutoML) |
// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; using System.Timers; using Microsoft.ML.Runtime; using Timer = System.Timers.Timer; namespace Microsoft.ML.AutoML { public interface IPerformanceMonitor : IDisposable { void Start(); void Pause(); void Stop(); double? GetPeakMemoryUsageInMegaByte(); double? GetPeakCpuUsage(); /// <summary> /// The handler function every time <see cref="PerformanceMetricsUpdated"/> get fired. /// </summary> void OnPerformanceMetricsUpdatedHandler(TrialSettings trialSettings, TrialPerformanceMetrics metrics, CancellationTokenSource trialCancellationTokenSource); public event EventHandler<TrialPerformanceMetrics> PerformanceMetricsUpdated; } public class DefaultPerformanceMonitor : IPerformanceMonitor { private readonly IChannel _logger; private readonly AutoMLExperiment.AutoMLExperimentSettings _settings; private Timer _timer; private double? _peakCpuUsage; private double? _peakMemoryUsage; private readonly int _checkIntervalInMilliseconds; private TimeSpan _totalCpuProcessorTime; public DefaultPerformanceMonitor(AutoMLExperiment.AutoMLExperimentSettings settings, IChannel logger, int checkIntervalInMilliseconds) { _settings = settings; _logger = logger; _checkIntervalInMilliseconds = checkIntervalInMilliseconds; } public event EventHandler<TrialPerformanceMetrics> PerformanceMetricsUpdated; public void Dispose() { Stop(); } public double? GetPeakCpuUsage() { return _peakCpuUsage; } public double? GetPeakMemoryUsageInMegaByte() { return _peakMemoryUsage; } public void Start() { if (_timer == null) { _timer = new Timer(_checkIntervalInMilliseconds); _totalCpuProcessorTime = Process.GetCurrentProcess().TotalProcessorTime; _timer.Elapsed += OnCheckCpuAndMemoryUsage; _timer.AutoReset = true; _logger?.Trace($"{typeof(DefaultPerformanceMonitor)} has been started"); } // trigger the PerformanceMetricsUpdated event and (re)start the timer _timer.Enabled = false; SampleCpuAndMemoryUsage(); _timer.Enabled = true; } public void Pause() { _timer.Enabled = false; } public void Stop() { _timer?.Stop(); _timer?.Dispose(); _timer = null; _peakCpuUsage = null; _peakMemoryUsage = null; } private void OnCheckCpuAndMemoryUsage(object source, ElapsedEventArgs e) { SampleCpuAndMemoryUsage(); } private void SampleCpuAndMemoryUsage() { // calculate CPU usage in % // the % of CPU usage is calculating in the following way // for every _totalCpuProcessorTime // total CPU time is _totalCpuProcessorTime * ProcessorCount // total CPU time used by current process is currentCpuProcessorTime // the % of CPU usage by current process is simply currentCpuProcessorTime / total CPU time. using (var process = Process.GetCurrentProcess()) { var currentCpuProcessorTime = Process.GetCurrentProcess().TotalProcessorTime; var elapseCpuProcessorTime = currentCpuProcessorTime - _totalCpuProcessorTime; var cpuUsedMs = elapseCpuProcessorTime.TotalMilliseconds; var cpuUsageInTotal = cpuUsedMs / (Environment.ProcessorCount * _checkIntervalInMilliseconds); _totalCpuProcessorTime = currentCpuProcessorTime; _peakCpuUsage = Math.Max(cpuUsageInTotal, _peakCpuUsage ?? 0); // calculate Memory Usage in MB var memoryUsage = process.WorkingSet64 * 1.0 / (1024 * 1024); _peakMemoryUsage = Math.Max(memoryUsage, _peakMemoryUsage ?? 0); var metrics = new TrialPerformanceMetrics() { CpuUsage = cpuUsageInTotal, MemoryUsage = memoryUsage, PeakCpuUsage = _peakCpuUsage, PeakMemoryUsage = _peakMemoryUsage }; _logger?.Trace($"current CPU: {cpuUsageInTotal}, current Memory(mb): {memoryUsage}"); PerformanceMetricsUpdated?.Invoke(this, metrics); } } public virtual void OnPerformanceMetricsUpdatedHandler(TrialSettings trialSettings, TrialPerformanceMetrics metrics, CancellationTokenSource trialCancellationTokenSource) { _logger.Trace($"maximum memory usage: {_settings.MaximumMemoryUsageInMegaByte}, PeakMemoryUsage: {metrics.PeakMemoryUsage} trialIsCancelled: {trialCancellationTokenSource.IsCancellationRequested}"); if (_settings.MaximumMemoryUsageInMegaByte is double d && metrics.PeakMemoryUsage > d && !trialCancellationTokenSource.IsCancellationRequested) { _logger.Trace($"cancel current trial {trialSettings.TrialId} because it uses {metrics.PeakMemoryUsage} mb memory and the maximum memory usage is {d}"); trialCancellationTokenSource.Cancel(); GC.AddMemoryPressure(Convert.ToInt64(metrics.PeakMemoryUsage) * 1024 * 1024); GC.Collect(); } } } }