File: AutoMLExperiment\IPerformanceMonitor.cs
Web Access
Project: src\src\Microsoft.ML.AutoML\Microsoft.ML.AutoML.csproj (Microsoft.ML.AutoML)
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
 
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Timers;
using Microsoft.ML.Runtime;
using Timer = System.Timers.Timer;
 
namespace Microsoft.ML.AutoML
{
    public interface IPerformanceMonitor : IDisposable
    {
        void Start();
 
        void Pause();
 
        void Stop();
 
        double? GetPeakMemoryUsageInMegaByte();
 
        double? GetPeakCpuUsage();
 
        /// <summary>
        /// The handler function every time <see cref="PerformanceMetricsUpdated"/> get fired.
        /// </summary>
        void OnPerformanceMetricsUpdatedHandler(TrialSettings trialSettings, TrialPerformanceMetrics metrics, CancellationTokenSource trialCancellationTokenSource);
 
 
        public event EventHandler<TrialPerformanceMetrics> PerformanceMetricsUpdated;
    }
 
    public class DefaultPerformanceMonitor : IPerformanceMonitor
    {
        private readonly IChannel _logger;
        private readonly AutoMLExperiment.AutoMLExperimentSettings _settings;
        private Timer _timer;
        private double? _peakCpuUsage;
        private double? _peakMemoryUsage;
        private readonly int _checkIntervalInMilliseconds;
        private TimeSpan _totalCpuProcessorTime;
 
        public DefaultPerformanceMonitor(AutoMLExperiment.AutoMLExperimentSettings settings, IChannel logger, int checkIntervalInMilliseconds)
        {
            _settings = settings;
            _logger = logger;
            _checkIntervalInMilliseconds = checkIntervalInMilliseconds;
        }
 
 
        public event EventHandler<TrialPerformanceMetrics> PerformanceMetricsUpdated;
 
 
        public void Dispose()
        {
            Stop();
        }
 
        public double? GetPeakCpuUsage()
        {
            return _peakCpuUsage;
        }
 
        public double? GetPeakMemoryUsageInMegaByte()
        {
            return _peakMemoryUsage;
        }
 
        public void Start()
        {
            if (_timer == null)
            {
                _timer = new Timer(_checkIntervalInMilliseconds);
                _totalCpuProcessorTime = Process.GetCurrentProcess().TotalProcessorTime;
                _timer.Elapsed += OnCheckCpuAndMemoryUsage;
                _timer.AutoReset = true;
                _logger?.Trace($"{typeof(DefaultPerformanceMonitor)} has been started");
            }
 
            // trigger the PerformanceMetricsUpdated event and (re)start the timer
            _timer.Enabled = false;
            SampleCpuAndMemoryUsage();
            _timer.Enabled = true;
        }
 
        public void Pause()
        {
            _timer.Enabled = false;
        }
 
        public void Stop()
        {
            _timer?.Stop();
            _timer?.Dispose();
            _timer = null;
            _peakCpuUsage = null;
            _peakMemoryUsage = null;
        }
 
        private void OnCheckCpuAndMemoryUsage(object source, ElapsedEventArgs e)
        {
            SampleCpuAndMemoryUsage();
        }
 
        private void SampleCpuAndMemoryUsage()
        {
            // calculate CPU usage in %
            // the % of CPU usage is calculating in the following way
            // for every _totalCpuProcessorTime
            // total CPU time is _totalCpuProcessorTime * ProcessorCount
            // total CPU time used by current process is currentCpuProcessorTime
            // the % of CPU usage by current process is simply currentCpuProcessorTime / total CPU time.
            using (var process = Process.GetCurrentProcess())
            {
                var currentCpuProcessorTime = Process.GetCurrentProcess().TotalProcessorTime;
                var elapseCpuProcessorTime = currentCpuProcessorTime - _totalCpuProcessorTime;
                var cpuUsedMs = elapseCpuProcessorTime.TotalMilliseconds;
                var cpuUsageInTotal = cpuUsedMs / (Environment.ProcessorCount * _checkIntervalInMilliseconds);
                _totalCpuProcessorTime = currentCpuProcessorTime;
                _peakCpuUsage = Math.Max(cpuUsageInTotal, _peakCpuUsage ?? 0);
 
                // calculate Memory Usage in MB
                var memoryUsage = process.WorkingSet64 * 1.0 / (1024 * 1024);
                _peakMemoryUsage = Math.Max(memoryUsage, _peakMemoryUsage ?? 0);
 
                var metrics = new TrialPerformanceMetrics()
                {
                    CpuUsage = cpuUsageInTotal,
                    MemoryUsage = memoryUsage,
                    PeakCpuUsage = _peakCpuUsage,
                    PeakMemoryUsage = _peakMemoryUsage
                };
 
                _logger?.Trace($"current CPU: {cpuUsageInTotal}, current Memory(mb): {memoryUsage}");
 
                PerformanceMetricsUpdated?.Invoke(this, metrics);
            }
        }
 
        public virtual void OnPerformanceMetricsUpdatedHandler(TrialSettings trialSettings, TrialPerformanceMetrics metrics, CancellationTokenSource trialCancellationTokenSource)
        {
            _logger.Trace($"maximum memory usage: {_settings.MaximumMemoryUsageInMegaByte}, PeakMemoryUsage: {metrics.PeakMemoryUsage} trialIsCancelled: {trialCancellationTokenSource.IsCancellationRequested}");
            if (_settings.MaximumMemoryUsageInMegaByte is double d && metrics.PeakMemoryUsage > d && !trialCancellationTokenSource.IsCancellationRequested)
            {
                _logger.Trace($"cancel current trial {trialSettings.TrialId} because it uses {metrics.PeakMemoryUsage} mb memory and the maximum memory usage is {d}");
                trialCancellationTokenSource.Cancel();
 
                GC.AddMemoryPressure(Convert.ToInt64(metrics.PeakMemoryUsage) * 1024 * 1024);
                GC.Collect();
            }
        }
    }
}