File: Linux\LinuxUtilizationParserCgroupV1.cs
Web Access
Project: src\src\Libraries\Microsoft.Extensions.Diagnostics.ResourceMonitoring\Microsoft.Extensions.Diagnostics.ResourceMonitoring.csproj (Microsoft.Extensions.Diagnostics.ResourceMonitoring)
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
 
using System;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using Microsoft.Extensions.ObjectPool;
using Microsoft.Shared.Diagnostics;
using Microsoft.Shared.Pools;
 
namespace Microsoft.Extensions.Diagnostics.ResourceMonitoring.Linux;
 
/// <remarks>
/// Parses Linux cgroup v1 files to retrieve resource utilization data.
/// This class is not thread safe.
/// When the same instance is called by multiple threads it may return corrupted data.
/// </remarks>
internal sealed class LinuxUtilizationParserCgroupV1 : ILinuxUtilizationParser
{
    private const float CpuShares = 1024;
    private static readonly ObjectPool<BufferWriter<char>> _sharedBufferWriterPool = BufferWriterPool.CreateBufferWriterPool<char>();
 
    /// <remarks>
    /// File contains the amount of CPU time (in microseconds) available to the group during each accounting period.
    /// </remarks>
    private static readonly FileInfo _cpuCfsQuotaUs = new("/sys/fs/cgroup/cpu/cpu.cfs_quota_us");
 
    /// <remarks>
    /// File contains the length of the accounting period in microseconds.
    /// </remarks>
    private static readonly FileInfo _cpuCfsPeriodUs = new("/sys/fs/cgroup/cpu/cpu.cfs_period_us");
 
    /// <remarks>
    /// Stat file contains information about all CPUs and their time.
    /// </remarks>
    /// <remarks>
    /// The file has format of whitespace separated values. Each value has its own meaning and unit.
    /// To know which value we read, why and what it means refer to proc (5) man page (its POSIX).
    /// </remarks>
    private static readonly FileInfo _procStat = new("/proc/stat");
 
    /// <remarks>
    /// File that contains information about available memory.
    /// </remarks>
    private static readonly FileInfo _memInfo = new("/proc/meminfo");
 
    /// <remarks>
    /// List of available CPUs for host.
    /// </remarks>
    private static readonly FileInfo _cpuSetCpus = new("/sys/fs/cgroup/cpuset/cpuset.cpus");
 
    /// <remarks>
    /// Cgroup memory limit.
    /// </remarks>
    private static readonly FileInfo _memoryLimitInBytes = new("/sys/fs/cgroup/memory/memory.limit_in_bytes");
 
    /// <summary>
    /// Cgroup memory stats.
    /// </summary>
    /// <remarks>
    /// Single line representing used memory by cgroup in bytes.
    /// </remarks>
    private static readonly FileInfo _memoryUsageInBytes = new("/sys/fs/cgroup/memory/memory.usage_in_bytes");
 
    /// <summary>
    /// Cgroup memory stats.
    /// </summary>
    /// <remarks>
    /// This file contains the details about memory usage.
    /// The format is (type of memory spent) (value) (unit of measure).
    /// </remarks>
    private static readonly FileInfo _memoryStat = new("/sys/fs/cgroup/memory/memory.stat");
 
    /// <summary>
    /// File containing usage in nanoseconds.
    /// </summary>
    /// <remarks>
    /// This value refers to the container/cgroup utilization.
    /// The format is single line with one number value.
    /// </remarks>
    private static readonly FileInfo _cpuacctUsage = new("/sys/fs/cgroup/cpuacct/cpuacct.usage");
 
    /// <summary>
    /// CPU weights, also known as shares in cgroup v1, is used for resource allocation.
    /// </summary>
    private static readonly FileInfo _cpuPodWeight = new("/sys/fs/cgroup/cpu/cpu.shares");
 
    private readonly IFileSystem _fileSystem;
    private readonly long _userHz;
 
    public LinuxUtilizationParserCgroupV1(IFileSystem fileSystem, IUserHz userHz)
    {
        _fileSystem = fileSystem;
        _userHz = userHz.Value;
    }
 
    public long GetCgroupCpuUsageInNanoseconds()
    {
        using ReturnableBufferWriter<char> bufferWriter = new(_sharedBufferWriterPool);
        _fileSystem.ReadAll(_cpuacctUsage, bufferWriter.Buffer);
 
        ReadOnlySpan<char> usage = bufferWriter.Buffer.WrittenSpan;
 
        _ = GetNextNumber(usage, out long nanoseconds);
 
        if (nanoseconds == -1)
        {
            Throw.InvalidOperationException($"Could not get cpu usage from '{_cpuacctUsage}'. Expected positive number, but got '{new string(usage)}'.");
        }
 
        return nanoseconds;
    }
 
    public long GetHostCpuUsageInNanoseconds()
    {
        const string StartingTokens = "cpu ";
        const int NumberOfColumnsRepresentingCpuUsage = 8;
        const int NanosecondsInSecond = 1_000_000_000;
 
        using ReturnableBufferWriter<char> bufferWriter = new(_sharedBufferWriterPool);
        _fileSystem.ReadFirstLine(_procStat, bufferWriter.Buffer);
 
        ReadOnlySpan<char> stat = bufferWriter.Buffer.WrittenSpan;
        long total = 0L;
 
        if (!bufferWriter.Buffer.WrittenSpan.StartsWith(StartingTokens))
        {
            Throw.InvalidOperationException($"Expected proc/stat to start with '{StartingTokens}' but it was '{new string(bufferWriter.Buffer.WrittenSpan)}'.");
        }
 
        stat = stat.Slice(StartingTokens.Length, stat.Length - StartingTokens.Length);
 
        for (int i = 0; i < NumberOfColumnsRepresentingCpuUsage; i++)
        {
            int next = GetNextNumber(stat, out long number);
 
            if (number != -1)
            {
                total += number;
            }
 
            if (next == -1)
            {
                Throw.InvalidOperationException(
                    $"'{_procStat}' should contain whitespace separated values according to POSIX. We've failed trying to get {i}th value. File content: '{new string(stat)}'.");
            }
 
            stat = stat.Slice(next, stat.Length - next);
        }
 
        return (long)(total / (double)_userHz * NanosecondsInSecond);
    }
 
    /// <remarks>
    /// When CGroup limits are set, we can calculate number of cores based on the file settings.
    /// It should be 99% of the cases when app is hosted in the container environment.
    /// Otherwise, we assume that all host's CPUs are available, which we read from proc/stat file.
    /// </remarks>
    public float GetCgroupLimitedCpus()
    {
        if (TryGetCpuUnitsFromCgroups(_fileSystem, out float cpus))
        {
            return cpus;
        }
 
        return GetHostCpuCount();
    }
 
    public float GetCgroupRequestCpu()
    {
        if (TryGetCgroupRequestCpu(_fileSystem, out float cpuUnits))
        {
            return cpuUnits;
        }
 
        return GetHostCpuCount();
    }
 
    public ulong GetAvailableMemoryInBytes()
    {
        const long UnsetCgroupMemoryLimit = 9_223_372_036_854_771_712;
        long maybeMemory = UnsetCgroupMemoryLimit;
 
        // Constrain the scope of the buffer because GetHostAvailableMemory is allocating its own buffer.
        using (ReturnableBufferWriter<char> bufferWriter = new(_sharedBufferWriterPool))
        {
            _fileSystem.ReadAll(_memoryLimitInBytes, bufferWriter.Buffer);
 
            ReadOnlySpan<char> memoryBuffer = bufferWriter.Buffer.WrittenSpan;
            _ = GetNextNumber(memoryBuffer, out maybeMemory);
 
            if (maybeMemory == -1)
            {
                Throw.InvalidOperationException($"Could not parse '{_memoryLimitInBytes}' content. Expected to find available memory in bytes but got '{new string(memoryBuffer)}' instead.");
            }
        }
 
        return maybeMemory == UnsetCgroupMemoryLimit
            ? GetHostAvailableMemory()
            : (ulong)maybeMemory;
    }
 
    public ulong GetMemoryUsageInBytes()
    {
        const string TotalInactiveFile = "total_inactive_file";
 
        using ReturnableBufferWriter<char> bufferWriter = new(_sharedBufferWriterPool);
        _fileSystem.ReadAll(_memoryStat, bufferWriter.Buffer);
        ReadOnlySpan<char> memoryFile = bufferWriter.Buffer.WrittenSpan;
 
        int index = memoryFile.IndexOf(TotalInactiveFile.AsSpan());
 
        if (index == -1)
        {
            Throw.InvalidOperationException($"Unable to find total_inactive_file from '{_memoryStat}'.");
        }
 
        ReadOnlySpan<char> inactiveMemorySlice = memoryFile.Slice(index + TotalInactiveFile.Length, memoryFile.Length - index - TotalInactiveFile.Length);
        _ = GetNextNumber(inactiveMemorySlice, out long inactiveMemory);
 
        if (inactiveMemory == -1)
        {
            Throw.InvalidOperationException($"The value of total_inactive_file found in '{_memoryStat}' is not a positive number: '{new string(inactiveMemorySlice)}'.");
        }
 
        bufferWriter.Buffer.Reset();
 
        _fileSystem.ReadAll(_memoryUsageInBytes, bufferWriter.Buffer);
 
        ReadOnlySpan<char> containerMemoryUsageFile = bufferWriter.Buffer.WrittenSpan;
        int next = GetNextNumber(containerMemoryUsageFile, out long containerMemoryUsage);
 
        // this file format doesn't expect to contain anything after the number.
        if (containerMemoryUsage == -1)
        {
            Throw.InvalidOperationException(
                $"We tried to read '{_memoryUsageInBytes}', and we expected to get a positive number but instead it was: '{new string(containerMemoryUsageFile)}'.");
        }
 
        bufferWriter.Buffer.Reset();
 
        long memoryUsage = containerMemoryUsage - inactiveMemory;
 
        if (memoryUsage < 0)
        {
            Throw.InvalidOperationException($"The total memory usage read from '{_memoryUsageInBytes}' is lesser than inactive memory usage read from '{_memoryStat}'.");
        }
 
        return (ulong)memoryUsage;
    }
 
    [SuppressMessage("Major Code Smell", "S109:Magic numbers should not be used",
        Justification = "Shifting bits left by number n is multiplying the value by 2 to the power of n.")]
    public ulong GetHostAvailableMemory()
    {
        // The value we are interested in starts with this. We just want to make sure it is true.
        const string MemTotal = "MemTotal:";
 
        using ReturnableBufferWriter<char> bufferWriter = new(_sharedBufferWriterPool);
        _fileSystem.ReadFirstLine(_memInfo, bufferWriter.Buffer);
        ReadOnlySpan<char> firstLine = bufferWriter.Buffer.WrittenSpan;
 
        if (!firstLine.StartsWith(MemTotal))
        {
            Throw.InvalidOperationException($"Could not parse '{_memInfo}'. We expected first line of the file to start with '{MemTotal}' but it was '{new string(firstLine)}' instead.");
        }
 
        ReadOnlySpan<char> totalMemory = firstLine.Slice(MemTotal.Length, firstLine.Length - MemTotal.Length);
 
        int next = GetNextNumber(totalMemory, out long totalMemoryAvailable);
 
        if (totalMemoryAvailable == -1)
        {
            Throw.InvalidOperationException($"Could not parse '{_memInfo}'. We expected to get total memory usage on first line but we've got: '{new string(firstLine)}'.");
        }
 
        if (next == -1 || totalMemory.Length - next < 2)
        {
            Throw.InvalidOperationException($"Could not parse '{_memInfo}'. We expected to get memory usage followed by the unit (kB, MB, GB) but found no unit: '{new string(firstLine)}'.");
        }
 
        ReadOnlySpan<char> unit = totalMemory.Slice(totalMemory.Length - 2, 2);
        ulong memory = (ulong)totalMemoryAvailable;
 
        ulong u = unit switch
        {
            "kB" => memory << 10,
            "MB" => memory << 20,
            "GB" => memory << 30,
            "TB" => memory << 40,
            _ => throw new InvalidOperationException(
                $"We tried to convert total memory usage value from '{_memInfo}' to bytes, but we've got a unit that we don't recognize: '{new string(unit)}'.")
        };
 
        return u;
    }
 
    /// <remarks>
    /// Comma-separated list of integers, with dashes ("-") to represent ranges. For example "0-1,5", or "0", or "1,2,3".
    /// Each value represents the zero-based index of a CPU.
    /// </remarks>
    public float GetHostCpuCount()
    {
        using ReturnableBufferWriter<char> bufferWriter = new(_sharedBufferWriterPool);
        _fileSystem.ReadFirstLine(_cpuSetCpus, bufferWriter.Buffer);
        ReadOnlySpan<char> stats = bufferWriter.Buffer.WrittenSpan;
 
        if (stats.IsEmpty)
        {
            ThrowException(stats);
        }
 
        long cpuCount = 0L;
 
        // Iterate over groups (comma-separated)
        while (true)
        {
            int groupIndex = stats.IndexOf(',');
 
            ReadOnlySpan<char> group = groupIndex == -1 ? stats : stats.Slice(0, groupIndex);
 
            int rangeIndex = group.IndexOf('-');
 
            if (rangeIndex == -1)
            {
                // Single number
                _ = GetNextNumber(group, out long singleCpu);
 
                if (singleCpu == -1)
                {
                    ThrowException(stats);
                }
 
                cpuCount += 1;
            }
            else
            {
                // Range
                ReadOnlySpan<char> first = group.Slice(0, rangeIndex);
                _ = GetNextNumber(first, out long startCpu);
 
                ReadOnlySpan<char> second = group.Slice(rangeIndex + 1);
                int next = GetNextNumber(second, out long endCpu);
 
                if (endCpu == -1 || startCpu == -1 || endCpu < startCpu || next != -1)
                {
                    ThrowException(stats);
                }
 
                cpuCount += endCpu - startCpu + 1;
            }
 
            if (groupIndex == -1)
            {
                break;
            }
 
            stats = stats.Slice(groupIndex + 1);
        }
 
        return cpuCount;
 
        static void ThrowException(ReadOnlySpan<char> content) =>
            Throw.InvalidOperationException(
                $"Could not parse '{_cpuSetCpus}'. Expected comma-separated list of integers, with dashes (\"-\") based ranges (\"0\", \"2-6,12\") but got '{new string(content)}'.");
    }
 
    /// <remarks>
    /// The input must contain only number. If there is something more than whitespace before the number, it will return failure (-1).
    /// </remarks>
    [SuppressMessage("Major Code Smell", "S109:Magic numbers should not be used",
        Justification = "We are adding another digit, so we need to multiply by ten.")]
    private static int GetNextNumber(ReadOnlySpan<char> buffer, out long number)
    {
        int numberStart = 0;
 
        while (numberStart < buffer.Length && char.IsWhiteSpace(buffer[numberStart]))
        {
            numberStart++;
        }
 
        if (numberStart == buffer.Length || !char.IsDigit(buffer[numberStart]))
        {
            number = -1;
            return -1;
        }
 
        int numberEnd = numberStart;
        number = 0;
 
        while (numberEnd < buffer.Length && char.IsDigit(buffer[numberEnd]))
        {
            int current = buffer[numberEnd] - '0';
            number *= 10;
            number += current;
            numberEnd++;
        }
 
        return numberEnd < buffer.Length ? numberEnd : -1;
    }
 
    private static bool TryGetCpuUnitsFromCgroups(IFileSystem fileSystem, out float cpuUnits)
    {
        using ReturnableBufferWriter<char> bufferWriter = new(_sharedBufferWriterPool);
        fileSystem.ReadFirstLine(_cpuCfsQuotaUs, bufferWriter.Buffer);
 
        ReadOnlySpan<char> quotaBuffer = bufferWriter.Buffer.WrittenSpan;
 
        if (quotaBuffer.IsEmpty || (quotaBuffer.Length == 2 && quotaBuffer[0] == '-' && quotaBuffer[1] == '1'))
        {
            cpuUnits = -1;
            return false;
        }
 
        int nextQuota = GetNextNumber(quotaBuffer, out long quota);
 
        if (quota == -1 || nextQuota != -1)
        {
            Throw.InvalidOperationException($"Could not parse '{_cpuCfsQuotaUs}'. Expected an integer but got: '{new string(quotaBuffer)}'.");
        }
 
        bufferWriter.Buffer.Reset();
 
        fileSystem.ReadFirstLine(_cpuCfsPeriodUs, bufferWriter.Buffer);
        ReadOnlySpan<char> periodBuffer = bufferWriter.Buffer.WrittenSpan;
 
        if (periodBuffer.IsEmpty || (periodBuffer.Length == 2 && periodBuffer[0] == '-' && periodBuffer[1] == '1'))
        {
            cpuUnits = -1;
            return false;
        }
 
        int nextPeriod = GetNextNumber(periodBuffer, out long period);
 
        if (period == -1 || nextPeriod != -1)
        {
            Throw.InvalidOperationException($"Could not parse '{_cpuCfsPeriodUs}'. Expected to get an integer but got: '{new string(periodBuffer)}'.");
        }
 
        cpuUnits = (float)quota / period;
        return true;
    }
 
    /// <summary>
    /// In cgroup v1 the CPU shares is used to determine the CPU allocation.
    /// in cgroup v2 the CPU weight is used to determine the CPU allocation.
    /// To calculate CPU request in cgroup v2 we need to read the CPU weight and convert it to CPU shares.
    /// But for cgroup v1 we can read the CPU shares directly from the file.
    /// 1024 equals 1 CPU core.
    /// In cgroup v1 on some systems the location of the CPU shares file is different.
    /// </summary>
    private static bool TryGetCgroupRequestCpu(IFileSystem fileSystem, out float cpuUnits)
    {
        if (!fileSystem.Exists(_cpuPodWeight))
        {
            cpuUnits = 0;
            return false;
        }
 
        using ReturnableBufferWriter<char> bufferWriter = new(_sharedBufferWriterPool);
        fileSystem.ReadFirstLine(_cpuPodWeight, bufferWriter.Buffer);
        ReadOnlySpan<char> cpuPodWeightBuffer = bufferWriter.Buffer.WrittenSpan;
        _ = GetNextNumber(cpuPodWeightBuffer, out long cpuPodWeight);
 
        if (cpuPodWeightBuffer.IsEmpty || (cpuPodWeightBuffer.Length == 2 && cpuPodWeightBuffer[0] == '-' && cpuPodWeightBuffer[1] == '1'))
        {
            Throw.InvalidOperationException($"Could not parse '{_cpuPodWeight}' content. Expected to find CPU weight but got '{new string(cpuPodWeightBuffer)}' instead.");
        }
 
        float result = cpuPodWeight / CpuShares;
        cpuUnits = result;
        return true;
    }
}