File: Telemetry\AgentTelemetryPersistenceTests.cs
Web Access
Project: src\tests\Aspire.Cli.Tests\Aspire.Cli.Tests.csproj (Aspire.Cli.Tests)
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
 
using System.Diagnostics;
using System.Net;
using System.Text.Json;
using Aspire.Cli.Telemetry;
using Aspire.Cli.Tests.Utils;
using Aspire.TestUtilities;
using Azure.Core.Pipeline;
using Azure.Monitor.OpenTelemetry.Exporter;
using Microsoft.AspNetCore.InternalTesting;
using Microsoft.DotNet.RemoteExecutor;
using Microsoft.Extensions.Logging.Abstractions;
using OpenTelemetry;
using OpenTelemetry.Trace;
 
namespace Aspire.Cli.Tests.Telemetry;
 
public class AgentTelemetryPersistenceTests(ITestOutputHelper outputHelper)
{
    [Fact]
    public async Task TelemetryManager_RequiresInitializationBeforeUse()
    {
        using var manager = CreateDisabledManager();
 
        Assert.False(manager.IsInitialized);
        Assert.Throws<InvalidOperationException>(() => manager.HasAzureMonitor);
        Assert.Throws<InvalidOperationException>(() => manager.HasProfilingProvider);
        Assert.Throws<InvalidOperationException>(() => manager.HasDiagnosticProvider);
        await Assert.ThrowsAsync<InvalidOperationException>(manager.ForceFlushProfilingAsync);
        await Assert.ThrowsAsync<InvalidOperationException>(manager.ForceFlushReportedAsync);
        Assert.False(await manager.TryShutdownAsync());
 
        manager.Initialize();
        Assert.True(manager.IsInitialized);
        Assert.False(manager.HasAzureMonitor);
        Assert.True(await manager.TryShutdownAsync());
        Assert.False(manager.IsInitialized);
        Assert.Throws<InvalidOperationException>(manager.Initialize);
        Assert.Throws<InvalidOperationException>(() => manager.HasAzureMonitor);
    }
 
    [Fact]
    public async Task TelemetryManager_ConcurrentInitializationAndRepeatedShutdownAreIdempotent()
    {
        using var manager = CreateDisabledManager();
        await Task.WhenAll(Enumerable.Range(0, 16).Select(_ => Task.Run(manager.Initialize)));
 
        Assert.True(manager.IsInitialized);
        var shutdownTasks = Enumerable.Range(0, 16)
            .Select(_ => Task.Run(manager.TryShutdownAsync))
            .ToArray();
        Assert.All(await Task.WhenAll(shutdownTasks), Assert.True);
        Assert.False(manager.IsInitialized);
        Assert.Throws<InvalidOperationException>(manager.Initialize);
    }
 
    [Fact]
    public async Task TelemetryManager_ConcurrentInitializeAndTryShutdownLeaveConsistentState()
    {
        using var manager = CreateDisabledManager();
        var start = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
        var initialization = Task.Run(async () =>
        {
            await start.Task;
            manager.Initialize();
        });
        var shutdown = Task.Run(async () =>
        {
            await start.Task;
            return await manager.TryShutdownAsync();
        });
 
        start.SetResult();
        await Task.WhenAll(initialization, shutdown);
 
        var wasShutDown = await shutdown;
        Assert.Equal(!wasShutDown, manager.IsInitialized);
        if (!wasShutDown)
        {
            Assert.True(await manager.TryShutdownAsync());
        }
    }
 
    [Fact]
    public async Task TelemetryManager_DisposeBeforeInitializationPreventsInitialization()
    {
        var manager = CreateDisabledManager();
        manager.Dispose();
 
        Assert.Throws<InvalidOperationException>(manager.Initialize);
        Assert.False(await manager.TryShutdownAsync());
    }
 
    [Fact]
    public async Task ForceFlushReportedAsync_WithoutProviderSucceeds()
    {
        using var manager = CreateDisabledManager();
 
        manager.Initialize();
        Assert.True(await manager.ForceFlushReportedAsync().DefaultTimeout());
    }
 
    private static TelemetryManager CreateDisabledManager()
        => new(
            new TelemetryConfiguration { ReportedTelemetryEnabled = false },
            new TelemetryTagsSource(NullLogger<TelemetryTagsSource>.Instance));
 
    [Fact]
    [OuterloopTest("Exercises the exporter's real three-minute lease expiry across processes.")]
    public void Uploader_DeliversPersistedEventAfterProducerExits()
    {
        using var workspace = TemporaryWorkspace.CreateForCli(outputHelper);
        var storage = workspace.CreateDirectory("storage").FullName;
        var lockPath = Path.Combine(workspace.Path, "uploader.lock");
        RemoteExecutor.Invoke(static async directory =>
        {
            TelemetryManager.ConfigureExporterForProcess(isAgentTelemetryInvocation: true);
            using var handler = new MockHttpMessageHandler(async (_, cancellationToken) =>
            {
                await Task.Delay(Timeout.Infinite, cancellationToken);
                throw new InvalidOperationException("The producer must exit without an upload completing.");
            });
            using var client = new HttpClient(handler);
            using var provider = CreateRecoveryProvider(directory, client);
            using var source = new ActivitySource("AgentRecoveryTest");
            using (var activity = source.StartActivity(TelemetryConstants.Activities.AgentTelemetry))
            {
                Assert.NotNull(activity);
                activity.SetTag(TelemetryConstants.Tags.AgentEventType, "skill_invocation");
                activity.SetTag(TelemetryConstants.Tags.AgentSkillName, "aspire");
            }
            Assert.True(await Task.Run(() => provider.ForceFlush(10_000)));
            Assert.True(AgentTelemetryUploader.HasPendingTelemetry(directory));
        }, storage).Dispose();
 
        // A producer can exit while holding the exporter's three-minute blob lease. Exercise real
        // expiry/recovery rather than editing private storage names or disabling exporter behavior.
        RemoteExecutor.Invoke(static async (directory, lockPath) =>
        {
            TelemetryManager.ConfigureExporterForProcess(isAgentTelemetryInvocation: true);
            var delivered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
            using var handler = new MockHttpMessageHandler(async (request, cancellationToken) =>
            {
                var payload = await request.Content!.ReadAsStringAsync(cancellationToken);
                foreach (var line in payload.Split('\n', StringSplitOptions.RemoveEmptyEntries))
                {
                    using var envelope = JsonDocument.Parse(line);
                    // Resource and process metrics can share a batch and need not have properties.
                    if (envelope.RootElement.GetProperty("data").GetProperty("baseData").TryGetProperty("properties", out var properties)
                        && properties.TryGetProperty(TelemetryConstants.Tags.AgentEventType, out var eventType))
                    {
                        Assert.Equal("skill_invocation", eventType.GetString());
                        Assert.Equal("aspire", properties.GetProperty(TelemetryConstants.Tags.AgentSkillName).GetString());
                        delivered.TrySetResult();
                    }
                }
                return new HttpResponseMessage(HttpStatusCode.OK)
                {
                    Content = new StringContent("""{"itemsReceived":1,"itemsAccepted":1,"errors":[]}""")
                };
            });
            using var client = new HttpClient(handler);
            using var provider = CreateRecoveryProvider(directory, client);
            using var timeout = new CancellationTokenSource(TimeSpan.FromMinutes(5));
            await AgentTelemetryUploader.DrainAsync(directory, lockPath, timeout.Token);
            Assert.True(delivered.Task.IsCompletedSuccessfully);
        }, storage, lockPath, new RemoteInvokeOptions { TimeOut = 360_000 }).Dispose();
    }
 
    private static TracerProvider CreateRecoveryProvider(string storage, HttpClient client)
        => Sdk.CreateTracerProviderBuilder()
            .AddSource("AgentRecoveryTest")
            .AddAzureMonitorTraceExporter(options =>
            {
                options.ConnectionString = "InstrumentationKey=11111111-1111-1111-1111-111111111111;IngestionEndpoint=https://localhost/";
                options.StorageDirectory = storage;
                options.EnableLiveMetrics = false;
                options.TracesPerSecond = null;
                options.SamplingRatio = 1;
                options.Transport = new HttpClientTransport(client);
            })
            .Build();
 
    [Theory]
    [InlineData(true)]
    [InlineData(false)]
    public void ConfigureExporterForProcess_PreservesOrdinaryCommands(bool isAgentTelemetryInvocation)
    {
        using var process = RemoteExecutor.Invoke(static value =>
        {
            var isAgent = bool.Parse(value);
            TelemetryManager.ConfigureExporterForProcess(isAgent);
 
            Assert.True(AppContext.TryGetSwitch("Azure.Monitor.OpenTelemetry.Exporter.PersistOnForceFlush", out var persist));
            Assert.Equal(isAgent, persist);
            Assert.True(AppContext.TryGetSwitch("Azure.Monitor.OpenTelemetry.Exporter.DisablePersistOnShutdown", out var disable));
            Assert.Equal(!isAgent, disable);
            Assert.Equal(isAgent ? 0 : null, AppContext.GetData("Azure.Monitor.OpenTelemetry.Exporter.ShutdownDrainBudgetMilliseconds"));
        }, isAgentTelemetryInvocation.ToString());
    }
 
    [Theory]
    [InlineData(true)]
    [InlineData(false)]
    public void ForceFlush_OnlyAgentTelemetryReturnsBeforeUpload(bool isAgentTelemetryInvocation)
    {
        using var workspace = TemporaryWorkspace.CreateForCli(outputHelper);
        using var process = RemoteExecutor.Invoke(static async (storageDirectory, value) =>
        {
            var isAgent = bool.Parse(value);
            TelemetryManager.ConfigureExporterForProcess(isAgent);
            var requestStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
            var releaseUpload = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
            var uploaded = new TaskCompletionSource<string>(TaskCreationOptions.RunContinuationsAsynchronously);
            using var handler = new MockHttpMessageHandler(async (request, cancellationToken) =>
            {
                var payload = await request.Content!.ReadAsStringAsync(cancellationToken);
                // The exporter also sends its resource/SDK metrics. Only gate our trace batch.
                if (payload.Contains(TelemetryConstants.Tags.AgentEventType, StringComparison.Ordinal))
                {
                    requestStarted.TrySetResult();
                    await releaseUpload.Task.WaitAsync(cancellationToken);
                    uploaded.TrySetResult(payload);
                }
                return new HttpResponseMessage(HttpStatusCode.OK)
                {
                    Content = new StringContent("""{"itemsReceived":3,"itemsAccepted":3,"errors":[]}""")
                };
            });
            using var httpClient = new HttpClient(handler);
            using var provider = Sdk.CreateTracerProviderBuilder()
                .AddSource("AgentPersistenceTest")
                .AddAzureMonitorTraceExporter(options =>
                {
                    // Synthetic data never reaches Azure: all requests use the in-process handler.
                    options.ConnectionString = $"InstrumentationKey={Guid.NewGuid()};IngestionEndpoint=https://localhost/";
                    options.StorageDirectory = storageDirectory;
                    options.EnableLiveMetrics = false;
                    options.TracesPerSecond = null;
                    options.SamplingRatio = 1;
                    options.Transport = new HttpClientTransport(httpClient);
                })
                .Build();
            using var source = new ActivitySource("AgentPersistenceTest");
            string[] eventTypes = ["skill_invocation", "tool_invocation", "reference_file_read"];
            foreach (var eventType in eventTypes)
            {
                using var activity = source.StartActivity(TelemetryConstants.Activities.AgentTelemetry);
                Assert.NotNull(activity);
                activity.SetTag(TelemetryConstants.Tags.AgentEventType, eventType);
            }
 
            try
            {
                var flush = Task.Run(() => provider.ForceFlush(10_000));
                if (isAgent)
                {
                    // A network-backed ForceFlush cannot finish until releaseUpload is signalled.
                    Assert.True(await flush.DefaultTimeout());
                }
 
                await requestStarted.Task.DefaultTimeout();
                Assert.False(uploaded.Task.IsCompleted);
                if (isAgent)
                {
                    Assert.NotEmpty(Directory.EnumerateFiles(storageDirectory, "*", SearchOption.AllDirectories));
                }
                else
                {
                    Assert.False(flush.IsCompleted);
                }
 
                releaseUpload.SetResult();
                Assert.True(await flush.DefaultTimeout());
                var payload = await uploaded.Task.DefaultTimeout();
                var actualEventTypes = payload.Split('\n', StringSplitOptions.RemoveEmptyEntries)
                    .Select(line =>
                    {
                        using var envelope = JsonDocument.Parse(line);
                        return envelope.RootElement.GetProperty("data").GetProperty("baseData").TryGetProperty("properties", out var properties)
                            && properties.TryGetProperty(TelemetryConstants.Tags.AgentEventType, out var eventType)
                            ? eventType.GetString()
                            : null;
                    })
                    .Where(eventType => eventType is not null);
                Assert.Equal(eventTypes.Order(), actualEventTypes.Order());
            }
            finally
            {
                releaseUpload.TrySetResult();
                provider.Shutdown();
            }
        }, workspace.Path, isAgentTelemetryInvocation.ToString());
    }
}