| File: Lifecycle\TerminalHostFailureDiagnosticService.cs | Web Access |
| Project: src\src\Aspire.Hosting\Aspire.Hosting.csproj (Aspire.Hosting) |
// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. using Aspire.Hosting.ApplicationModel; using Aspire.Hosting.Dcp; using Aspire.Shared; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; namespace Aspire.Hosting.Lifecycle; /// <summary> /// Observes <see cref="TerminalHostResource"/> state transitions and, when one transitions /// into a terminal failure state, surfaces the failure to the user by: /// <list type="number"> /// <item>Unhiding the (normally hidden) host resource in the dashboard / <c>aspire ps</c> so /// the failure is discoverable instead of buried under the parent's "WaitUntilStarted" /// hang.</item> /// <item>Writing a diagnostic line into the host's resource log stream — the place the user /// naturally lands when they click the unhidden resource — telling them what likely /// went wrong and how to recover.</item> /// </list> /// <para> /// The most common failure mode this surfaces is a CLI / AppHost version mismatch: a user upgrades /// their AppHost to a version that calls <c>.WithTerminal()</c> but has an older bundled /// <c>aspire-managed</c> binary on PATH (resolved via the DashboardPath inference in /// <see cref="DcpOptions"/>). The old binary rejects the <c>terminalhost</c> subcommand and exits /// non-zero. Before this service existed, the only symptom was the parent resource hanging on /// <c>WaitUntilStarted</c> against a hidden, silently-failing dependency. /// </para> /// </summary> internal sealed class TerminalHostFailureDiagnosticService( ResourceNotificationService notifications, ResourceLoggerService loggers, IOptions<DcpOptions> dcpOptions, ILogger<TerminalHostFailureDiagnosticService> logger) : BackgroundService { // States we treat as terminal-failure for a terminal host. Finished/Exited with non-zero // exit code or no exit code (e.g. host crashed before reporting) is a failure; with // exit code 0 it is a clean shutdown during AppHost stop and must not be surfaced as // a failure (otherwise every successful run would log a phantom error on app shutdown). private static readonly string[] s_terminalStates = [ KnownResourceStates.FailedToStart, KnownResourceStates.Finished, KnownResourceStates.Exited, ]; protected override async Task ExecuteAsync(CancellationToken stoppingToken) { // DCP can re-emit terminal-state events during recycle attempts and shutdown. Track // which hosts we've already diagnosed (by resource id, which is the per-replica DCP // name) so we only write the diagnostic block once per host instance. var diagnosedHosts = new HashSet<string>(StringComparer.Ordinal); try { await foreach (var evt in notifications.WatchAsync(stoppingToken).ConfigureAwait(false)) { if (evt.Resource is not TerminalHostResource host) { continue; } var state = evt.Snapshot.State?.Text; if (state is null || Array.IndexOf(s_terminalStates, state) < 0) { continue; } // Treat exit code 0 (or "Finished"/"Exited" + zero) as a clean stop, not a // failure. FailedToStart is always a failure regardless of exit code. if (state != KnownResourceStates.FailedToStart && evt.Snapshot.ExitCode is 0) { continue; } if (!diagnosedHosts.Add(evt.ResourceId)) { continue; } await SurfaceFailureAsync(host, evt.Snapshot, stoppingToken).ConfigureAwait(false); } } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { // Normal shutdown. } catch (Exception ex) { // Diagnostic services must never bring down the host. Log and exit. logger.LogDebug(ex, "TerminalHostFailureDiagnosticService stopped unexpectedly."); } } private async Task SurfaceFailureAsync(TerminalHostResource host, CustomResourceSnapshot snapshot, CancellationToken cancellationToken) { _ = cancellationToken; // Unhide the failed host so it shows up in the dashboard resource list and aspire ps. // Hosts are hidden by default (see ConfigureTerminalHostAnnotations) because they are // an implementation detail of WithTerminal(); on failure we make them discoverable so // the user has a place to land that explains the failure. try { await notifications.PublishUpdateAsync(host, s => s with { IsHidden = false }) .ConfigureAwait(false); } catch (Exception ex) { logger.LogDebug(ex, "Failed to unhide terminal host '{HostName}' after failure.", host.Name); } var resourceLogger = loggers.GetLogger(host); var exitInfo = snapshot.ExitCode is { } code ? $" (exit code {code.ToString(System.Globalization.CultureInfo.InvariantCulture)})" : string.Empty; var binaryPath = dcpOptions.Value.TerminalHostPath ?? "<unresolved>"; resourceLogger.LogError( "Terminal host for '{TargetName}' (replica {ReplicaIndex}) failed to start{ExitInfo}. Binary: '{TerminalHostPath}'.", host.Parent.Name, host.ParentReplicaIndex, exitInfo, binaryPath); if (LooksLikeBundledCliHost(dcpOptions.Value.TerminalHostPath, dcpOptions.Value.TerminalHostInvocationArgs)) { // We resolved the host to the bundled aspire-managed multi-mode binary and asked it // to dispatch via the "terminalhost" subcommand. Pre-13.4 aspire-managed builds only // accept dashboard|server|nuget, so this combination most commonly fails when an // older bundled aspire CLI is paired with an AppHost that calls .WithTerminal(). resourceLogger.LogError( "If you recently added .WithTerminal() to a resource, the bundled aspire CLI on this machine may be older than this AppHost and may not ship the 'terminalhost' command. Run `aspire update --self` to upgrade the CLI, then re-run the AppHost. If the issue persists after upgrading, please open an issue at https://aka.ms/aspire/new-issue."); } } private static bool LooksLikeBundledCliHost(string? path, string? invocationArgs) { if (string.IsNullOrEmpty(path) || string.IsNullOrEmpty(invocationArgs)) { return false; } // The bundled-CLI path is uniquely identified by the combination of "aspire-managed" // as the binary name and "terminalhost" as the leading invocation arg (the dispatcher // subcommand). Per-RID NuGet terminal-host packages (historical, pre-bundle-cutover) // shipped their own dedicated binary without a leading subcommand arg, so they would // not match. return invocationArgs.Trim().Equals("terminalhost", StringComparison.Ordinal) && BundleDiscovery.IsAspireManagedBinary(path); } }