File: tests\Shared\Hex1bAutomatorTestHelpers.cs
Web Access
Project: src\tests\Aspire.Cli.EndToEnd.Tests\Aspire.Cli.EndToEnd.Tests.csproj (Aspire.Cli.EndToEnd.Tests)
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
 
using System.Diagnostics;
using Aspire.Cli.Resources;
using Hex1b.Automation;
using Hex1b.Input;
using Xunit;
 
namespace Aspire.Tests.Shared;
 
/// <summary>
/// Extension methods for <see cref="Hex1bTerminalAutomator"/> providing Aspire-specific
/// shell prompt detection and common CLI interaction patterns.
/// </summary>
internal static class Hex1bAutomatorTestHelpers
{
    /// <summary>
    /// Waits for any prompt (success or error) matching the current sequence counter.
    /// </summary>
    internal static async Task WaitForAnyPromptAsync(
        this Hex1bTerminalAutomator auto,
        SequenceCounter counter,
        TimeSpan? timeout = null)
    {
        var effectiveTimeout = timeout ?? TimeSpan.FromSeconds(500);
 
        await auto.WaitUntilAsync(snapshot =>
        {
            var successSearcher = new CellPatternSearcher()
                .FindPattern(counter.Value.ToString())
                .RightText(" OK] $ ");
            var errorSearcher = new CellPatternSearcher()
                .FindPattern(counter.Value.ToString())
                .RightText(" ERR:");
 
            return successSearcher.Search(snapshot).Count > 0 || errorSearcher.Search(snapshot).Count > 0;
        }, timeout: effectiveTimeout, description: $"any prompt [{counter.Value} OK/ERR] $");
 
        counter.Increment();
    }
 
    /// <summary>
    /// Repeatedly types a shell command until the first line of command output matches the expected text
    /// or the timeout expires. Each attempt waits for either the first output line or the next prompt,
    /// then waits for the prompt if output appeared first.
    /// </summary>
    internal static async Task ExecuteCommandUntilOutputAsync(
        this Hex1bTerminalAutomator auto,
        SequenceCounter counter,
        string commandText,
        string desiredOutput,
        TimeSpan? timeout = null,
        TimeSpan? retryInterval = null)
    {
        ArgumentNullException.ThrowIfNull(auto);
        ArgumentNullException.ThrowIfNull(counter);
        ArgumentException.ThrowIfNullOrWhiteSpace(commandText);
        ArgumentException.ThrowIfNullOrWhiteSpace(desiredOutput);
 
        var effectiveTimeout = timeout ?? TimeSpan.FromSeconds(30);
        var effectiveRetryInterval = retryInterval ?? TimeSpan.FromSeconds(5);
        var stopwatch = Stopwatch.StartNew();
        var attempt = 0;
 
        while (stopwatch.Elapsed < effectiveTimeout)
        {
            attempt++;
            var expectedPromptSequence = counter.Value;
            var sawPrompt = false;
            var firstOutputMatched = false;
 
            await auto.TypeAsync(commandText);
            await auto.EnterAsync();
 
            var remaining = effectiveTimeout - stopwatch.Elapsed;
            if (remaining <= TimeSpan.Zero)
            {
                break;
            }
 
            var waitThisAttempt = remaining < effectiveRetryInterval ? remaining : effectiveRetryInterval;
 
            try
            {
                await auto.WaitUntilAsync(snapshot =>
                {
                    var firstOutputLine = TryGetFirstOutputLine(snapshot, commandText);
                    if (firstOutputLine is not null)
                    {
                        if (IsPromptLine(firstOutputLine, expectedPromptSequence))
                        {
                            sawPrompt = true;
                            return true;
                        }
 
                        firstOutputMatched = firstOutputLine.Contains(desiredOutput, StringComparison.Ordinal);
                        sawPrompt = IsPromptVisible(snapshot, expectedPromptSequence);
                        return true;
                    }
 
                    if (IsPromptVisible(snapshot, expectedPromptSequence))
                    {
                        sawPrompt = true;
                        return true;
                    }
 
                    return false;
                }, timeout: waitThisAttempt, description: $"waiting for first output or prompt after '{commandText}' (attempt {attempt})");
            }
            catch (TimeoutException) when (stopwatch.Elapsed < effectiveTimeout)
            {
                continue;
            }
 
            if (!sawPrompt)
            {
                var promptGrace = TimeSpan.FromSeconds(1);
                try
                {
                    await auto.WaitForAnyPromptAsync(counter, promptGrace);
                    sawPrompt = true;
                }
                catch (TimeoutException) when (stopwatch.Elapsed < effectiveTimeout + promptGrace)
                {
                    continue;
                }
            }
            else
            {
                counter.Increment();
            }
 
            if (firstOutputMatched)
            {
                return; // success
            }
 
            remaining = effectiveTimeout - stopwatch.Elapsed;
            if (remaining <= TimeSpan.Zero)
            {
                break;
            }
 
            var delayBeforeRetry = remaining < effectiveRetryInterval ? remaining : effectiveRetryInterval;
            if (delayBeforeRetry > TimeSpan.FromMilliseconds(1))
            {
                await auto.WaitAsync(delayBeforeRetry);
            }
        }
 
        throw new TimeoutException(
            $"Timed out after {effectiveTimeout} waiting for the first output line from '{commandText}' to contain '{desiredOutput}'.");
    }
 
    private static bool IsPromptVisible(IHex1bTerminalRegion snapshot, int expectedPromptSequence)
    {
        var successSearcher = new CellPatternSearcher()
            .FindPattern(expectedPromptSequence.ToString())
            .RightText(" OK] $ ");
        var errorSearcher = new CellPatternSearcher()
            .FindPattern(expectedPromptSequence.ToString())
            .RightText(" ERR:");
 
        return successSearcher.Search(snapshot).Count > 0 || errorSearcher.Search(snapshot).Count > 0;
    }
 
    private static bool IsPromptLine(string line, int expectedPromptSequence)
    {
        return line.Contains($"[{expectedPromptSequence} OK] $", StringComparison.Ordinal) ||
            line.Contains($"[{expectedPromptSequence} ERR:", StringComparison.Ordinal);
    }
 
    private static string? TryGetFirstOutputLine(IHex1bTerminalRegion snapshot, string commandText)
    {
        var commandLineIndex = FindCommandLineIndex(snapshot, commandText);
        if (commandLineIndex < 0)
        {
            return null;
        }
 
        for (var lineIndex = commandLineIndex + 1; lineIndex < snapshot.Height; lineIndex++)
        {
            var line = snapshot.GetLineTrimmed(lineIndex);
            if (!string.IsNullOrWhiteSpace(line))
            {
                return line;
            }
        }
 
        return null;
    }
 
    private static int FindCommandLineIndex(IHex1bTerminalRegion snapshot, string commandText)
    {
        for (var lineIndex = snapshot.Height - 1; lineIndex >= 0; lineIndex--)
        {
            var line = snapshot.GetLineTrimmed(lineIndex);
            if (line.EndsWith(commandText, StringComparison.Ordinal) ||
                line.Contains($"$ {commandText}", StringComparison.Ordinal) ||
                line.Contains($"# {commandText}", StringComparison.Ordinal))
            {
                return lineIndex;
            }
        }
 
        for (var lineIndex = snapshot.Height - 1; lineIndex >= 0; lineIndex--)
        {
            var line = snapshot.GetLineTrimmed(lineIndex);
            if (line.Contains(commandText, StringComparison.Ordinal))
            {
                return lineIndex;
            }
        }
 
        return -1;
    }
 
    /// <summary>
    /// Waits for a successful command prompt, but fails fast if an error prompt is detected.
    /// </summary>
    internal static async Task WaitForSuccessPromptAsync(
        this Hex1bTerminalAutomator auto,
        SequenceCounter counter,
        TimeSpan? timeout = null)
    {
        var effectiveTimeout = timeout ?? TimeSpan.FromSeconds(500);
        var sawError = false;
 
        await auto.WaitUntilAsync(snapshot =>
        {
            var successSearcher = new CellPatternSearcher()
                .FindPattern(counter.Value.ToString())
                .RightText(" OK] $ ");
 
            if (successSearcher.Search(snapshot).Count > 0)
            {
                return true;
            }
 
            var errorSearcher = new CellPatternSearcher()
                .FindPattern(counter.Value.ToString())
                .RightText(" ERR:");
 
            if (errorSearcher.Search(snapshot).Count > 0)
            {
                sawError = true;
                return true;
            }
 
            return false;
        }, timeout: effectiveTimeout, description: $"success prompt [{counter.Value} OK] $ (fail-fast on error)");
 
        if (sawError)
        {
            throw new InvalidOperationException(
                $"Command failed with non-zero exit code (detected ERR prompt at sequence {counter.Value}). Check the terminal recording for details.");
        }
 
        counter.Increment();
    }
 
    /// <summary>
    /// Types a shell command, waits for it to complete successfully, and advances the prompt counter.
    /// </summary>
    internal static async Task RunCommandAsync(
        this Hex1bTerminalAutomator auto,
        string command,
        SequenceCounter counter,
        TimeSpan? timeout = null)
    {
        await auto.TypeAsync(command);
        await auto.EnterAsync();
        await auto.WaitForSuccessPromptAsync(counter, timeout);
    }
 
    /// <summary>
    /// Configures a numbered bash prompt and changes into the provided workspace directory.
    /// </summary>
    internal static async Task PrepareBashEnvironmentAsync(
        this Hex1bTerminalAutomator auto,
        string workspacePath,
        SequenceCounter counter,
        TimeSpan? timeout = null)
    {
        var effectiveTimeout = timeout ?? TimeSpan.FromSeconds(30);
        var waitingForInputPattern = new CellPatternSearcher()
            .Find("b")
            .RightUntil("$")
            .Right(' ')
            .Right(' ');
 
        await auto.WaitUntilAsync(
            s => waitingForInputPattern.Search(s).Count > 0,
            timeout: effectiveTimeout,
            description: "initial bash prompt");
        await auto.WaitAsync(500);
 
        await auto.TypeAsync(AspireCliShellCommandHelpers.NumberedPromptSetupCommand);
        await auto.EnterAsync();
        await auto.WaitForSuccessPromptAsync(counter);
 
        await auto.RunCommandAsync($"cd {AspireCliShellCommandHelpers.QuoteBashArg(workspacePath)}", counter);
    }
 
    /// <summary>
    /// Extracts a localhive archive into <c>~/.aspire</c>.
    /// </summary>
    internal static async Task ExtractLocalHiveArchiveAsync(
        this Hex1bTerminalAutomator auto,
        string archivePath,
        SequenceCounter counter)
    {
        await auto.RunCommandAsync(
            AspireCliShellCommandHelpers.GetExtractLocalHiveArchiveCommand(archivePath),
            counter,
            TimeSpan.FromSeconds(30));
    }
 
    /// <summary>
    /// Configures Aspire to use the extracted localhive packages.
    /// </summary>
    internal static async Task ConfigureLocalHiveAsync(
        this Hex1bTerminalAutomator auto,
        SequenceCounter counter)
    {
        foreach (var command in AspireCliShellCommandHelpers.GetConfigureLocalHiveCommands())
        {
            await auto.RunCommandAsync(command, counter);
        }
    }
 
    /// <summary>
    /// Sources the standard <c>~/.aspire</c> environment for CLI or bundle execution.
    /// </summary>
    internal static async Task SourceAspireEnvironmentAsync(
        this Hex1bTerminalAutomator auto,
        SequenceCounter counter,
        bool includeBundlePath = false)
    {
        await auto.RunCommandAsync(
            AspireCliShellCommandHelpers.GetSourceAspireEnvironmentCommand(includeBundlePath),
            counter,
            TimeSpan.FromSeconds(30));
    }
 
    /// <summary>
    /// Logs the installed Aspire CLI version.
    /// </summary>
    internal static async Task LogAspireCliVersionAsync(
        this Hex1bTerminalAutomator auto,
        SequenceCounter counter)
    {
        await auto.RunCommandAsync(
            AspireCliShellCommandHelpers.AspireCliVersionCommand,
            counter,
            TimeSpan.FromSeconds(30));
    }
 
    /// <summary>
    /// Waits for <c>aspire add</c> to either finish directly or stop on the version-selection prompt.
    /// </summary>
    internal static async Task WaitForAspireAddCompletionAsync(
        this Hex1bTerminalAutomator auto,
        SequenceCounter counter,
        TimeSpan? timeout = null)
    {
        var effectiveTimeout = timeout ?? TimeSpan.FromMinutes(2);
        var sawVersionPrompt = false;
        var successPrompt = new CellPatternSearcher()
            .FindPattern(counter.Value.ToString())
            .RightText(" OK] $ ");
        var errorPrompt = new CellPatternSearcher()
            .FindPattern(counter.Value.ToString())
            .RightText(" ERR:");
        var waitingForVersionSelection = new CellPatternSearcher()
            .Find("What version would you like to install?");
        var waitingForLegacyVersionSelection = new CellPatternSearcher()
            .Find("based on NuGet.config");
 
        // We intentionally do NOT check for completion text like "was added successfully"
        // because it persists on the terminal from prior aspire-add runs and would match
        // stale output when multiple aspire-add commands run in sequence. Instead we rely
        // on the shell success/error prompt which uses a unique counter value.
 
        await auto.WaitUntilAsync(s =>
            {
                if (waitingForVersionSelection.Search(s).Count > 0 || waitingForLegacyVersionSelection.Search(s).Count > 0)
                {
                    sawVersionPrompt = true;
                    return true;
                }
 
                return successPrompt.Search(s).Count > 0
                    || errorPrompt.Search(s).Count > 0;
            },
            timeout: effectiveTimeout,
            description: "aspire add completion or version-selection prompt");
 
        if (!sawVersionPrompt)
        {
            await auto.WaitForSuccessPromptAsync(counter, effectiveTimeout);
            return;
        }
 
        await auto.EnterAsync();
        await auto.WaitForSuccessPromptAsync(counter, effectiveTimeout);
    }
 
    /// <summary>
    /// Handles the agent init confirmation prompt that appears after aspire init/new,
    /// then waits for the shell success prompt. Supports CLI versions with and without agent init chaining.
    /// </summary>
    internal static async Task DeclineAgentInitPromptAsync(
        this Hex1bTerminalAutomator auto,
        SequenceCounter counter,
        TimeSpan? timeout = null)
    {
        var effectiveTimeout = timeout ?? TimeSpan.FromSeconds(500);
 
        var agentInitPrompt = new CellPatternSearcher()
            .Find("configure AI agent environments");
 
        var agentInitFound = false;
        var errorPromptFound = false;
 
        // Wait for either the agent init prompt (new CLI) or the success prompt (old CLI).
        await auto.WaitUntilAsync(s =>
        {
            if (agentInitPrompt.Search(s).Count > 0)
            {
                agentInitFound = true;
                return true;
            }
            var successSearcher = new CellPatternSearcher()
                .FindPattern(counter.Value.ToString())
                .RightText(" OK] $ ");
            if (successSearcher.Search(s).Count > 0)
            {
                return true;
            }
 
            var errorSearcher = new CellPatternSearcher()
                .FindPattern(counter.Value.ToString())
                .RightText(" ERR:");
            errorPromptFound = errorSearcher.Search(s).Count > 0;
            return errorPromptFound;
        }, timeout: effectiveTimeout, description: $"agent init prompt, success prompt [{counter.Value} OK] $, or error prompt [{counter.Value} ERR:*] $");
 
        if (errorPromptFound)
        {
            throw new InvalidOperationException($"Command failed with error prompt [{counter.Value} ERR:*] while waiting for the agent init prompt or success prompt.");
        }
 
        if (!agentInitFound)
        {
            counter.Increment();
            return;
        }
 
        await auto.WaitAsync(500);
        await auto.TypeAsync("n");
 
        // Do not send Enter after typing "n" — the Spectre Console [Y/n] confirmation
        // prompt accepts a single character. Sending Enter risks a race: if aspire init
        // exits after reading "n" but before the Enter is delivered, bash receives the
        // Enter and executes a phantom blank command, advancing CMDCOUNT and desyncing
        // the test counter from the shell counter.
 
        await auto.WaitForSuccessPromptAsync(counter, effectiveTimeout);
    }
 
    /// <summary>
    /// Runs <c>aspire new</c> interactively, selecting the specified template and responding to all prompts.
    /// </summary>
    internal static async Task AspireNewAsync(
        this Hex1bTerminalAutomator auto,
        string projectName,
        SequenceCounter counter,
        AspireTemplate template = AspireTemplate.Starter,
        bool useRedisCache = true,
        bool useDevLocalhost = false)
    {
        var templateTimeout = TimeSpan.FromSeconds(60);
 
        // Step 1: Type aspire new and wait for the template list
        await auto.TypeAsync("aspire new");
        await auto.EnterAsync();
        await auto.WaitUntilAsync(
            s => new CellPatternSearcher().Find("> Starter App").Search(s).Count > 0,
            timeout: templateTimeout,
            description: "template selection list (> Starter App)");
 
        // Step 2: Navigate to and select the desired template
        switch (template)
        {
            case AspireTemplate.Starter:
                await auto.EnterAsync(); // First option, no navigation needed
                break;
 
            case AspireTemplate.JsReact:
                await auto.DownAsync();
                await auto.WaitUntilAsync(
                    s => new CellPatternSearcher().Find("> Starter App (ASP.NET Core/React, C# AppHost)").Search(s).Count > 0,
                    timeout: TimeSpan.FromSeconds(5),
                    description: "JS React template selected");
                await auto.EnterAsync();
                break;
 
            case AspireTemplate.ExpressReact:
                await auto.DownAsync();
                await auto.DownAsync();
                await auto.WaitUntilAsync(
                    s => new CellPatternSearcher().Find("> Starter App (Express/React, TypeScript AppHost)").Search(s).Count > 0,
                    timeout: TimeSpan.FromSeconds(5),
                    description: "Express React template selected");
                await auto.EnterAsync();
                break;
 
            case AspireTemplate.PythonReact:
                await auto.DownAsync();
                await auto.DownAsync();
                await auto.DownAsync();
                await auto.WaitUntilAsync(
                    s => new CellPatternSearcher().Find("> Starter App (FastAPI/React, TypeScript AppHost)").Search(s).Count > 0,
                    timeout: TimeSpan.FromSeconds(5),
                    description: "Python React template selected");
                await auto.EnterAsync();
                break;
 
            case AspireTemplate.EmptyAppHost:
                await auto.TypeAsync("Empty AppHost");
                await auto.EnterAsync();
                try
                {
                    await auto.WaitUntilAsync(
                        s => new CellPatternSearcher().Find("Which language would you like to use?").Search(s).Count > 0,
                        timeout: TimeSpan.FromSeconds(5),
                        description: "AppHost language prompt");
                    await auto.EnterAsync();
                }
                catch (Hex1bAutomationException)
                {
                }
                break;
 
            case AspireTemplate.TypeScriptEmptyAppHost:
                await auto.TypeAsync("Empty AppHost");
                await auto.EnterAsync();
                await auto.WaitUntilAsync(
                    s => new CellPatternSearcher().Find("Which language would you like to use?").Search(s).Count > 0,
                    timeout: TimeSpan.FromSeconds(10),
                    description: "AppHost language prompt");
                await auto.TypeAsync("TypeScript");
                await auto.EnterAsync();
                break;
 
            case AspireTemplate.JavaEmptyAppHost:
                await auto.TypeAsync("Empty AppHost");
                await auto.EnterAsync();
                await auto.WaitUntilAsync(
                    s => new CellPatternSearcher().Find("Which language would you like to use?").Search(s).Count > 0,
                    timeout: TimeSpan.FromSeconds(10),
                    description: "AppHost language prompt");
                await auto.TypeAsync("Java");
                await auto.EnterAsync();
                break;
 
            default:
                throw new ArgumentOutOfRangeException(nameof(template), template, $"Unsupported template: {template}");
        }
        // Step 3: Enter the project name. The CLI resolves the selected template's version against
        // the active channel's feed somewhere in this window, and slow feeds (staging/daily darc
        // feeds) can take well over 10s — far longer than nuget.org. Use templateTimeout here (and
        // for the output-path prompt below) so emulated staging/daily runs don't time out waiting
        // for the prompt; otherwise the wait throws while `aspire new` is still open and the
        // teardown `exit` gets consumed by the live prompt, hanging the whole run.
        await auto.WaitUntilAsync(
            s => new CellPatternSearcher().Find("Enter the project name").Search(s).Count > 0,
            timeout: templateTimeout,
            description: "project name prompt");
        await auto.TypeAsync(projectName);
        await auto.EnterAsync();
 
        // Step 4: Accept default output path
        await auto.WaitUntilAsync(
            s => new CellPatternSearcher().Find("Enter the output path").Search(s).Count > 0,
            timeout: templateTimeout,
            description: "output path prompt");
        await auto.EnterAsync();
 
        // Step 5: URLs prompt (all templates have this). The CLI may spend time
        // resolving template versions after the output path is entered, so reuse
        // the template-selection timeout for this first post-resolution prompt.
        await auto.WaitUntilAsync(
            s => new CellPatternSearcher().Find("Use *.dev.localhost URLs").Search(s).Count > 0,
            timeout: templateTimeout,
            description: "URLs prompt");
        if (useDevLocalhost)
        {
            await auto.TypeAsync("y");
        }
        else
        {
            await auto.EnterAsync(); // Accept default "No"
        }
 
        // Step 6: Redis prompt (only Starter, JsReact, PythonReact)
        if (template is AspireTemplate.Starter or AspireTemplate.JsReact or AspireTemplate.PythonReact)
        {
            await auto.WaitUntilAsync(
                s => new CellPatternSearcher().Find("Use Redis Cache").Search(s).Count > 0,
                timeout: TimeSpan.FromSeconds(10),
                description: "Redis cache prompt");
 
            if (!useRedisCache)
            {
                await auto.TypeAsync("n");
            }
            else
            {
                await auto.EnterAsync();
            }
        }
 
        // Step 7: Test project prompt (only Starter)
        if (template is AspireTemplate.Starter)
        {
            await auto.WaitUntilAsync(
                s => new CellPatternSearcher().Find("Do you want to create a test project?").Search(s).Count > 0,
                timeout: TimeSpan.FromSeconds(10),
                description: "test project prompt");
            await auto.EnterAsync(); // Accept default "No"
        }
 
        // Step 8: Decline the agent init prompt and wait for success
        await auto.DeclineAgentInitPromptAsync(counter);
    }
 
    /// <summary>
    /// Runs <c>aspire init --language csharp</c> and handles the NuGet.config, URLs, and agent init prompts.
    /// </summary>
    internal static async Task AspireInitAsync(
        this Hex1bTerminalAutomator auto,
        SequenceCounter counter)
    {
        // Match the actual prompt text shape — the trailing '?' avoids false-matching
        // informational confirmation messages like "Created NuGet.config..." which contain
        // the same substring but are not Y/n prompts. The two real prompts are
        // "Create NuGet.config for selected channels?" and "Update NuGet.config to add
        // missing package sources for the selected channel?" — both end in '?'.
        var waitingForNuGetConfigPrompt = new CellPatternSearcher()
            .Find("NuGet.config?");
 
        var waitingForUrlsPrompt = new CellPatternSearcher()
            .Find("Use *.dev.localhost URLs");
 
        var waitingForInitComplete = new CellPatternSearcher()
            .Find("Aspire initialization complete");
 
        var waitingForAgentInitPrompt = new CellPatternSearcher()
            .Find("configure AI agent environments");
 
        await auto.TypeAsync("aspire init --language csharp");
        await auto.EnterAsync();
 
        var handledNuGetConfigPrompt = false;
        var handledUrlsPrompt = false;
 
        while (true)
        {
            var initState = "unknown";
            await auto.WaitUntilAsync(s =>
            {
                if (!handledNuGetConfigPrompt && waitingForNuGetConfigPrompt.Search(s).Count > 0)
                {
                    initState = "nuget-config";
                    return true;
                }
 
                if (!handledUrlsPrompt && waitingForUrlsPrompt.Search(s).Count > 0)
                {
                    initState = "urls";
                    return true;
                }
 
                if (waitingForAgentInitPrompt.Search(s).Count > 0)
                {
                    initState = "agent-init";
                    return true;
                }
 
                if (waitingForInitComplete.Search(s).Count > 0)
                {
                    initState = "init-complete";
                    return true;
                }
 
                return false;
            }, timeout: TimeSpan.FromMinutes(2), description: "NuGet.config prompt, URLs prompt, agent init prompt, or init completion");
 
            if (initState is "nuget-config" or "urls")
            {
                if (initState == "nuget-config")
                {
                    handledNuGetConfigPrompt = true;
                }
                else
                {
                    handledUrlsPrompt = true;
                }
 
                await auto.EnterAsync();
                continue;
            }
 
            await auto.DeclineAgentInitPromptAsync(counter);
            return;
        }
    }
 
    /// <summary>
    /// Waits for the deploy/destroy pipeline to complete, failing immediately if the pipeline reports failure
    /// instead of waiting for the full timeout to elapse.
    /// </summary>
    internal static async Task WaitForPipelineSuccessAsync(
        this Hex1bTerminalAutomator auto,
        TimeSpan? timeout = null)
    {
        var effectiveTimeout = timeout ?? TimeSpan.FromMinutes(5);
        var pipelineSucceeded = false;
        string? terminalOutput = null;
 
        await auto.WaitUntilAsync(s =>
        {
            if (s.ContainsText(ConsoleActivityLoggerStrings.PipelineFailed))
            {
                terminalOutput = s.GetText();
                return true;
            }
 
            if (s.ContainsText(ConsoleActivityLoggerStrings.PipelineSucceeded))
            {
                pipelineSucceeded = true;
                return true;
            }
 
            return false;
        }, timeout: effectiveTimeout, description: "pipeline succeeded or failed");
 
        if (!pipelineSucceeded)
        {
            // Azure sometimes reports that a region's shared compute capacity is temporarily
            // exhausted while provisioning, e.g.:
            //   ManagedEnvironmentCapacityHeavyUsageError / "code": "AKSCapacityHeavyUsage"
            //   "AKS is experiencing heavy usage in region westus3. We are working on adding
            //    new capacity. In the meantime, please consider creating new AKS clusters in a
            //    different region."
            // That is an Azure-side infrastructure condition, not a product or deployment defect:
            // it is the quota-bearing region being oversubscribed, and it does not clear on a short
            // retry (and cannot be moved to another region because quota is allocated per-region).
            // Treat it as an environmental skip — consistent with how these deployment tests already
            // Assert.Skip when an Azure subscription or login is unavailable — so transient regional
            // capacity weather does not turn the nightly red.
            if (terminalOutput is not null && ContainsTransientAzureCapacityError(terminalOutput))
            {
                Assert.Skip(
                    "Skipped: Azure reported transient regional capacity exhaustion during provisioning " +
                    $"(not a product failure). Terminal output:{Environment.NewLine}{terminalOutput}");
            }
 
            throw new InvalidOperationException($"Pipeline failed unexpectedly. Terminal output:{Environment.NewLine}{terminalOutput}");
        }
    }
 
    // Azure error codes that indicate a region's shared capacity is temporarily exhausted. These
    // are infrastructure-side conditions (not product defects) that do not clear on a same-region
    // retry, so the deployment tests treat them as skips rather than failures. Matched
    // case-insensitively against the deploy pipeline's terminal output.
    private static readonly string[] s_transientAzureCapacityErrorMarkers =
    [
        "AKSCapacityHeavyUsage",
        "ManagedEnvironmentCapacityHeavyUsageError",
    ];
 
    private static bool ContainsTransientAzureCapacityError(string terminalOutput)
    {
        foreach (var marker in s_transientAzureCapacityErrorMarkers)
        {
            if (terminalOutput.Contains(marker, StringComparison.OrdinalIgnoreCase))
            {
                return true;
            }
        }
 
        return false;
    }
}