File: CheckAzurePipelinesTestResults.cs
Web Access
Project: src\src\Microsoft.DotNet.Helix\Sdk\Microsoft.DotNet.Helix.Sdk.csproj (Microsoft.DotNet.Helix.Sdk)
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
 
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using Microsoft.Build.Framework;
using Microsoft.DotNet.Helix.Sdk;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
 
namespace Microsoft.DotNet.Helix.AzureDevOps
{
    public class CheckAzurePipelinesTestResults : AzureDevOpsTask
    {
        public int[] TestRunIds { get; set; }
 
        public ITaskItem[] ExpectedTestFailures { get; set; }
 
        public string EnableFlakyTestSupport { get; set; }
 
        [Required]
        public ITaskItem[] WorkItems { get; set; }
 
        protected override async Task ExecuteCoreAsync(HttpClient client)
        {
            if (ExpectedTestFailures?.Length > 0)
            {
                await ValidateExpectedTestFailuresAsync(client);
                return;
            }
 
            if (!string.IsNullOrEmpty(EnableFlakyTestSupport))
            {
                await CheckTestResultsWithFlakySupport(client);
                return;
            }
 
 
            await CheckTestResultsAsync(client);
        }
 
        private async Task CheckTestResultsAsync(HttpClient client)
        {
            foreach (int testRunId in TestRunIds)
            {
                bool runComplete = false;
                int triesToWait = 3;
                JObject data = null;
 
                do
                {
                    data = await RetryAsync(
                    async () =>
                    {
                        using var req = new HttpRequestMessage(
                            HttpMethod.Get,
                            $"{CollectionUri}{TeamProject}/_apis/test/runs/{testRunId}?api-version=6.0");
                        using HttpResponseMessage res = await client.SendAsync(req);
                        return await ParseResponseAsync(req, res);
                    });
                    // This retry does not use the RetryAsync() function as that one only retries for network/timeout issues
                    triesToWait--;
                    runComplete = CheckAzurePipelinesTestRunIsComplete(data);
                    if (!runComplete && triesToWait > 0)
                    {
                        Log.LogWarning($"Test run {testRunId} is not in completed state.  Will check back in 10 seconds.");
                        await Task.Delay(10000);
                    }
                }
                while (!runComplete && triesToWait > 0);
 
                if (data != null && data["runStatistics"] is JArray runStatistics)
                {
                    var failed = runStatistics.Children()
                        .FirstOrDefault(stat => stat["outcome"]?.ToString() == "Failed");
                    if (failed != null)
                    {
                        await LogErrorsForFailedRun(client, testRunId);
                    }
                    else
                    {
                        Log.LogMessage(MessageImportance.Low, $"Test run {testRunId} has not failed.");
                    }
                }
            }
        }
 
        private bool CheckAzurePipelinesTestRunIsComplete(JObject data)
        {
            // Context: https://github.com/dotnet/arcade/issues/11942
            // it seems it's possible if checking immediately after a run is closed to not see all results
            // Since we pass/fail build tasks based off failed test items, it's very important that we not miss this.
            // This check will add logging if /_apis/test/runs/ manages to get called while incomplete.
            if (data == null)
            {
                return false;
            }
            var stateCompleted = data["state"]?.Value<string>()?.Equals("Completed");
            var postProcessStateCompleted = data["postProcessState"]?.Value<string>()?.Equals("Complete");
 
            return (stateCompleted == true && postProcessStateCompleted == true);
        }
 
        private async Task LogErrorsForFailedRun(HttpClient client, int testRunId)
        {
            JObject data = await RetryAsync(
                async () =>
                {
                    using var req = new HttpRequestMessage(
                        HttpMethod.Get,
                        $"{CollectionUri}{TeamProject}/_apis/test/runs/{testRunId}/results?outcomes=Failed&$top=100&api-version=6.0");
                    using HttpResponseMessage res = await client.SendAsync(req);
                    return await ParseResponseAsync(req, res);
                });
            int count = data.Value<int>("count");
            IEnumerable<JObject> entries = data.Value<JArray>("value").Cast<JObject>();
            if (count == 0)
            {
                Log.LogError(FailureCategory.Test, $"Test run {testRunId} has one or more failing tests based on run statistics, but I couldn't find the failures.");
                return;
            }
 
            foreach (JObject result in entries)
            {
                string name = result.Value<string>("automatedTestName");
                string comment = result.Value<string>("comment");
                JObject helixData;
                try
                {
                    helixData = JObject.Parse(comment);
                }
                catch (JsonException)
                {
                    helixData = null;
                }
                string jobId = helixData?.Value<string>("HelixJobId");
                string workItemName = helixData?.Value<string>("HelixWorkItemName");
                ITaskItem workItem = null;
                if (helixData != null && !string.IsNullOrEmpty(jobId) && !string.IsNullOrEmpty(workItemName))
                {
                    workItem = WorkItems.FirstOrDefault(t =>
                        t.GetMetadata("JobName") == jobId && t.GetMetadata("WorkItemName") == workItemName);
                }
 
                if (workItem != null)
                {
                    Log.LogError(FailureCategory.Test, $"Test {name} has failed. Check the Test tab or this console log: {workItem.GetMetadata("ConsoleOutputUri")}");
                }
                else
                {
                    Log.LogError(FailureCategory.Test, $"Test {name} has failed. Check the Test tab for details.");
                }
            }
        }
 
        private async Task CheckTestResultsWithFlakySupport(HttpClient client)
        {
            JObject data = await RetryAsync(
                async () =>
                {
                    using (var req = new HttpRequestMessage(
                        HttpMethod.Get,
                        $"{CollectionUri}{TeamProject}/_apis/test/resultsummarybybuild?buildId={BuildId}&api-version=5.1-preview.2")
                    )
                    {
                        using (HttpResponseMessage res = await client.SendAsync(req))
                        {
                            return await ParseResponseAsync(req, res);
                        }
                    }
                });
 
            if (data != null && data["aggregatedResultsAnalysis"] is JObject aggregatedResultsAnalysis &&
                aggregatedResultsAnalysis["resultsByOutcome"] is JObject resultsByOutcome)
            {
                foreach (JProperty property in resultsByOutcome.Properties())
                {
                    string outcome = property.Name.ToLowerInvariant();
                    var outcomeResults = (JObject) property.Value;
                    int count = outcomeResults["count"].ToObject<int>();
                    var message = $"Build has {count} {outcome} tests.";
                    if (outcome == "failed")
                    {
                        Log.LogError(FailureCategory.Test, message);
                    }
                    else
                    {
                        Log.LogMessage(MessageImportance.High, message);
                    }
                }
            }
            else
            {
                Log.LogError(FailureCategory.Helix, "Unable to get test report from build.");
            }
        }
 
        private async Task ValidateExpectedTestFailuresAsync(HttpClient client)
        {
            foreach (var runId in TestRunIds)
            {
                JObject data = await RetryAsync(
                    async () =>
                    {
                        using (var req = new HttpRequestMessage(
                            HttpMethod.Get,
                            $"{CollectionUri}{TeamProject}/_apis/test/runs/{runId}/results?api-version=5.0&outcomes=Failed")
                        )
                        {
                            using (HttpResponseMessage res = await client.SendAsync(req))
                            {
                                return await ParseResponseAsync(req, res);
                            }
                        }
                    });
 
                if (data != null)
                {
                    var failedResults = (JArray)data["value"];
                    HashSet<string> expectedFailures = ExpectedTestFailures?.Select(i => i.GetMetadata("Identity")).ToHashSet() ?? new HashSet<string>();
                    foreach (var failedResult in failedResults)
                    {
                        var testName = (string)failedResult["automatedTestName"];
                        if (expectedFailures.Contains(testName))
                        {
                            expectedFailures.Remove(testName);
                            Log.LogMessage($"TestRun {runId}: Test {testName} has failed and was expected to fail.");
                        }
                        else
                        {
                            Log.LogError(FailureCategory.Test, $"TestRun {runId}: Test {testName} has failed and is not expected to fail.");
                        }
                    }
 
                    foreach (string expectedFailure in expectedFailures)
                    {
                        Log.LogError(FailureCategory.Test, $"TestRun {runId}: Test {expectedFailure} was expected to fail but did not fail.");
                    }
                }
            }
        }
    }
}