| File: ValidationRules.cs | Web Access |
| Project: src\tools\AnalyzeCiFailure\AnalyzeCiFailure.csproj (AnalyzeCiFailure) |
// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. using System.Collections.Immutable; using static AnalyzeCiFailure.CiFailureJson; namespace AnalyzeCiFailure; /// <summary>Enforces evidence coverage, cause compatibility, identity and attribution rules for a diagnosis.</summary> internal static class ValidationRules { internal const int MaximumIndividualCauseIssues = 10; internal const int MaximumCommentBytes = 65000; internal const string TestMismatch = "Analysis failed_tests do not match trusted test failure evidence"; internal static void RequirePermittedVerdict(AnalysisReport analysis, ValidatedRun run) { var permitted = run.Scope switch { RunScope.Main => analysis.Verdict is AnalysisVerdict.TransientInfrastructure or AnalysisVerdict.FlakyTest or AnalysisVerdict.MainRepositoryBreakage or AnalysisVerdict.Mixed, RunScope.PullRequest => analysis.Verdict is AnalysisVerdict.TransientInfrastructure or AnalysisVerdict.FlakyTest or AnalysisVerdict.CodeIssue or AnalysisVerdict.Mixed, _ => false, }; if (!permitted) { throw new ValidationException($"Verdict {EnumText(analysis.Verdict.GetValueOrDefault())} is not permitted for run scope {EnumText(run.Scope)}"); } } internal static void RequireExactJobCoverage(AnalysisReport analysis, ImmutableArray<FailedJob> jobs, ValidatedRun run) { if (!(analysis.Jobs.Length > 0 && analysis.Jobs.All(j => j.Classification is JobClassification.TransientInfrastructure or JobClassification.FlakyTest or JobClassification.CodeIssue or JobClassification.MainRepositoryBreakage))) { throw new ValidationException("Analysis must classify every failed job with a recognized classification"); } var ids = analysis.Jobs.Select(j => j.NormalizedId).ToArray(); // Failed-job summaries must match the trusted numeric representation as well as its // value. Cause references and uniqueness use mathematical numeric identity instead. if (!(ids.Distinct().Count() == ids.Length && ids.Length == jobs.Length && analysis.Jobs.All(reported => jobs.Any(collected => reported.NormalizedId == collected.Id && reported.SerializedId == collected.SerializedId)))) { throw new ValidationException("Analysis failed-job IDs do not match the trusted failed jobs"); } if (analysis.Jobs.Any(j => (run.Scope == RunScope.Main && j.Classification == JobClassification.CodeIssue) || (run.Scope == RunScope.PullRequest && j.Classification == JobClassification.MainRepositoryBreakage))) { throw new ValidationException($"Analysis contains a failed-job classification that is not permitted for run scope {EnumText(run.Scope)}"); } } internal static ImmutableArray<int> RequireExactTestCoverage(AnalysisReport analysis, ImmutableArray<TestIdentity> collected, ImmutableArray<FailedJob> jobs) { var matched = ImmutableArray.CreateBuilder<int>(); var used = new HashSet<int>(); foreach (var test in analysis.Tests) { var candidates = Enumerable.Range(start: 0, collected.Length).Where(i => collected[i].Job == test.Job && test.NormalizedJobId is not null && collected[i].Name == test.Name && collected[i].JobId == test.NormalizedJobId && collected[i].ObservationId == test.ObservationId).ToArray(); if (!(candidates.Length == 1 && used.Add(candidates[0]))) { throw new ValidationException(TestMismatch); } matched.Add(candidates[0]); } if (!(used.Count == collected.Length && analysis.Tests.All(t => jobs.Any(j => j.Name == t.Job)))) { throw new ValidationException(TestMismatch); } return matched.ToImmutable(); } internal static void ValidateCauseCompatibility(FailureCause cause, AnalysisReport analysis, ImmutableArray<FailedJob> jobs, ValidatedRun run) { if (!analysis.Causes.Contains(cause.Id)) { throw new ValidationException($"Cause {Display(cause.FileName)} is not referenced by the analysis summary"); } if (!(cause.Type is CauseType.FlakyTest or CauseType.InfrastructureFailure || (run.Scope == RunScope.Main && cause.Type == CauseType.MainRepositoryBreakage))) { throw new ValidationException($"Cause {Display(cause.FileName)} type {EnumText(cause.Type)} is not permitted for run scope {EnumText(run.Scope)}"); } foreach (var jobId in cause.JobIds) { var job = jobs.FirstOrDefault(j => j.Id == jobId); var classification = analysis.Jobs.FirstOrDefault(j => j.NormalizedId == jobId)?.Classification; var hasFlakyTestInJob = job is not null && analysis.Tests.Any(t => t.Classification == TestClassification.Flaky && t.NormalizedJobId == job.Id); var compatible = cause.Type switch { CauseType.InfrastructureFailure => classification == JobClassification.TransientInfrastructure, CauseType.MainRepositoryBreakage => classification == JobClassification.MainRepositoryBreakage, CauseType.FlakyTest => classification == JobClassification.FlakyTest || (classification is JobClassification.CodeIssue or JobClassification.MainRepositoryBreakage && hasFlakyTestInJob), _ => false, }; if (!(job is not null && compatible)) { throw new ValidationException(IncompatibleJob(cause.FileName)); } } } internal static void RequireCauseFields(FailureCause cause) { if (!(cause.JobIds.Length > 0 && cause.JobIds.Distinct().Count() == cause.JobIds.Length && (cause.Type == CauseType.FlakyTest ? !cause.Tests.IsDefault : cause.Tests.IsDefault))) { throw new ValidationException($"Cause {Display(cause.FileName)} contains unsupported or publisher-owned fields"); } } internal static void RequireUnchangedCauseType(FailureCause cause, StoredCauseIdentity prior) { if (!(prior.Type == cause.Type)) { throw new ValidationException($"Cause {Display(cause.FileName)} cannot change type from {EnumText(prior.Type)} to {EnumText(cause.Type)}"); } } internal static void ValidateVerdict(AnalysisReport analysis, ImmutableArray<FailureCause> causes, ValidatedRun run) { var infraJobs = analysis.Jobs.Count(j => j.Classification == JobClassification.TransientInfrastructure); var flakyJobs = analysis.Jobs.Count(j => j.Classification == JobClassification.FlakyTest); var codeJobs = analysis.Jobs.Count(j => j.Classification == JobClassification.CodeIssue); var mainJobs = analysis.Jobs.Count(j => j.Classification == JobClassification.MainRepositoryBreakage); var hasFlakyTests = analysis.Tests.Any(t => t.Classification == TestClassification.Flaky); var hasFlakyEvidence = flakyJobs > 0 || hasFlakyTests; var hasTransientEvidence = infraJobs > 0 || hasFlakyEvidence; var infraCauses = causes.Count(c => c.Type == CauseType.InfrastructureFailure); var flakyCauses = causes.Count(c => c.Type == CauseType.FlakyTest); var mainCauses = causes.Count(c => c.Type == CauseType.MainRepositoryBreakage); switch (analysis.Verdict) { case AnalysisVerdict.TransientInfrastructure: if (analysis.Tests.Any(test => test.Classification != TestClassification.TransientInfrastructure)) { throw new ValidationException("Analysis failed_tests are incompatible with verdict transient-infra"); } if (!(infraJobs == analysis.Jobs.Length && causes.Length > 0 && infraCauses == causes.Length)) { throw new ValidationException("A transient-infra verdict requires every failed job and cause to be an infrastructure failure"); } break; case AnalysisVerdict.FlakyTest: if (analysis.Tests.Any(t => t.Classification == TestClassification.CodeIssue)) { throw new ValidationException("Analysis failed_tests are incompatible with verdict flaky-test"); } if (!(flakyJobs > 0 && infraJobs + flakyJobs == analysis.Jobs.Length && causes.Length > 0 && flakyCauses > 0 && mainCauses == 0)) { throw new ValidationException("A flaky-test verdict requires at least one flaky job, only transient failed jobs, and only transient causes"); } break; case AnalysisVerdict.CodeIssue: if (hasFlakyTests) { throw new ValidationException("Analysis failed_tests are incompatible with verdict code-issue"); } if (!(codeJobs == analysis.Jobs.Length && causes.Length == 0)) { throw new ValidationException("A code-issue verdict requires every failed job to be a code issue and must not include cause files"); } break; case AnalysisVerdict.MainRepositoryBreakage: if (hasFlakyTests) { throw new ValidationException("Analysis failed_tests are incompatible with verdict main-repository-breakage"); } if (!(mainJobs == analysis.Jobs.Length && mainCauses > 0 && mainCauses == causes.Length)) { throw new ValidationException("A main-repository-breakage verdict requires every failed job and cause to be a main repository breakage"); } break; case AnalysisVerdict.Mixed when run.Scope == RunScope.Main: if (!(mainJobs > 0 && hasTransientEvidence && mainCauses > 0 && mainCauses != causes.Length)) { throw new ValidationException("A mixed verdict for main requires a main-breakage job and cause plus transient job or test evidence and cause"); } break; case AnalysisVerdict.Mixed: if (!(codeJobs > 0 && hasTransientEvidence && causes.Length > 0)) { throw new ValidationException("A mixed verdict for a pull request requires a code-issue job plus transient job or test evidence and a transient cause"); } break; } RequireCauseClassificationCoverage(analysis, causes); } internal static void RequireCauseClassificationCoverage(AnalysisReport analysis, ImmutableArray<FailureCause> causes) { var infraJobs = analysis.Jobs.Count(job => job.Classification == JobClassification.TransientInfrastructure); var mainJobs = analysis.Jobs.Count(job => job.Classification == JobClassification.MainRepositoryBreakage); var hasFlakyEvidence = analysis.Jobs.Any(job => job.Classification == JobClassification.FlakyTest) || analysis.Tests.Any(test => test.Classification == TestClassification.Flaky); var infraCauses = causes.Count(cause => cause.Type == CauseType.InfrastructureFailure); var flakyCauses = causes.Count(cause => cause.Type == CauseType.FlakyTest); var mainCauses = causes.Count(cause => cause.Type == CauseType.MainRepositoryBreakage); if (!((infraJobs > 0) == (infraCauses > 0) && hasFlakyEvidence == (flakyCauses > 0) && (mainJobs > 0) == (mainCauses > 0))) { throw new ValidationException("Failed-job classifications and persisted cause types do not match"); } } internal static void RequireCauseCoverage(AnalysisReport analysis, ImmutableArray<FailureCause> causes) { foreach (var test in analysis.Tests.Where(test => test.Classification == TestClassification.TransientInfrastructure)) { if (!analysis.Jobs.Any(job => job.NormalizedId == test.NormalizedJobId && job.Classification == JobClassification.TransientInfrastructure) || !causes.Any(cause => cause.Type == CauseType.InfrastructureFailure && cause.JobIds.Contains(test.NormalizedJobId!.Value))) { throw new ValidationException("Every infrastructure test observation must belong to an infrastructure job covered by an infrastructure cause."); } } foreach (var job in analysis.Jobs) { CauseType? type = job.Classification switch { JobClassification.TransientInfrastructure => CauseType.InfrastructureFailure, JobClassification.FlakyTest => CauseType.FlakyTest, JobClassification.MainRepositoryBreakage => CauseType.MainRepositoryBreakage, _ => null, }; if (!(type is null || causes.Any(c => c.Type == type && c.JobIds.Contains(job.NormalizedId)))) { throw new ValidationException("Every transient, flaky, and main-breakage failed job must be covered by a matching cause"); } } var flaky = analysis.Tests.Where(t => t.Classification == TestClassification.Flaky).ToArray(); foreach (var cause in causes.Where(c => c.Type == CauseType.FlakyTest)) { if (!cause.Tests.All(o => cause.JobIds.Contains(o.JobId) && flaky.Any(t => t.Name == o.Name && t.NormalizedJobId == o.JobId && t.ObservationId == o.ObservationId))) { throw new ValidationException("Cause test observation must reference a validated flaky test in its trusted job"); } if (!cause.JobIds.All(id => cause.Tests.Any(o => o.JobId == id))) { throw new ValidationException("Every cause job must contain a referenced test observation"); } } // With no persisted causes, verdict validation determines whether analysis can // publish. Test-to-cause joins apply only when causes are present. if (causes.Length > 0) { if (!flaky.All(t => causes.Any(c => c.Type == CauseType.FlakyTest && c.Tests.Any(o => o.Name == t.Name && o.JobId == t.NormalizedJobId && o.ObservationId == t.ObservationId)))) { throw new ValidationException("Every flaky test and job observation must be covered by a matching cause"); } } } internal static bool UsesAggregatePublication(int causeCount) => causeCount > MaximumIndividualCauseIssues; internal static void RequireCommentBudget(int? commentBytes) { if (!(commentBytes is not null)) { throw new ValidationException("Unable to render the PR comment during validation"); } if (!(commentBytes <= MaximumCommentBytes)) { throw new ValidationException("Rendered PR comment exceeds the 65000-byte publication budget"); } } private static string IncompatibleJob(string name) => $"Cause {Display(name)} references an unknown or incompatible failed job"; }