3 instantiations of EvaluationDiagnostic
Microsoft.Extensions.AI.Evaluation (3)
EvaluationDiagnostic.cs (3)
45
=> new
EvaluationDiagnostic
(EvaluationDiagnosticSeverity.Informational, message);
57
=> new
EvaluationDiagnostic
(EvaluationDiagnosticSeverity.Warning, message);
69
=> new
EvaluationDiagnostic
(EvaluationDiagnosticSeverity.Error, message);
165 references to EvaluationDiagnostic
Microsoft.Extensions.AI.Evaluation (47)
CompositeEvaluator.cs (1)
161
metric.AddDiagnostics(
EvaluationDiagnostic
.Error(message));
EvaluationDiagnostic.cs (18)
17
/// <see cref="
EvaluationDiagnostic
"/>.
20
/// An error, warning or informational message describing the <see cref="
EvaluationDiagnostic
"/>.
26
/// <see cref="
EvaluationDiagnostic
"/>.
31
/// Gets or sets an error, warning or informational message describing the <see cref="
EvaluationDiagnostic
"/>.
36
/// Returns an <see cref="
EvaluationDiagnostic
"/> with the supplied <paramref name="message"/> and with
39
/// <param name="message">An informational message describing the <see cref="
EvaluationDiagnostic
"/>.</param>
41
/// An <see cref="
EvaluationDiagnostic
"/> with <see cref="Severity"/> set to
44
public static
EvaluationDiagnostic
Informational(string message)
48
/// Returns an <see cref="
EvaluationDiagnostic
"/> with the supplied <paramref name="message"/> and with
51
/// <param name="message">A warning message describing the <see cref="
EvaluationDiagnostic
"/>.</param>
53
/// An <see cref="
EvaluationDiagnostic
"/> with <see cref="Severity"/> set to
56
public static
EvaluationDiagnostic
Warning(string message)
60
/// Returns an <see cref="
EvaluationDiagnostic
"/> with the supplied <paramref name="message"/> and with
63
/// <param name="message">An error message describing the <see cref="
EvaluationDiagnostic
"/>.</param>
65
/// An <see cref="
EvaluationDiagnostic
"/> with <see cref="Severity"/> set to
68
public static
EvaluationDiagnostic
Error(string message)
71
/// <summary>Returns a string representation of the <see cref="
EvaluationDiagnostic
"/>.</summary>
72
/// <returns>A string representation of the <see cref="
EvaluationDiagnostic
"/>.</returns>
EvaluationDiagnosticSeverity.cs (4)
7
/// An enumeration that identifies the set of possible values for <see cref="
EvaluationDiagnostic
.Severity"/>.
12
/// A value that indicates that the <see cref="
EvaluationDiagnostic
"/> is informational.
17
/// A value that indicates that the <see cref="
EvaluationDiagnostic
"/> represents a warning.
22
/// A value that indicates that the <see cref="
EvaluationDiagnostic
"/> represents an error.
EvaluationMetric.cs (2)
58
/// Gets or sets a collection of zero or more <see cref="
EvaluationDiagnostic
"/>s associated with the current
61
public IList<
EvaluationDiagnostic
>? Diagnostics { get; set; }
EvaluationMetricExtensions.cs (13)
50
/// <see cref="
EvaluationDiagnostic
"/> matching the supplied <paramref name="predicate"/>.
54
/// A predicate that returns <see langword="true"/> if a matching <see cref="
EvaluationDiagnostic
"/> is found;
59
/// <see cref="
EvaluationDiagnostic
"/> matching the supplied <paramref name="predicate"/>; <see langword="false"/>
64
Func<
EvaluationDiagnostic
, bool>? predicate = null)
76
/// Adds the supplied <see cref="
EvaluationDiagnostic
"/>s to the supplied <see cref="EvaluationMetric"/>'s
80
/// <param name="diagnostics">The <see cref="
EvaluationDiagnostic
"/>s to be added.</param>
81
public static void AddDiagnostics(this EvaluationMetric metric, IEnumerable<
EvaluationDiagnostic
> diagnostics)
88
metric.Diagnostics ??= new List<
EvaluationDiagnostic
>();
90
foreach (
EvaluationDiagnostic
diagnostic in diagnostics)
98
/// Adds the supplied <see cref="
EvaluationDiagnostic
"/>s to the supplied <see cref="EvaluationMetric"/>'s
102
/// <param name="diagnostics">The <see cref="
EvaluationDiagnostic
"/>s to be added.</param>
103
public static void AddDiagnostics(this EvaluationMetric metric, params
EvaluationDiagnostic
[] diagnostics)
104
=> metric.AddDiagnostics(diagnostics as IEnumerable<
EvaluationDiagnostic
>);
EvaluationResultExtensions.cs (9)
54
/// <param name="diagnostics">The <see cref="
EvaluationDiagnostic
"/>s that are to be added.</param>
55
public static void AddDiagnosticsToAllMetrics(this EvaluationResult result, IEnumerable<
EvaluationDiagnostic
> diagnostics)
72
/// <param name="diagnostics">The <see cref="
EvaluationDiagnostic
"/>s that are to be added.</param>
73
public static void AddDiagnosticsToAllMetrics(this EvaluationResult result, params
EvaluationDiagnostic
[] diagnostics)
74
=> AddDiagnosticsToAllMetrics(result, diagnostics as IEnumerable<
EvaluationDiagnostic
>);
78
/// <paramref name="result"/> contains an <see cref="
EvaluationDiagnostic
"/> matching the supplied
83
/// A predicate that returns <see langword="true"/> if a matching <see cref="
EvaluationDiagnostic
"/> is found;
88
/// <paramref name="result"/> contains an <see cref="
EvaluationDiagnostic
"/> matching the supplied
93
Func<
EvaluationDiagnostic
, bool>? predicate = null)
Microsoft.Extensions.AI.Evaluation.Integration.Tests (39)
ResultsTests.cs (39)
423
metric1.AddDiagnostics(
EvaluationDiagnostic
.Error("Error 1"));
424
metric1.AddDiagnostics(
EvaluationDiagnostic
.Error("Error 2"));
425
metric1.AddDiagnostics(
EvaluationDiagnostic
.Warning("Warning 1"));
426
metric1.AddDiagnostics(
EvaluationDiagnostic
.Informational("Informational 1"));
427
metric1.AddDiagnostics(
EvaluationDiagnostic
.Informational("Informational 2"));
431
metric2.AddDiagnostics(
EvaluationDiagnostic
.Warning("Warning 1"));
432
metric2.AddDiagnostics(
EvaluationDiagnostic
.Warning("Warning 2"));
433
metric2.AddDiagnostics(
EvaluationDiagnostic
.Informational("Informational 2"));
437
metric3.AddDiagnostics(
EvaluationDiagnostic
.Error("Error 1"));
438
metric3.AddDiagnostics(
EvaluationDiagnostic
.Error("Error 2"));
443
metric4.AddDiagnostics(
EvaluationDiagnostic
.Warning("Warning 1"));
444
metric4.AddDiagnostics(
EvaluationDiagnostic
.Warning("Warning 2"));
448
metric5.AddDiagnostics(
EvaluationDiagnostic
.Informational("Informational 1"));
476
metric1.AddDiagnostics(
EvaluationDiagnostic
.Error("Error 1"));
477
metric1.AddDiagnostics(
EvaluationDiagnostic
.Error("Error 2"));
478
metric1.AddDiagnostics(
EvaluationDiagnostic
.Warning("Warning 1"));
479
metric1.AddDiagnostics(
EvaluationDiagnostic
.Informational("Informational 1"));
480
metric1.AddDiagnostics(
EvaluationDiagnostic
.Informational("Informational 2"));
484
metric2.AddDiagnostics(
EvaluationDiagnostic
.Warning("Warning 1"));
485
metric2.AddDiagnostics(
EvaluationDiagnostic
.Warning("Warning 2"));
486
metric2.AddDiagnostics(
EvaluationDiagnostic
.Informational("Informational 2"));
490
metric3.AddDiagnostics(
EvaluationDiagnostic
.Error("Error 1"));
491
metric3.AddDiagnostics(
EvaluationDiagnostic
.Error("Error 2"));
496
metric4.AddDiagnostics(
EvaluationDiagnostic
.Warning("Warning 1"));
497
metric4.AddDiagnostics(
EvaluationDiagnostic
.Warning("Warning 2"));
501
metric5.AddDiagnostics(
EvaluationDiagnostic
.Informational("Informational 1"));
535
metric1.AddDiagnostics(
EvaluationDiagnostic
.Error("Error 1"));
536
metric1.AddDiagnostics(
EvaluationDiagnostic
.Error("Error 2"));
537
metric1.AddDiagnostics(
EvaluationDiagnostic
.Warning("Warning 1"));
538
metric1.AddDiagnostics(
EvaluationDiagnostic
.Informational("Informational 1"));
539
metric1.AddDiagnostics(
EvaluationDiagnostic
.Informational("Informational 2"));
543
metric2.AddDiagnostics(
EvaluationDiagnostic
.Warning("Warning 1"));
544
metric2.AddDiagnostics(
EvaluationDiagnostic
.Warning("Warning 2"));
545
metric2.AddDiagnostics(
EvaluationDiagnostic
.Informational("Informational 2"));
549
metric3.AddDiagnostics(
EvaluationDiagnostic
.Error("Error 1"));
550
metric3.AddDiagnostics(
EvaluationDiagnostic
.Error("Error 2"));
555
metric4.AddDiagnostics(
EvaluationDiagnostic
.Warning("Warning 1"));
556
metric4.AddDiagnostics(
EvaluationDiagnostic
.Warning("Warning 2"));
560
metric5.AddDiagnostics(
EvaluationDiagnostic
.Informational("Informational 1"));
Microsoft.Extensions.AI.Evaluation.NLP (3)
BLEUEvaluator.cs (3)
56
EvaluationDiagnostic
.Error($"The {nameof(modelResponse)} supplied for evaluation was null or empty."));
65
EvaluationDiagnostic
.Error(
74
EvaluationDiagnostic
.Error(
Microsoft.Extensions.AI.Evaluation.Quality (40)
CoherenceEvaluator.cs (1)
83
EvaluationDiagnostic
.Error($"The {nameof(modelResponse)} supplied for evaluation was null or empty."));
CompletenessEvaluator.cs (2)
80
EvaluationDiagnostic
.Error($"The {nameof(modelResponse)} supplied for evaluation was null or empty."));
89
EvaluationDiagnostic
.Error(
EquivalenceEvaluator.cs (2)
79
EvaluationDiagnostic
.Error($"The {nameof(modelResponse)} supplied for evaluation was null or empty."));
88
EvaluationDiagnostic
.Error(
EvaluationMetricExtensions.cs (4)
78
metric.AddDiagnostics(
EvaluationDiagnostic
.Informational(
90
EvaluationDiagnostic
.Error(
138
EvaluationDiagnostic
.Error(
161
EvaluationDiagnostic
.Error(
FluencyEvaluator.cs (1)
77
EvaluationDiagnostic
.Error($"The {nameof(modelResponse)} supplied for evaluation was null or empty."));
GroundednessEvaluator.cs (2)
78
EvaluationDiagnostic
.Error($"The {nameof(modelResponse)} supplied for evaluation was null or empty."));
87
EvaluationDiagnostic
.Error(
IntentResolutionEvaluator.cs (8)
91
EvaluationDiagnostic
.Error(
100
EvaluationDiagnostic
.Error(
112
EvaluationDiagnostic
.Error(
127
EvaluationDiagnostic
.Error(
133
EvaluationDiagnostic
.Error(
336
EvaluationDiagnostic
.Error("The model failed to produce a valid evaluation response."));
358
EvaluationDiagnostic
.Error(
373
EvaluationDiagnostic
.Error(
RelevanceEvaluator.cs (2)
81
EvaluationDiagnostic
.Error(
90
EvaluationDiagnostic
.Error($"The {nameof(modelResponse)} supplied for evaluation was null or empty."));
RelevanceTruthAndCompletenessEvaluator.cs (5)
99
EvaluationDiagnostic
.Error(
108
EvaluationDiagnostic
.Error($"The {nameof(modelResponse)} supplied for evaluation was null or empty."));
281
EvaluationDiagnostic
.Error("The model failed to produce a valid evaluation response."));
303
EvaluationDiagnostic
.Error(
318
EvaluationDiagnostic
.Error(
RetrievalEvaluator.cs (3)
86
EvaluationDiagnostic
.Error(
96
EvaluationDiagnostic
.Error(
105
EvaluationDiagnostic
.Error(
TaskAdherenceEvaluator.cs (5)
90
EvaluationDiagnostic
.Error(
99
EvaluationDiagnostic
.Error(
111
EvaluationDiagnostic
.Error(
126
EvaluationDiagnostic
.Error(
132
EvaluationDiagnostic
.Error(
ToolCallAccuracyEvaluator.cs (5)
92
EvaluationDiagnostic
.Error(
104
EvaluationDiagnostic
.Error($"The {nameof(modelResponse)} supplied for evaluation did not contain any tool calls (i.e., {nameof(FunctionCallContent)}s)."));
113
EvaluationDiagnostic
.Error(
122
EvaluationDiagnostic
.Error(
133
EvaluationDiagnostic
.Error(
Microsoft.Extensions.AI.Evaluation.Reporting (4)
ScenarioRunResultExtensions.cs (4)
16
/// <paramref name="result"/> contains an <see cref="
EvaluationDiagnostic
"/> matching the supplied
21
/// A predicate that returns <see langword="true"/> if a matching <see cref="
EvaluationDiagnostic
"/> is found;
26
/// <paramref name="result"/> contains an <see cref="
EvaluationDiagnostic
"/> matching the supplied
31
Func<
EvaluationDiagnostic
, bool>? predicate = null)
Microsoft.Extensions.AI.Evaluation.Reporting.Tests (15)
ResultStoreTester.cs (1)
25
numericMetric.AddDiagnostics(
EvaluationDiagnostic
.Informational("Informational Message"));
ScenarioRunResultTests.cs (14)
40
booleanMetric.AddDiagnostics(
EvaluationDiagnostic
.Error("error"));
41
booleanMetric.AddDiagnostics(
EvaluationDiagnostic
.Warning("warning"));
51
numericMetric.AddDiagnostics(
EvaluationDiagnostic
.Informational("info"));
59
metricWithNoValue.AddDiagnostics(
EvaluationDiagnostic
.Error("error"));
60
metricWithNoValue.AddDiagnostics(
EvaluationDiagnostic
.Informational("info"));
131
booleanMetric.AddDiagnostics(
EvaluationDiagnostic
.Error("error"));
132
booleanMetric.AddDiagnostics(
EvaluationDiagnostic
.Warning("warning"));
142
numericMetric.AddDiagnostics(
EvaluationDiagnostic
.Informational("info"));
150
metricWithNoValue.AddDiagnostics(
EvaluationDiagnostic
.Error("error"));
151
metricWithNoValue.AddDiagnostics(
EvaluationDiagnostic
.Informational("info"));
377
private class DiagnosticComparer : IEqualityComparer<
EvaluationDiagnostic
>
381
public bool Equals(
EvaluationDiagnostic
? x,
EvaluationDiagnostic
? y)
384
public int GetHashCode(
EvaluationDiagnostic
obj)
Microsoft.Extensions.AI.Evaluation.Safety (17)
ContentSafetyEvaluator.cs (1)
132
(string payload, IReadOnlyList<
EvaluationDiagnostic
>? diagnostics) =
ContentSafetyServicePayloadUtilities.cs (15)
15
internal static (string payload, IReadOnlyList<
EvaluationDiagnostic
>? diagnostics) GetPayload(
78
private static (string payload, IReadOnlyList<
EvaluationDiagnostic
>? diagnostics)
94
List<
EvaluationDiagnostic
>? diagnostics;
166
private static (string payload, IReadOnlyList<
EvaluationDiagnostic
>? diagnostics)
188
List<
EvaluationDiagnostic
>? diagnostics;
249
private static (string payload, IReadOnlyList<
EvaluationDiagnostic
>? diagnostics) GetConversationPayload(
266
List<
EvaluationDiagnostic
>? diagnostics;
390
List<
EvaluationDiagnostic
>? diagnostics,
490
List<
EvaluationDiagnostic
>? diagnostics = null;
495
EvaluationDiagnostic
.Warning(
505
EvaluationDiagnostic
.Warning(
517
EvaluationDiagnostic
.Warning(
526
EvaluationDiagnostic
.Warning(
541
EvaluationDiagnostic
.Warning(
555
EvaluationDiagnostic
.Warning(
EvaluationMetricExtensions.cs (1)
100
metric.AddDiagnostics(
EvaluationDiagnostic
.Informational(serializedData));