6 references to SimpleWordTokenizer
Microsoft.Extensions.AI.Evaluation.NLP (6)
BLEUEvaluator.cs (2)
84
string[][] references = context.References.Select(reference =>
SimpleWordTokenizer
.WordTokenize(reference).ToArray()).ToArray();
85
string[] hypothesis =
SimpleWordTokenizer
.WordTokenize(modelResponse.Text).ToArray();
F1Evaluator.cs (2)
75
string[] reference =
SimpleWordTokenizer
.WordTokenize(context.GroundTruth).ToArray();
76
string[] hypothesis =
SimpleWordTokenizer
.WordTokenize(modelResponse.Text).ToArray();
GLEUEvaluator.cs (2)
84
string[][] references = context.References.Select(reference =>
SimpleWordTokenizer
.WordTokenize(reference).ToArray()).ToArray();
85
string[] hypothesis =
SimpleWordTokenizer
.WordTokenize(modelResponse.Text).ToArray();