6 references to SimpleWordTokenizer
Microsoft.Extensions.AI.Evaluation.NLP (6)
BLEUEvaluator.cs (2)
86
reference =>
SimpleWordTokenizer
.WordTokenize(reference).ToArray()).ToArray();
88
string[] hypothesis =
SimpleWordTokenizer
.WordTokenize(modelResponse.Text).ToArray();
F1Evaluator.cs (2)
75
string[] reference =
SimpleWordTokenizer
.WordTokenize(context.GroundTruth).ToArray();
76
string[] hypothesis =
SimpleWordTokenizer
.WordTokenize(modelResponse.Text).ToArray();
GLEUEvaluator.cs (2)
85
reference =>
SimpleWordTokenizer
.WordTokenize(reference).ToArray()).ToArray();
87
string[] hypothesis =
SimpleWordTokenizer
.WordTokenize(modelResponse.Text).ToArray();