6 references to WordTokenize
Microsoft.Extensions.AI.Evaluation.NLP (6)
BLEUEvaluator.cs (2)
84
string[][] references = context.References.Select(reference => SimpleWordTokenizer.
WordTokenize
(reference).ToArray()).ToArray();
85
string[] hypothesis = SimpleWordTokenizer.
WordTokenize
(modelResponse.Text).ToArray();
F1Evaluator.cs (2)
75
string[] reference = SimpleWordTokenizer.
WordTokenize
(context.GroundTruth).ToArray();
76
string[] hypothesis = SimpleWordTokenizer.
WordTokenize
(modelResponse.Text).ToArray();
GLEUEvaluator.cs (2)
84
string[][] references = context.References.Select(reference => SimpleWordTokenizer.
WordTokenize
(reference).ToArray()).ToArray();
85
string[] hypothesis = SimpleWordTokenizer.
WordTokenize
(modelResponse.Text).ToArray();