2 references to WordTokenize
Microsoft.Extensions.AI.Evaluation.NLP (2)
BLEUEvaluator.cs (2)
82
var references = context.References.Select(reference => SimpleWordTokenizer.
WordTokenize
(reference));
83
var hypothesis = SimpleWordTokenizer.
WordTokenize
(modelResponse.Text);