2 references to SimpleWordTokenizer
Microsoft.Extensions.AI.Evaluation.NLP (2)
BLEUEvaluator.cs (2)
82
var references = context.References.Select(reference =>
SimpleWordTokenizer
.WordTokenize(reference));
83
var hypothesis =
SimpleWordTokenizer
.WordTokenize(modelResponse.Text);