6 references to SimpleWordTokenizer
Microsoft.Extensions.AI.Evaluation.NLP (6)
BLEUEvaluator.cs (2)
84string[][] references = context.References.Select(reference => SimpleWordTokenizer.WordTokenize(reference).ToArray()).ToArray(); 85string[] hypothesis = SimpleWordTokenizer.WordTokenize(modelResponse.Text).ToArray();
F1Evaluator.cs (2)
75string[] reference = SimpleWordTokenizer.WordTokenize(context.GroundTruth).ToArray(); 76string[] hypothesis = SimpleWordTokenizer.WordTokenize(modelResponse.Text).ToArray();
GLEUEvaluator.cs (2)
84string[][] references = context.References.Select(reference => SimpleWordTokenizer.WordTokenize(reference).ToArray()).ToArray(); 85string[] hypothesis = SimpleWordTokenizer.WordTokenize(modelResponse.Text).ToArray();