WordTokenize

22 references to WordTokenize

Microsoft.Extensions.AI.Evaluation.NLP (6)

BLEUEvaluator.cs (2)

86reference => SimpleWordTokenizer.WordTokenize(reference).ToArray()).ToArray(); 88string[] hypothesis = SimpleWordTokenizer.WordTokenize(modelResponse.Text).ToArray();

F1Evaluator.cs (2)

75string[] reference = SimpleWordTokenizer.WordTokenize(context.GroundTruth).ToArray(); 76string[] hypothesis = SimpleWordTokenizer.WordTokenize(modelResponse.Text).ToArray();

GLEUEvaluator.cs (2)

85reference => SimpleWordTokenizer.WordTokenize(reference).ToArray()).ToArray(); 87string[] hypothesis = SimpleWordTokenizer.WordTokenize(modelResponse.Text).ToArray();

Microsoft.Extensions.AI.Evaluation.NLP.Tests (16)

BLEUAlgorithmTests.cs (6)

203SimpleWordTokenizer.WordTokenize("It is a guide to action that ensures that the military will forever heed Party commands").ToArray(), 204SimpleWordTokenizer.WordTokenize("It is the guiding principle which guarantees the military forces always being under the command of the Party").ToArray(), 205SimpleWordTokenizer.WordTokenize("It is the practical guide for the army always to heed the directions of the party").ToArray(), 207string[] hypothesis = SimpleWordTokenizer.WordTokenize("It is a guide to action which ensures that the military always obeys the commands of the party").ToArray(); 218SimpleWordTokenizer.WordTokenize("he was interested in world history because he read the book").ToArray(), 220string[] hypothesis = SimpleWordTokenizer.WordTokenize("he read the book because he was interested in world history").ToArray();

GLEUAlgorithmTests.cs (6)

127SimpleWordTokenizer.WordTokenize("It is a guide to action that ensures that the military will forever heed Party commands").ToArray(), 128SimpleWordTokenizer.WordTokenize("It is the guiding principle which guarantees the military forces always being under the command of the Party").ToArray(), 129SimpleWordTokenizer.WordTokenize("It is the practical guide for the army always to heed the directions of the party").ToArray(), 131string[] hypothesis = SimpleWordTokenizer.WordTokenize("It is a guide to action which ensures that the military always obeys the commands of the party").ToArray(); 142SimpleWordTokenizer.WordTokenize("he was interested in world history because he read the book").ToArray(), 144string[] hypothesis = SimpleWordTokenizer.WordTokenize("he read the book because he was interested in world history").ToArray();

SimpleTokenizerTests.cs (4)

49var result = SimpleWordTokenizer.WordTokenize(input); 58var result = SimpleWordTokenizer.WordTokenize(input); 67var result = SimpleWordTokenizer.WordTokenize(input); 76var result = SimpleWordTokenizer.WordTokenize(input);