3 references to CountTokens
Microsoft.ML.Tokenizers.Tests (3)
UnigramTests.cs (3)
783
Assert.Equal(2, tokenizer.
CountTokens
("a b", addBeginningOfSentence: false, addEndOfSentence: false));
810
Assert.Equal(2, tokenizer.
CountTokens
("a b", addBeginningOfSentence: false, addEndOfSentence: false));
833
Assert.Equal(3, tokenizer.
CountTokens
("a b", addBeginningOfSentence: false, addEndOfSentence: false));