9 references to CountTokens
Microsoft.ML.Tokenizers.Tests (9)
BpeTests.cs (1)
458
Assert.Equal(expectedIds.Length, tokenizer.
CountTokens
(text.AsSpan()));
CodeGenTests.cs (4)
429
Assert.Equal(ids.Length, codeGenTokenizer.
CountTokens
(text.AsSpan()));
631
count = codeGenTokenizer.
CountTokens
(text.AsSpan());
764
count = codeGenTokenizer.
CountTokens
(text.AsSpan());
919
count = codeGenTokenizer.
CountTokens
(text.AsSpan());
EnglishRobertaTests.cs (1)
210
Assert.Equal(expectedIds.Length, tokenizer.
CountTokens
(text.AsSpan()));
LlamaTests.cs (2)
345
Assert.Equal(0, llamaTokenizer.
CountTokens
(Span<char>.Empty));
633
Assert.Equal(expectedIds.Length, tokenizer.
CountTokens
(text.AsSpan()));
TiktokenTests.cs (1)
597
Assert.Equal(expectedIds.Length, tokenizer.
CountTokens
(text.AsSpan()));