36 references to CountTokens
Microsoft.Extensions.AI.Integration.Tests (1)
ReducingChatClientTests.cs (1)
103
sum += _tokenizer.
CountTokens
(text.Text);
Microsoft.ML.Tokenizers.Tests (35)
BpeTests.cs (4)
296
Assert.Equal(ids.Length, tokenizer.
CountTokens
(sentence));
414
Assert.Equal(12, tokenizer.
CountTokens
(text));
490
Assert.Equal(expectedIds.Length, tokenizer.
CountTokens
(text));
814
Assert.Equal(ids.Count(), tokenizer.
CountTokens
(text));
CodeGenTests.cs (4)
428
Assert.Equal(ids.Length, codeGenTokenizer.
CountTokens
(text));
629
count = codeGenTokenizer.
CountTokens
(text);
762
count = codeGenTokenizer.
CountTokens
(text);
917
count = codeGenTokenizer.
CountTokens
(text);
EnglishRobertaTests.cs (4)
209
Assert.Equal(expectedIds.Length, tokenizer.
CountTokens
(text));
251
idsCount = tokenizer.
CountTokens
((string)p[0]);
257
idsCount = tokenizer.
CountTokens
((string)p[0]);
261
idsCount = tokenizer.
CountTokens
((string)p[0]);
LlamaTests.cs (4)
251
Assert.Equal(ids.Length, llamaTokenizer.
CountTokens
(input));
344
Assert.Equal(0, llamaTokenizer.
CountTokens
((string)null!));
632
Assert.Equal(expectedIds.Length, tokenizer.
CountTokens
(text));
672
tokenCount = tokenizer.
CountTokens
(kvp.Key);
TiktokenTests.cs (14)
149
int idsCount = tokenizer.
CountTokens
(text);
202
int idsCount = GPT4.
CountTokens
(text);
220
int idsCount = gpt4Tokenizer.
CountTokens
(text);
249
int idsCount = GPT4.
CountTokens
(text);
264
int idsCount = GPT4.
CountTokens
(text);
274
int idsCount = GPT4.
CountTokens
(text);
293
int idsCount = tokenizer.
CountTokens
(text);
310
idsCount = tokenizer.
CountTokens
(text);
331
int idsCount = GPT2.
CountTokens
(text);
350
int idsCount = P50kBase.
CountTokens
(text);
369
int idsCount = P50kEdit.
CountTokens
(text);
388
int idsCount = R50kBase.
CountTokens
(text);
642
Assert.Equal(expectedIds.Length, tokenizer.
CountTokens
(text));
736
Assert.Equal(expectedIds.Length, GPT4.
CountTokens
(text));
TokenizerTests.cs (3)
48
Assert.Equal(5, tokenizer.
CountTokens
("hello"));
200
Assert.Equal(0, tokenizer.
CountTokens
(s.Substring(index1)));
208
Assert.Equal(0, tokenizer.
CountTokens
(s.Substring(0, index2)));
WordPieceTests.cs (2)
64
Assert.Equal(0, tokenizer.
CountTokens
(""));
117
Assert.Equal(5, tokenizer.
CountTokens
(text));