32 references to CountTokens
Microsoft.ML.Tokenizers.Tests (32)
BpeTests.cs (3)
265
Assert.Equal(ids.Length, tokenizer.
CountTokens
(sentence));
381
Assert.Equal(12, tokenizer.
CountTokens
(text));
457
Assert.Equal(expectedIds.Length, tokenizer.
CountTokens
(text));
CodeGenTests.cs (4)
428
Assert.Equal(ids.Length, codeGenTokenizer.
CountTokens
(text));
629
count = codeGenTokenizer.
CountTokens
(text);
762
count = codeGenTokenizer.
CountTokens
(text);
917
count = codeGenTokenizer.
CountTokens
(text);
EnglishRobertaTests.cs (4)
209
Assert.Equal(expectedIds.Length, tokenizer.
CountTokens
(text));
251
idsCount = tokenizer.
CountTokens
((string)p[0]);
257
idsCount = tokenizer.
CountTokens
((string)p[0]);
261
idsCount = tokenizer.
CountTokens
((string)p[0]);
LlamaTests.cs (4)
251
Assert.Equal(ids.Length, llamaTokenizer.
CountTokens
(input));
344
Assert.Equal(0, llamaTokenizer.
CountTokens
((string)null!));
632
Assert.Equal(expectedIds.Length, tokenizer.
CountTokens
(text));
672
tokenCount = tokenizer.
CountTokens
(kvp.Key);
TitokenTests.cs (14)
143
int idsCount = tokenizer.
CountTokens
(text);
196
int idsCount = GPT4.
CountTokens
(text);
214
int idsCount = gpt4Tokenizer.
CountTokens
(text);
243
int idsCount = GPT4.
CountTokens
(text);
258
int idsCount = GPT4.
CountTokens
(text);
268
int idsCount = GPT4.
CountTokens
(text);
285
int idsCount = GPT4o.
CountTokens
(text);
302
idsCount = GPT4o.
CountTokens
(text);
322
int idsCount = GPT2.
CountTokens
(text);
341
int idsCount = P50kBase.
CountTokens
(text);
360
int idsCount = P50kEdit.
CountTokens
(text);
379
int idsCount = R50kBase.
CountTokens
(text);
592
Assert.Equal(expectedIds.Length, tokenizer.
CountTokens
(text));
686
Assert.Equal(expectedIds.Length, GPT4.
CountTokens
(text));
TokenizerTests.cs (3)
48
Assert.Equal(5, tokenizer.
CountTokens
("hello"));
200
Assert.Equal(0, tokenizer.
CountTokens
(s.Substring(index1)));
208
Assert.Equal(0, tokenizer.
CountTokens
(s.Substring(0, index2)));