37 references to CountTokens
Microsoft.Extensions.AI.Integration.Tests (1)
ReducingChatClientTests.cs (1)
102
sum += _tokenizer.
CountTokens
(text.Text);
Microsoft.Extensions.DataIngestion.Tests (1)
Chunkers\HeaderChunkerTests.cs (1)
181
Assert.All(chunks, chunk => Assert.InRange(tokenizer.
CountTokens
(chunk.Content), 1, 50));
Microsoft.ML.Tokenizers.Tests (35)
BpeTests.cs (4)
309
Assert.Equal(ids.Length, tokenizer.
CountTokens
(sentence));
427
Assert.Equal(12, tokenizer.
CountTokens
(text));
503
Assert.Equal(expectedIds.Length, tokenizer.
CountTokens
(text));
827
Assert.Equal(ids.Count(), tokenizer.
CountTokens
(text));
CodeGenTests.cs (4)
428
Assert.Equal(ids.Length, codeGenTokenizer.
CountTokens
(text));
629
count = codeGenTokenizer.
CountTokens
(text);
762
count = codeGenTokenizer.
CountTokens
(text);
917
count = codeGenTokenizer.
CountTokens
(text);
EnglishRobertaTests.cs (4)
209
Assert.Equal(expectedIds.Length, tokenizer.
CountTokens
(text));
251
idsCount = tokenizer.
CountTokens
((string)p[0]);
257
idsCount = tokenizer.
CountTokens
((string)p[0]);
261
idsCount = tokenizer.
CountTokens
((string)p[0]);
LlamaTests.cs (4)
251
Assert.Equal(ids.Length, llamaTokenizer.
CountTokens
(input));
344
Assert.Equal(0, llamaTokenizer.
CountTokens
((string)null!));
632
Assert.Equal(expectedIds.Length, tokenizer.
CountTokens
(text));
672
tokenCount = tokenizer.
CountTokens
(kvp.Key);
TiktokenTests.cs (14)
150
int idsCount = tokenizer.
CountTokens
(text);
203
int idsCount = GPT4.
CountTokens
(text);
221
int idsCount = gpt4Tokenizer.
CountTokens
(text);
250
int idsCount = GPT4.
CountTokens
(text);
265
int idsCount = GPT4.
CountTokens
(text);
275
int idsCount = GPT4.
CountTokens
(text);
294
int idsCount = tokenizer.
CountTokens
(text);
311
idsCount = tokenizer.
CountTokens
(text);
332
int idsCount = GPT2.
CountTokens
(text);
351
int idsCount = P50kBase.
CountTokens
(text);
370
int idsCount = P50kEdit.
CountTokens
(text);
389
int idsCount = R50kBase.
CountTokens
(text);
646
Assert.Equal(expectedIds.Length, tokenizer.
CountTokens
(text));
740
Assert.Equal(expectedIds.Length, GPT4.
CountTokens
(text));
TokenizerTests.cs (3)
48
Assert.Equal(5, tokenizer.
CountTokens
("hello"));
200
Assert.Equal(0, tokenizer.
CountTokens
(s.Substring(index1)));
208
Assert.Equal(0, tokenizer.
CountTokens
(s.Substring(0, index2)));
WordPieceTests.cs (2)
64
Assert.Equal(0, tokenizer.
CountTokens
(""));
117
Assert.Equal(5, tokenizer.
CountTokens
(text));