35 references to CountTokens
Microsoft.Extensions.AI.Integration.Tests (1)
ReducingChatClientTests.cs (1)
162sum += _tokenizer.CountTokens(text.Text);
Microsoft.ML.Tokenizers.Tests (34)
BpeTests.cs (3)
265Assert.Equal(ids.Length, tokenizer.CountTokens(sentence)); 381Assert.Equal(12, tokenizer.CountTokens(text)); 457Assert.Equal(expectedIds.Length, tokenizer.CountTokens(text));
CodeGenTests.cs (4)
428Assert.Equal(ids.Length, codeGenTokenizer.CountTokens(text)); 629count = codeGenTokenizer.CountTokens(text); 762count = codeGenTokenizer.CountTokens(text); 917count = codeGenTokenizer.CountTokens(text);
EnglishRobertaTests.cs (4)
209Assert.Equal(expectedIds.Length, tokenizer.CountTokens(text)); 251idsCount = tokenizer.CountTokens((string)p[0]); 257idsCount = tokenizer.CountTokens((string)p[0]); 261idsCount = tokenizer.CountTokens((string)p[0]);
LlamaTests.cs (4)
251Assert.Equal(ids.Length, llamaTokenizer.CountTokens(input)); 344Assert.Equal(0, llamaTokenizer.CountTokens((string)null!)); 632Assert.Equal(expectedIds.Length, tokenizer.CountTokens(text)); 672tokenCount = tokenizer.CountTokens(kvp.Key);
TiktokenTests.cs (14)
147int idsCount = tokenizer.CountTokens(text); 200int idsCount = GPT4.CountTokens(text); 218int idsCount = gpt4Tokenizer.CountTokens(text); 247int idsCount = GPT4.CountTokens(text); 262int idsCount = GPT4.CountTokens(text); 272int idsCount = GPT4.CountTokens(text); 289int idsCount = GPT4o.CountTokens(text); 306idsCount = GPT4o.CountTokens(text); 326int idsCount = GPT2.CountTokens(text); 345int idsCount = P50kBase.CountTokens(text); 364int idsCount = P50kEdit.CountTokens(text); 383int idsCount = R50kBase.CountTokens(text); 610Assert.Equal(expectedIds.Length, tokenizer.CountTokens(text)); 704Assert.Equal(expectedIds.Length, GPT4.CountTokens(text));
TokenizerTests.cs (3)
48Assert.Equal(5, tokenizer.CountTokens("hello")); 200Assert.Equal(0, tokenizer.CountTokens(s.Substring(index1))); 208Assert.Equal(0, tokenizer.CountTokens(s.Substring(0, index2)));
WordPieceTests.cs (2)
64Assert.Equal(0, tokenizer.CountTokens("")); 117Assert.Equal(5, tokenizer.CountTokens(text));