5 references to TokenizerTests
Microsoft.ML.Tokenizers.Tests (5)
BpeTests.cs (1)
383TokenizerTests.TestTokenLimits(tokenizer);
EnglishRobertaTests.cs (1)
116TokenizerTests.TestTokenLimits(tokenizer);
LlamaTests.cs (1)
376TokenizerTests.TestTokenLimits(llamaTokenizer);
TiktokenTests.cs (2)
227TokenizerTests.TestTokenLimits(gpt4Tokenizer); 315TokenizerTests.TestTokenLimits(GPT4o);