5 references to TokenizerTests
Microsoft.ML.Tokenizers.Tests (5)
BpeTests.cs (1)
383
TokenizerTests
.TestTokenLimits(tokenizer);
EnglishRobertaTests.cs (1)
116
TokenizerTests
.TestTokenLimits(tokenizer);
LlamaTests.cs (1)
376
TokenizerTests
.TestTokenLimits(llamaTokenizer);
TitokenTests.cs (2)
226
TokenizerTests
.TestTokenLimits(gpt4Tokenizer);
314
TokenizerTests
.TestTokenLimits(GPT4o);