5 references to TestTokenLimits
Microsoft.ML.Tokenizers.Tests (5)
BpeTests.cs (1)
416TokenizerTests.TestTokenLimits(tokenizer);
EnglishRobertaTests.cs (1)
116TokenizerTests.TestTokenLimits(tokenizer);
LlamaTests.cs (1)
376TokenizerTests.TestTokenLimits(llamaTokenizer);
TiktokenTests.cs (2)
232TokenizerTests.TestTokenLimits(gpt4Tokenizer); 322TokenizerTests.TestTokenLimits(tokenizer);