5 references to TestTokenLimits
Microsoft.ML.Tokenizers.Tests (5)
BpeTests.cs (1)
429
TokenizerTests.
TestTokenLimits
(tokenizer);
EnglishRobertaTests.cs (1)
116
TokenizerTests.
TestTokenLimits
(tokenizer);
LlamaTests.cs (1)
376
TokenizerTests.
TestTokenLimits
(llamaTokenizer);
TiktokenTests.cs (2)
233
TokenizerTests.
TestTokenLimits
(gpt4Tokenizer);
323
TokenizerTests.
TestTokenLimits
(tokenizer);