3 references to MaxTokenLengthToCache
Microsoft.ML.Tokenizers (3)
Model\CodeGenTokenizer.cs (3)
466
if (textSpan.Length <=
MaxTokenLengthToCache
)
1150
if (length <=
MaxTokenLengthToCache
)
1214
if (length <=
MaxTokenLengthToCache
)