3 references to MaxTokenLengthToCache
Microsoft.ML.Tokenizers (3)
Model\CodeGenTokenizer.cs (3)
463
if (textSpan.Length <=
MaxTokenLengthToCache
)
1147
if (length <=
MaxTokenLengthToCache
)
1211
if (length <=
MaxTokenLengthToCache
)