4 references to MaxWordLengthToCache
Microsoft.ML.Tokenizers (4)
Model\TiktokenTokenizer.cs (4)
341
if (text.Length <=
MaxWordLengthToCache
)
455
if (text.Length <=
MaxWordLengthToCache
)
620
if (text.Length <=
MaxWordLengthToCache
)
746
if (text.Length <=
MaxWordLengthToCache
)