4 references to MaxWordLengthToCache
Microsoft.ML.Tokenizers (4)
Model\TiktokenTokenizer.cs (4)
340
if (text.Length <=
MaxWordLengthToCache
)
454
if (text.Length <=
MaxWordLengthToCache
)
619
if (text.Length <=
MaxWordLengthToCache
)
745
if (text.Length <=
MaxWordLengthToCache
)