5 references to Add
Microsoft.ML.Tokenizers (5)
Model\TiktokenTokenizer.cs (5)
143
_cache!.
Add
(specialToken.Key, new[] { (Id: specialToken.Value, TokenIndex0: 0, TokenLength: specialToken.Key.Length) });
342
_cache.
Add
(textAsString, encodedTokens);
457
_cache.
Add
(textAsString, encodedTokens);
622
_cache.
Add
(textAsString, encodedTokens);
748
_cache.
Add
(textAsString, encodedTokens);