5 references to Add
Microsoft.ML.Tokenizers (5)
Model\TiktokenTokenizer.cs (5)
144
_cache!.
Add
(specialToken.Key, new[] { (Id: specialToken.Value, TokenIndex0: 0, TokenLength: specialToken.Key.Length) });
343
_cache.
Add
(textAsString, encodedTokens);
458
_cache.
Add
(textAsString, encodedTokens);
623
_cache.
Add
(textAsString, encodedTokens);
749
_cache.
Add
(textAsString, encodedTokens);