2 writes to _cache
Microsoft.ML.Tokenizers (2)
Model\TiktokenTokenizer.cs (2)
96_cache = new LruCache<(int Id, int TokenIndex, int TokenLength)[]>(cacheSize); 106_cache = new LruCache<(int Id, int TokenIndex, int TokenLength)[]>(cacheSize);
10 references to _cache
Microsoft.ML.Tokenizers (10)
Model\TiktokenTokenizer.cs (10)
136Debug.Assert(_cache is not null); 144_cache!.Add(specialToken.Key, new[] { (Id: specialToken.Value, TokenIndex0: 0, TokenLength: specialToken.Key.Length) }); 303if (_cache.TryGetValue(text, out (int Id, int TokenIndex, int TokenLength)[] value)) 343_cache.Add(textAsString, encodedTokens); 428if (_cache.TryGetValue(text, out (int Id, int TokenIndex, int TokenLength)[] value)) 458_cache.Add(textAsString, encodedTokens); 594if (_cache.TryGetValue(text, out (int Id, int TokenIndex, int TokenLength)[] value)) 623_cache.Add(textAsString, encodedTokens); 720if (_cache.TryGetValue(text, out (int Id, int TokenIndex, int TokenLength)[] value)) 749_cache.Add(textAsString, encodedTokens);