3 writes to _vocab
Microsoft.ML.Tokenizers (3)
Model\TiktokenTokenizer.cs (3)
87_vocab = vocab ?? throw new ArgumentNullException(nameof(vocab)); 91_vocab = vocab!; 107(_encoder, _vocab, _decoder) = LoadTiktokenBpeAsync(vocabStream, useAsync: false).GetAwaiter().GetResult();
5 references to _vocab
Microsoft.ML.Tokenizers (5)
Model\TiktokenTokenizer.cs (5)
317if (_vocab.TryGetValue(text, out (int Id, string Token) mappedId)) 433if (_vocab.TryGetValue(text, out (int Id, string Token) mappedId)) 599if (_vocab.TryGetValue(text, out _)) 725if (_vocab.TryGetValue(text, out _)) 991internal IReadOnlyDictionary<string, int> Vocabulary => _vocabOriginal ??= _vocab.ToDictionary(kvp => kvp.Key.Data!, kvp => kvp.Value.Id);