3 writes to _vocab
Microsoft.ML.Tokenizers (3)
Model\TiktokenTokenizer.cs (3)
87
_vocab
= vocab ?? throw new ArgumentNullException(nameof(vocab));
91
_vocab
= vocab!;
107
(_encoder,
_vocab
, _decoder) = LoadTiktokenBpeAsync(vocabStream, useAsync: false).GetAwaiter().GetResult();
5 references to _vocab
Microsoft.ML.Tokenizers (5)
Model\TiktokenTokenizer.cs (5)
317
if (
_vocab
.TryGetValue(text, out (int Id, string Token) mappedId))
433
if (
_vocab
.TryGetValue(text, out (int Id, string Token) mappedId))
599
if (
_vocab
.TryGetValue(text, out _))
725
if (
_vocab
.TryGetValue(text, out _))
991
internal IReadOnlyDictionary<string, int> Vocabulary => _vocabOriginal ??=
_vocab
.ToDictionary(kvp => kvp.Key.Data!, kvp => kvp.Value.Id);