1 write to _vocab
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
246_vocab = vocab ?? new Dictionary<StringSpanOrdinalKey, int>();
10 references to _vocab
Microsoft.ML.Tokenizers (10)
Model\BPETokenizer.cs (10)
59if (!_vocab.TryGetValue(value, out int id)) 251foreach (KeyValuePair<StringSpanOrdinalKey, int> kvp in _vocab) 272if (!_vocab.TryGetValue(mergeValues.a, out int aId)) 277if (!_vocab.TryGetValue(mergeValues.b, out int bId)) 288if (!_vocab.TryGetValue(newToken, out int newId)) 589private int? MapTokenToId(ReadOnlySpan<char> token) => _vocab.TryGetValue(token, out int value) ? value : null; 609public IReadOnlyDictionary<string, int> Vocabulary => _vocabOriginal ??= new ReadOnlyDictionary<string, int>(_vocab.ToDictionary(kvp => kvp.Key.Data!, kvp => kvp.Value)); 928if (_vocab.TryGetValue(s, out int id)) 950if (!_vocab.TryGetValue(UnknownToken, out int value)) 959if (!_vocab.TryGetValue(UnknownToken, out int value))