1 write to _vocab
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
231_vocab = vocab ?? new Dictionary<StringSpanOrdinalKey, int>();
10 references to _vocab
Microsoft.ML.Tokenizers (10)
Model\BPETokenizer.cs (10)
59if (!_vocab.TryGetValue(value, out int id)) 236foreach (KeyValuePair<StringSpanOrdinalKey, int> kvp in _vocab) 257if (!_vocab.TryGetValue(mergeValues.a, out int aId)) 262if (!_vocab.TryGetValue(mergeValues.b, out int bId)) 273if (!_vocab.TryGetValue(newToken, out int newId)) 574private int? MapTokenToId(ReadOnlySpan<char> token) => _vocab.TryGetValue(token, out int value) ? value : null; 594public IReadOnlyDictionary<string, int> Vocabulary => _vocabOriginal ??= new ReadOnlyDictionary<string, int>(_vocab.ToDictionary(kvp => kvp.Key.Data!, kvp => kvp.Value)); 913if (_vocab.TryGetValue(s, out int id)) 935if (!_vocab.TryGetValue(UnknownToken, out int value)) 944if (!_vocab.TryGetValue(UnknownToken, out int value))