1 write to _vocab
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
314_vocab = vocab ?? new Dictionary<StringSpanOrdinalKey, int>();
12 references to _vocab
Microsoft.ML.Tokenizers (12)
Model\BPETokenizer.cs (12)
60if (!_vocab.TryGetValue(value, out int id)) 318if (!_vocab.TryGetValue(beginningOfSentenceToken, out int aId)) 329if (!_vocab.TryGetValue(endOfSentenceToken, out int aId)) 342foreach (KeyValuePair<StringSpanOrdinalKey, int> kvp in _vocab) 363if (!_vocab.TryGetValue(mergeValues.a, out int aId)) 368if (!_vocab.TryGetValue(mergeValues.b, out int bId)) 379if (!_vocab.TryGetValue(newToken, out int newId)) 741private int? MapTokenToId(ReadOnlySpan<char> token) => _vocab.TryGetValue(token, out int value) ? value : null; 761public IReadOnlyDictionary<string, int> Vocabulary => _vocabOriginal ??= new ReadOnlyDictionary<string, int>(_vocab.ToDictionary(kvp => kvp.Key.Data!, kvp => kvp.Value)); 1250if (_vocab.TryGetValue(s, out int id)) 1272if (!_vocab.TryGetValue(UnknownToken, out int value)) 1281if (!_vocab.TryGetValue(UnknownToken, out int value))