1 write to _vocab
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
319_vocab = vocab ?? new Dictionary<StringSpanOrdinalKey, int>();
12 references to _vocab
Microsoft.ML.Tokenizers (12)
Model\BPETokenizer.cs (12)
60if (!_vocab.TryGetValue(value, out int id)) 323if (_vocab.TryGetValue(beginningOfSentenceToken, out int aId) is false && specialTokens?.TryGetValue(beginningOfSentenceToken, out aId) is false) 334if (_vocab.TryGetValue(endOfSentenceToken, out int aId) is false && specialTokens?.TryGetValue(endOfSentenceToken, out aId) is false) 347foreach (KeyValuePair<StringSpanOrdinalKey, int> kvp in _vocab) 368if (!_vocab.TryGetValue(mergeValues.a, out int aId)) 373if (!_vocab.TryGetValue(mergeValues.b, out int bId)) 384if (!_vocab.TryGetValue(newToken, out int newId)) 746private int? MapTokenToId(ReadOnlySpan<char> token) => _vocab.TryGetValue(token, out int value) ? value : null; 766public IReadOnlyDictionary<string, int> Vocabulary => _vocabOriginal ??= new ReadOnlyDictionary<string, int>(_vocab.ToDictionary(kvp => kvp.Key.Data!, kvp => kvp.Value)); 1254if (_vocab.TryGetValue(s, out int id)) 1276if (!_vocab.TryGetValue(UnknownToken, out int value)) 1285if (!_vocab.TryGetValue(UnknownToken, out int value))