1 write to _vocab
Microsoft.ML.Tokenizers (1)
Model\EnglishRobertaTokenizer.cs (1)
166_vocab = GetVocabulary(vocabularyStream);
5 references to _vocab
Microsoft.ML.Tokenizers (5)
Model\EnglishRobertaTokenizer.cs (5)
167_vocabReverse = _vocab.ReverseSorted(); 240foreach (var item in _vocab) 757private int? MapTokenToId(ReadOnlySpan<char> token) => _vocab.TryGetValue(token, out int value) ? value : null; 961return new List<EncodedToken> { new EncodedToken(_vocab[new StringSpanOrdinalKey(tokenValue)], tokenValue, new Range(indexMapping[0], indexMapping[0] + 1)) }; 1050tokens.Add(new EncodedToken(_vocab[new StringSpanOrdinalKey(w)], w, new Range(indexMapping[index], indexMapping[index] + w.Length)));