1 write to _vocab
Microsoft.ML.Tokenizers (1)
Model\EnglishRobertaTokenizer.cs (1)
166
_vocab
= GetVocabulary(vocabularyStream);
5 references to _vocab
Microsoft.ML.Tokenizers (5)
Model\EnglishRobertaTokenizer.cs (5)
167
_vocabReverse =
_vocab
.ReverseSorted();
240
foreach (var item in
_vocab
)
757
private int? MapTokenToId(ReadOnlySpan<char> token) =>
_vocab
.TryGetValue(token, out int value) ? value : null;
961
return new List<EncodedToken> { new EncodedToken(
_vocab
[new StringSpanOrdinalKey(tokenValue)], tokenValue, new Range(indexMapping[0], indexMapping[0] + 1)) };
1050
tokens.Add(new EncodedToken(
_vocab
[new StringSpanOrdinalKey(w)], w, new Range(indexMapping[index], indexMapping[index] + w.Length)));