1 write to _vocab
Microsoft.ML.Tokenizers (1)
Model\EnglishRobertaTokenizer.cs (1)
154
_vocab
= GetVocabulary(vocabularyStream);
5 references to _vocab
Microsoft.ML.Tokenizers (5)
Model\EnglishRobertaTokenizer.cs (5)
155
_vocabReverse =
_vocab
.ReverseSorted();
229
foreach (var item in
_vocab
)
746
private int? MapTokenToId(ReadOnlySpan<char> token) =>
_vocab
.TryGetValue(token, out int value) ? value : null;
950
return new List<EncodedToken> { new EncodedToken(
_vocab
[new StringSpanOrdinalKey(tokenValue)], tokenValue, (indexMapping[0], 1)) };
1039
tokens.Add(new EncodedToken(
_vocab
[new StringSpanOrdinalKey(w)], w, (indexMapping[index], w.Length)));