1 write to _specialTokensReverse
Microsoft.ML.Tokenizers (1)
Model\SentencePieceTokenizer.cs (1)
81
_specialTokensReverse
= new Dictionary<int, string>();
4 references to _specialTokensReverse
Microsoft.ML.Tokenizers (4)
Model\SentencePieceTokenizer.cs (4)
86
_specialTokensReverse
.Add(item.Value, item.Key);
289
tokens.Add(new EncodedToken(id,
_specialTokensReverse
![id], new Range(Offset, Offset + Length)));
1757
else if (tokenizer.
_specialTokensReverse
?.TryGetValue(id, out string? specialToken) is true)
1998
else if (!tokenizer.
_specialTokensReverse
?.TryGetValue(id, out specialToken) is true)