1 write to SpecialTokensReverse
Microsoft.ML.Tokenizers (1)
Model\SentencePieceBaseModel.cs (1)
42
SpecialTokensReverse
= new Dictionary<int, string>();
5 references to SpecialTokensReverse
Microsoft.ML.Tokenizers (5)
Model\SentencePieceBaseModel.cs (3)
47
SpecialTokensReverse
.Add(item.Value, item.Key);
354
else if (model.
SpecialTokensReverse
?.TryGetValue(id, out string? specialToken) is true)
594
model.
SpecialTokensReverse
?.TryGetValue(id, out specialToken);
Model\SentencePieceBpeModel.cs (1)
125
tokens.Add(new EncodedToken(id,
SpecialTokensReverse
![id], new Range(Offset, Offset + Length)));
Model\SentencePieceUnigramModel.cs (1)
237
tokens.Add(new EncodedToken(id,
SpecialTokensReverse
![id], new Range(progressOffset, progressOffset + Length)));