2 writes to SpecialTokensReverse
Microsoft.ML.Tokenizers (2)
Model\SentencePieceBaseModel.cs (2)
42SpecialTokensReverse = new Dictionary<int, string>(); 89SpecialTokensReverse = new Dictionary<int, string>();
6 references to SpecialTokensReverse
Microsoft.ML.Tokenizers (6)
Model\SentencePieceBaseModel.cs (4)
47SpecialTokensReverse.Add(item.Value, item.Key); 94SpecialTokensReverse.Add(item.Value, item.Key); 398else if (model.SpecialTokensReverse?.TryGetValue(id, out string? specialToken) is true) 638model.SpecialTokensReverse?.TryGetValue(id, out specialToken);
Model\SentencePieceBpeModel.cs (1)
125tokens.Add(new EncodedToken(id, SpecialTokensReverse![id], new Range(Offset, Offset + Length)));
Model\SentencePieceUnigramModel.cs (1)
512tokens.Add(new EncodedToken(id, SpecialTokensReverse![id], new Range(progressOffset, progressOffset + Length)));