2 writes to SpecialTokensReverse
Microsoft.ML.Tokenizers (2)
Model\SentencePieceBaseModel.cs (2)
42SpecialTokensReverse = new Dictionary<int, string>(); 103SpecialTokensReverse = new Dictionary<int, string>();
6 references to SpecialTokensReverse
Microsoft.ML.Tokenizers (6)
Model\SentencePieceBaseModel.cs (4)
47SpecialTokensReverse.Add(item.Value, item.Key); 108SpecialTokensReverse.Add(item.Value, item.Key); 415else if (model.SpecialTokensReverse?.TryGetValue(id, out string? specialToken) is true) 655model.SpecialTokensReverse?.TryGetValue(id, out specialToken);
Model\SentencePieceBpeModel.cs (1)
171tokens.Add(new EncodedToken(id, SpecialTokensReverse![id], new Range(Offset, Offset + Length)));
Model\SentencePieceUnigramModel.cs (1)
287tokens.Add(new EncodedToken(id, SpecialTokensReverse![id], new Range(progressOffset, progressOffset + Length)));