1 write to _specialTokensReverse
Microsoft.ML.Tokenizers (1)
Model\SentencePieceBpeTokenizer.cs (1)
81_specialTokensReverse = new Dictionary<int, string>();
4 references to _specialTokensReverse
Microsoft.ML.Tokenizers (4)
Model\SentencePieceBpeTokenizer.cs (4)
86_specialTokensReverse.Add(item.Value, item.Key); 289tokens.Add(new EncodedToken(id, _specialTokensReverse![id], (Offset, Length))); 1757else if (tokenizer._specialTokensReverse?.TryGetValue(id, out string? specialToken) is true) 1998else if (!tokenizer._specialTokensReverse?.TryGetValue(id, out specialToken) is true)