1 write to _specialTokens
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
259_specialTokens = specialTokens.ToDictionary(kvp => new StringSpanOrdinalKey(kvp.Key), kvp => (kvp.Value, kvp.Key));
3 references to _specialTokens
Microsoft.ML.Tokenizers (3)
Model\BPETokenizer.cs (3)
983if (_specialTokens?.TryGetValue(text, out (int specialTokenId, string specialToken) value) is true) 1057if (_specialTokens?.TryGetValue(text, out (int specialTokenId, string specialToken) value) is true && maxTokens > 0) 1092if (_specialTokens?.TryGetValue(text, out (int specialTokenId, string specialToken) value) is true && maxTokens > 0)