1 write to _specialTokens
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
350_specialTokens = specialTokens.ToDictionary(kvp => new StringSpanOrdinalKey(kvp.Key), kvp => (kvp.Value, kvp.Key));
3 references to _specialTokens
Microsoft.ML.Tokenizers (3)
Model\BPETokenizer.cs (3)
1305if (_specialTokens?.TryGetValue(text, out (int specialTokenId, string specialToken) value) is true) 1414if (_specialTokens?.TryGetValue(text, out (int specialTokenId, string specialToken) value) is true && maxTokens > 0) 1496if (_specialTokens?.TryGetValue(text, out (int specialTokenId, string specialToken) value) is true && maxTokens > 0)