1 write to _specialTokens
Microsoft.ML.Tokenizers (1)
Model\CodeGenTokenizer.cs (1)
134_specialTokens = specialTokens.ToDictionary(kvp => new StringSpanOrdinalKey(kvp.Key), kvp => (kvp.Value, kvp.Key));
8 references to _specialTokens
Microsoft.ML.Tokenizers (8)
Model\CodeGenTokenizer.cs (8)
429if (_specialTokens is not null && _specialTokens.TryGetValue(textSpan, out (int specialTokenId, string specialToken) value)) 1109if (_specialTokens is not null && _specialTokens.TryGetValue(textSpan, out (int specialTokenId, string specialToken) value) && maxTokens > 0) 1173if (_specialTokens is not null && _specialTokens.TryGetValue(textSpan, out (int specialTokenId, string specialToken) value) && maxTokens > 0) 1556if (_specialTokens is not null && _specialTokens.TryGetValue(token, out (int Id, string Token) specialToken))