1 write to _specialTokens
Microsoft.ML.Tokenizers (1)
Model\CodeGenTokenizer.cs (1)
137_specialTokens = specialTokens.ToDictionary(kvp => new StringSpanOrdinalKey(kvp.Key), kvp => (kvp.Value, kvp.Key));
8 references to _specialTokens
Microsoft.ML.Tokenizers (8)
Model\CodeGenTokenizer.cs (8)
432if (_specialTokens is not null && _specialTokens.TryGetValue(textSpan, out (int specialTokenId, string specialToken) value)) 1112if (_specialTokens is not null && _specialTokens.TryGetValue(textSpan, out (int specialTokenId, string specialToken) value) && maxTokens > 0) 1176if (_specialTokens is not null && _specialTokens.TryGetValue(textSpan, out (int specialTokenId, string specialToken) value) && maxTokens > 0) 1559if (_specialTokens is not null && _specialTokens.TryGetValue(token, out (int Id, string Token) specialToken))