1 write to _addedTokens
Microsoft.ML.Tokenizers (1)
Model\CodeGenTokenizer.cs (1)
134_addedTokens = addedTokens.ToDictionary(kvp => new StringSpanOrdinalKey(kvp.Key), kvp => (kvp.Value, kvp.Key));
8 references to _addedTokens
Microsoft.ML.Tokenizers (8)
Model\CodeGenTokenizer.cs (8)
428if (_addedTokens is not null && _addedTokens.TryGetValue(textSpan, out (int addedTokenId, string addedToken) value)) 1107if (_addedTokens is not null && _addedTokens.TryGetValue(textSpan, out (int addedTokenId, string addedToken) value) && maxTokens > 0) 1171if (_addedTokens is not null && _addedTokens.TryGetValue(textSpan, out (int addedTokenId, string addedToken) value) && maxTokens > 0) 1554if (_addedTokens is not null && _addedTokens.TryGetValue(token, out (int Id, string Token) addedToken))