1 write to _addedTokens
Microsoft.ML.Tokenizers (1)
Model\CodeGenTokenizer.cs (1)
134
_addedTokens
= addedTokens.ToDictionary(kvp => new StringSpanOrdinalKey(kvp.Key), kvp => (kvp.Value, kvp.Key));
8 references to _addedTokens
Microsoft.ML.Tokenizers (8)
Model\CodeGenTokenizer.cs (8)
428
if (
_addedTokens
is not null &&
_addedTokens
.TryGetValue(textSpan, out (int addedTokenId, string addedToken) value))
1107
if (
_addedTokens
is not null &&
_addedTokens
.TryGetValue(textSpan, out (int addedTokenId, string addedToken) value) && maxTokens > 0)
1171
if (
_addedTokens
is not null &&
_addedTokens
.TryGetValue(textSpan, out (int addedTokenId, string addedToken) value) && maxTokens > 0)
1554
if (
_addedTokens
is not null &&
_addedTokens
.TryGetValue(token, out (int Id, string Token) addedToken))