22 references to TryGetValue
Microsoft.ML.Tokenizers (22)
Model\BPETokenizer.cs (1)
544
private int? MapTokenToId(ReadOnlySpan<char> token) => _vocab.
TryGetValue
(token, out int value) ? value : null;
Model\CodeGenTokenizer.cs (3)
1549
if (_vocab.
TryGetValue
(token, out (int Id, string Token) value))
1554
if (_addedTokens is not null && _addedTokens.
TryGetValue
(token, out (int Id, string Token) addedToken))
1678
if (_vocab.
TryGetValue
(text.Slice(symbols[index].pieceSpan.Index, symbols[index].pieceSpan.Length), out (int Id, string Token) value))
Model\EnglishRobertaTokenizer.cs (1)
746
private int? MapTokenToId(ReadOnlySpan<char> token) => _vocab.
TryGetValue
(token, out int value) ? value : null;
Model\SentencePieceBpeTokenizer.cs (13)
287
if (_specialTokens!.
TryGetValue
(text.Slice(Offset, Length), out int id))
332
if (_vocab.
TryGetValue
(text.Slice(symbols[index].pieceSpan.Index, symbols[index].pieceSpan.Length), out (int Id, float Score, byte Type) tokenInfo))
426
if (!_vocab.
TryGetValue
(text.Slice(pieceSpan.Index, pieceSpan.Length), out (int Id, float Score, byte Type) id))
618
if (idsCount < maxTokens && _specialTokens!.
TryGetValue
(text.Slice(Offset, Length), out int id))
681
if (_vocab.
TryGetValue
(text.Slice(symbols[index].pieceSpan.Index, symbols[index].pieceSpan.Length), out (int Id, float Score, byte Type) tokenInfo))
803
if (!_vocab.
TryGetValue
(text.Slice(pieceSpan.Index, pieceSpan.Length), out (int Id, float Score, byte Type) id))
951
if (idsCount < maxTokens && _specialTokens!.
TryGetValue
(text.Slice(Offset, Length), out int id))
1005
if (_vocab.
TryGetValue
(text.Slice(symbols[index].pieceSpan.Index, symbols[index].pieceSpan.Length), out (int Id, float Score, byte Type) tokenInfo))
1118
if (!_vocab.
TryGetValue
(text.Slice(pieceSpan.Index, pieceSpan.Length), out (int Id, float Score, byte Type) id))
1324
if (_specialTokens!.
TryGetValue
(text.Slice(current.Offset, current.Length), out int id))
1385
if (_vocab.
TryGetValue
(text.Slice(symbols[index].pieceSpan.Index, symbols[index].pieceSpan.Length), out (int Id, float Score, byte Type) tokenInfo))
1498
if (!_vocab.
TryGetValue
(text.Slice(pieceSpan.Index, pieceSpan.Length), out (int Id, float Score, byte Type) id))
2240
if (!_vocab.
TryGetValue
(textSpan.Slice(symbols[left].pieceSpan.Index, pieceLength), out (int Id, float Score, byte Type) leftId))
Model\TiktokenTokenizer.cs (4)
316
if (_vocab.
TryGetValue
(text, out (int Id, string Token) mappedId))
432
if (_vocab.
TryGetValue
(text, out (int Id, string Token) mappedId))
598
if (_vocab.
TryGetValue
(text, out _))
724
if (_vocab.
TryGetValue
(text, out _))