22 references to TryGetValue
Microsoft.ML.Tokenizers (22)
Model\BPETokenizer.cs (1)
544private int? MapTokenToId(ReadOnlySpan<char> token) => _vocab.TryGetValue(token, out int value) ? value : null;
Model\CodeGenTokenizer.cs (3)
1549if (_vocab.TryGetValue(token, out (int Id, string Token) value)) 1554if (_addedTokens is not null && _addedTokens.TryGetValue(token, out (int Id, string Token) addedToken)) 1678if (_vocab.TryGetValue(text.Slice(symbols[index].pieceSpan.Index, symbols[index].pieceSpan.Length), out (int Id, string Token) value))
Model\EnglishRobertaTokenizer.cs (1)
746private int? MapTokenToId(ReadOnlySpan<char> token) => _vocab.TryGetValue(token, out int value) ? value : null;
Model\SentencePieceBpeTokenizer.cs (13)
287if (_specialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 332if (_vocab.TryGetValue(text.Slice(symbols[index].pieceSpan.Index, symbols[index].pieceSpan.Length), out (int Id, float Score, byte Type) tokenInfo)) 426if (!_vocab.TryGetValue(text.Slice(pieceSpan.Index, pieceSpan.Length), out (int Id, float Score, byte Type) id)) 618if (idsCount < maxTokens && _specialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 681if (_vocab.TryGetValue(text.Slice(symbols[index].pieceSpan.Index, symbols[index].pieceSpan.Length), out (int Id, float Score, byte Type) tokenInfo)) 803if (!_vocab.TryGetValue(text.Slice(pieceSpan.Index, pieceSpan.Length), out (int Id, float Score, byte Type) id)) 951if (idsCount < maxTokens && _specialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 1005if (_vocab.TryGetValue(text.Slice(symbols[index].pieceSpan.Index, symbols[index].pieceSpan.Length), out (int Id, float Score, byte Type) tokenInfo)) 1118if (!_vocab.TryGetValue(text.Slice(pieceSpan.Index, pieceSpan.Length), out (int Id, float Score, byte Type) id)) 1324if (_specialTokens!.TryGetValue(text.Slice(current.Offset, current.Length), out int id)) 1385if (_vocab.TryGetValue(text.Slice(symbols[index].pieceSpan.Index, symbols[index].pieceSpan.Length), out (int Id, float Score, byte Type) tokenInfo)) 1498if (!_vocab.TryGetValue(text.Slice(pieceSpan.Index, pieceSpan.Length), out (int Id, float Score, byte Type) id)) 2240if (!_vocab.TryGetValue(textSpan.Slice(symbols[left].pieceSpan.Index, pieceLength), out (int Id, float Score, byte Type) leftId))
Model\TiktokenTokenizer.cs (4)
316if (_vocab.TryGetValue(text, out (int Id, string Token) mappedId)) 432if (_vocab.TryGetValue(text, out (int Id, string Token) mappedId)) 598if (_vocab.TryGetValue(text, out _)) 724if (_vocab.TryGetValue(text, out _))