29 references to TryGetValue
Microsoft.ML.Tokenizers (29)
Model\BPETokenizer.cs (4)
589private int? MapTokenToId(ReadOnlySpan<char> token) => _vocab.TryGetValue(token, out int value) ? value : null; 983if (_specialTokens?.TryGetValue(text, out (int specialTokenId, string specialToken) value) is true) 1057if (_specialTokens?.TryGetValue(text, out (int specialTokenId, string specialToken) value) is true && maxTokens > 0) 1092if (_specialTokens?.TryGetValue(text, out (int specialTokenId, string specialToken) value) is true && maxTokens > 0)
Model\CodeGenTokenizer.cs (3)
1554if (_vocab.TryGetValue(token, out (int Id, string Token) value)) 1559if (_specialTokens is not null && _specialTokens.TryGetValue(token, out (int Id, string Token) specialToken)) 1684if (_vocab.TryGetValue(text.Slice(symbols[index].pieceSpan.Index, symbols[index].pieceSpan.Length), out (int Id, string Token) value))
Model\EnglishRobertaTokenizer.cs (1)
757private int? MapTokenToId(ReadOnlySpan<char> token) => _vocab.TryGetValue(token, out int value) ? value : null;
Model\SentencePieceBpeModel.cs (13)
169if (InternalSpecialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 214if (_vocab.TryGetValue(text.Slice(symbols[index].pieceSpan.Index, symbols[index].pieceSpan.Length), out (int Id, float Score, byte Type) tokenInfo)) 308if (!_vocab.TryGetValue(text.Slice(pieceSpan.Index, pieceSpan.Length), out (int Id, float Score, byte Type) id)) 429if (idsCount < maxTokens && InternalSpecialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 492if (_vocab.TryGetValue(text.Slice(symbols[index].pieceSpan.Index, symbols[index].pieceSpan.Length), out (int Id, float Score, byte Type) tokenInfo)) 614if (!_vocab.TryGetValue(text.Slice(pieceSpan.Index, pieceSpan.Length), out (int Id, float Score, byte Type) id)) 706if (idsCount < maxTokens && InternalSpecialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 760if (_vocab.TryGetValue(text.Slice(symbols[index].pieceSpan.Index, symbols[index].pieceSpan.Length), out (int Id, float Score, byte Type) tokenInfo)) 873if (!_vocab.TryGetValue(text.Slice(pieceSpan.Index, pieceSpan.Length), out (int Id, float Score, byte Type) id)) 976if (InternalSpecialTokens!.TryGetValue(text.Slice(current.Offset, current.Length), out int id)) 1036if (_vocab.TryGetValue(text.Slice(symbols[index].pieceSpan.Index, symbols[index].pieceSpan.Length), out (int Id, float Score, byte Type) tokenInfo)) 1149if (!_vocab.TryGetValue(text.Slice(pieceSpan.Index, pieceSpan.Length), out (int Id, float Score, byte Type) id)) 1244if (!_vocab.TryGetValue(textSpan.Slice(symbols[left].pieceSpan.Index, pieceLength), out (int Id, float Score, byte Type) leftId))
Model\SentencePieceUnigramModel.cs (4)
285if (InternalSpecialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 718if (InternalSpecialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 1089if (InternalSpecialTokens!.TryGetValue(text.Slice(Offset, Length), out int id)) 1356if (InternalSpecialTokens!.TryGetValue(text.Slice(current.Offset, current.Length), out int id))
Model\TiktokenTokenizer.cs (4)
317if (_vocab.TryGetValue(text, out (int Id, string Token) mappedId)) 433if (_vocab.TryGetValue(text, out (int Id, string Token) mappedId)) 599if (_vocab.TryGetValue(text, out _)) 725if (_vocab.TryGetValue(text, out _))