13 references to TryGetValue
Microsoft.ML.Tokenizers (13)
Model\BPETokenizer.cs (8)
60
if (!_vocab.
TryGetValue
(value, out int id))
323
if (_vocab.
TryGetValue
(beginningOfSentenceToken, out int aId) is false && specialTokens?.TryGetValue(beginningOfSentenceToken, out aId) is false)
334
if (_vocab.
TryGetValue
(endOfSentenceToken, out int aId) is false && specialTokens?.TryGetValue(endOfSentenceToken, out aId) is false)
368
if (!_vocab.
TryGetValue
(mergeValues.a, out int aId))
373
if (!_vocab.
TryGetValue
(mergeValues.b, out int bId))
384
if (!_vocab.
TryGetValue
(newToken, out int newId))
1276
if (!_vocab.
TryGetValue
(UnknownToken, out int value))
1285
if (!_vocab.
TryGetValue
(UnknownToken, out int value))
Model\CodeGenTokenizer.cs (3)
151
if (!_vocab.
TryGetValue
(UnknownToken!, out (int unknownId, string token) value))
161
if (!_vocab.
TryGetValue
(BeginningOfSentenceToken!, out (int beggingOfSentenceId, string token) value))
171
if (!_vocab.
TryGetValue
(EndOfSentenceToken!, out (int endOfSentenceId, string token) value))
Model\SentencePieceBpeModel.cs (1)
40
ByteCodeToIdOffset = _vocab.
TryGetValue
("<0x00>", out (int Id, float Score, byte Type) value) ? value.Id : MaxByteId;
Model\WordPieceTokenizer.cs (1)
62
if (!vocab!.
TryGetValue
(options.UnknownToken, out int id))