3 references to FindPieceId
Microsoft.ML.Tokenizers (3)
Model\SentencePieceTokenizer.cs (3)
743
int padId = mergedSpecialTokens.TryGetValue("<pad>", out int p) ? p :
FindPieceId
(vocab, "<pad>");
952
int vocabId =
FindPieceId
(vocab, tokenName);
995
int id = specialTokens.TryGetValue(tokenName, out int specialId) ? specialId :
FindPieceId
(vocab, tokenName);