2 references to CheckSpecialId
Microsoft.ML.Tokenizers (2)
Model\SentencePieceUnigramModel.cs (2)
119
CheckSpecialId
(addBos, FindSpecialTokenId(ValidateVocab(pieces, unkId), "<s>"), "addBeginningOfSentence"),
120
CheckSpecialId
(addEos, FindSpecialTokenId(pieces, "</s>"), "addEndOfSentence"),