2 references to SentencePieceTokenizer
Microsoft.ML.Tokenizers (2)
Model\LlamaTokenizer.cs (1)
19
internal LlamaTokenizer(ModelProto modelProto, bool addBos, bool addEos, IReadOnlyDictionary<string, int>? addedTokens = null) :
base
(modelProto, addBos, addEos, addedTokens)
Model\SentencePieceTokenizer.cs (1)
468
return new
SentencePieceTokenizer
(modelProto, addBeginOfSentence, addEndOfSentence, specialTokens);