3 references to EncodeInternal
Microsoft.ML.Tokenizers (3)
Model\SentencePieceTokenizer.cs (3)
264EncodeInternal(textToEncode, addBeginningOfSentence, addEndOfSentence, tokens);
285EncodeInternal(text.Slice(currentOffset, Offset - currentOffset), addBeginOfSentence: false, addEndOfSentence: false, tokens);
298EncodeInternal(text.Slice(currentOffset), addBeginOfSentence: false, addEndOfSentence: false, tokens);