4 references to AddProcessorAffix
Microsoft.ML.Tokenizers (4)
Model\SentencePieceTokenizer.cs (4)
835AddProcessorAffix(postProcessor, "cls", prefixTokens, vocab, specialTokens);
836AddProcessorAffix(postProcessor, "sep", suffixTokens, vocab, specialTokens);
840AddProcessorAffix(postProcessor, "cls", prefixTokens, vocab, specialTokens);
841AddProcessorAffix(postProcessor, "sep", suffixTokens, vocab, specialTokens);