10 references to Append
Microsoft.ML.Tokenizers (10)
Model\BPETokenizer.cs (2)
641
sb.
Append
(s);
656
sb.
Append
(s);
Model\EnglishRobertaTokenizer.cs (1)
777
sb.
Append
(s);
Model\SentencePieceTokenizer.cs (6)
1719
sb.
Append
(token);
1725
sb.
Append
(token);
1748
sb.
Append
(tokenizer.BeginningOfSentenceToken);
1752
sb.
Append
(tokenizer.EndOfSentenceToken);
1756
sb.
Append
(tokenizer.UnknownToken);
1760
sb.
Append
(specialToken);
Model\WordPieceTokenizer.cs (1)
705
sb.
Append
(token);