3 references to Append
Microsoft.ML.Tokenizers (3)
Model\SentencePieceBaseModel.cs (2)
291
sb.
Append
(charPoolArray.AsSpan(0, charCount));
332
sb.
Append
(!prefixRemoved && token[0] == prefixSuffixChar ? token.AsSpan(1) : token.AsSpan());
Model\WordPieceTokenizer.cs (1)
696
sb.
Append
(token.AsSpan().Slice(ContinuingSubwordPrefix.Length));