3 references to Append
Microsoft.ML.Tokenizers (3)
Model\SentencePieceBaseModel.cs (2)
291sb.Append(charPoolArray.AsSpan(0, charCount)); 332sb.Append(!prefixRemoved && token[0] == prefixSuffixChar ? token.AsSpan(1) : token.AsSpan());
Model\WordPieceTokenizer.cs (1)
696sb.Append(token.AsSpan().Slice(ContinuingSubwordPrefix.Length));