4 instantiations of ValueStringBuilder
Microsoft.ML.Tokenizers (4)
Model\BPETokenizer.cs (1)
631ValueStringBuilder sb = new ValueStringBuilder();
Model\EnglishRobertaTokenizer.cs (1)
771ValueStringBuilder sb = new ValueStringBuilder();
Model\SentencePieceTokenizer.cs (1)
1552ValueStringBuilder sb = new(stackalloc char[256]);
Model\WordPieceTokenizer.cs (1)
681ValueStringBuilder sb = new ValueStringBuilder();
8 references to ValueStringBuilder
Microsoft.ML.Tokenizers (8)
Model\BPETokenizer.cs (1)
631ValueStringBuilder sb = new ValueStringBuilder();
Model\EnglishRobertaTokenizer.cs (1)
771ValueStringBuilder sb = new ValueStringBuilder();
Model\SentencePieceTokenizer.cs (5)
1552ValueStringBuilder sb = new(stackalloc char[256]); 1676static void FlushBytes(ref int bytesCount, ref byte[]? bytesPoolArray, ref char[]? charPoolArray, ref ValueStringBuilder sb) 1696static void EncodeByte(int id, int oneByteUtf8EncodingMaxId, int byteCodeToIdOffset, ref int bytesCount, ref byte[]? bytesPoolArray, ref ValueStringBuilder sb) 1710static void AppendTokenWithCheckingPrefix(bool addDummyPrefix, bool treatWhitespaceAsSuffix, string token, char prefixSuffixChar, ref ValueStringBuilder sb, ref bool prefixRemoved, ref int suffixIndex) 1739static void TryDecodeAsSpecialToken(SentencePieceTokenizer tokenizer, int id, bool considerSpecialTokens, ref ValueStringBuilder sb)
Model\WordPieceTokenizer.cs (1)
681ValueStringBuilder sb = new ValueStringBuilder();