3 instantiations of ValueStringBuilder
Microsoft.ML.Tokenizers (3)
Model\BPETokenizer.cs (1)
586ValueStringBuilder sb = new ValueStringBuilder();
Model\EnglishRobertaTokenizer.cs (1)
760ValueStringBuilder sb = new ValueStringBuilder();
Model\SentencePieceBpeTokenizer.cs (1)
1551ValueStringBuilder sb = new(stackalloc char[256]);
7 references to ValueStringBuilder
Microsoft.ML.Tokenizers (7)
Model\BPETokenizer.cs (1)
586ValueStringBuilder sb = new ValueStringBuilder();
Model\EnglishRobertaTokenizer.cs (1)
760ValueStringBuilder sb = new ValueStringBuilder();
Model\SentencePieceBpeTokenizer.cs (5)
1551ValueStringBuilder sb = new(stackalloc char[256]); 1675static void FlushBytes(ref int bytesCount, ref byte[]? bytesPoolArray, ref char[]? charPoolArray, ref ValueStringBuilder sb) 1695static void EncodeByte(int id, int oneByteUtf8EncodingMaxId, int byteCodeToIdOffset, ref int bytesCount, ref byte[]? bytesPoolArray, ref ValueStringBuilder sb) 1709static void AppendTokenWithCheckingPrefix(bool addDummyPrefix, bool treatWhitespaceAsSuffix, string token, char prefixSuffixChar, ref ValueStringBuilder sb, ref bool prefixRemoved, ref int suffixIndex) 1738static void TryDecodeAsSpecialToken(SentencePieceBpeTokenizer tokenizer, int id, bool considerSpecialTokens, ref ValueStringBuilder sb)