3 instantiations of ValueStringBuilder
Microsoft.ML.Tokenizers (3)
Model\BPETokenizer.cs (1)
586
ValueStringBuilder sb = new
ValueStringBuilder
();
Model\EnglishRobertaTokenizer.cs (1)
760
ValueStringBuilder sb = new
ValueStringBuilder
();
Model\SentencePieceBpeTokenizer.cs (1)
1551
ValueStringBuilder sb =
new
(stackalloc char[256]);
7 references to ValueStringBuilder
Microsoft.ML.Tokenizers (7)
Model\BPETokenizer.cs (1)
586
ValueStringBuilder
sb = new ValueStringBuilder();
Model\EnglishRobertaTokenizer.cs (1)
760
ValueStringBuilder
sb = new ValueStringBuilder();
Model\SentencePieceBpeTokenizer.cs (5)
1551
ValueStringBuilder
sb = new(stackalloc char[256]);
1675
static void FlushBytes(ref int bytesCount, ref byte[]? bytesPoolArray, ref char[]? charPoolArray, ref
ValueStringBuilder
sb)
1695
static void EncodeByte(int id, int oneByteUtf8EncodingMaxId, int byteCodeToIdOffset, ref int bytesCount, ref byte[]? bytesPoolArray, ref
ValueStringBuilder
sb)
1709
static void AppendTokenWithCheckingPrefix(bool addDummyPrefix, bool treatWhitespaceAsSuffix, string token, char prefixSuffixChar, ref
ValueStringBuilder
sb, ref bool prefixRemoved, ref int suffixIndex)
1738
static void TryDecodeAsSpecialToken(SentencePieceBpeTokenizer tokenizer, int id, bool considerSpecialTokens, ref
ValueStringBuilder
sb)