4 instantiations of ValueStringBuilder
Microsoft.ML.Tokenizers (4)
Model\BPETokenizer.cs (1)
631ValueStringBuilder sb = new ValueStringBuilder();
Model\EnglishRobertaTokenizer.cs (1)
771ValueStringBuilder sb = new ValueStringBuilder();
Model\SentencePieceBaseModel.cs (1)
220ValueStringBuilder sb = new(stackalloc char[256]);
Model\WordPieceTokenizer.cs (1)
681ValueStringBuilder sb = new ValueStringBuilder();
8 references to ValueStringBuilder
Microsoft.ML.Tokenizers (8)
Model\BPETokenizer.cs (1)
631ValueStringBuilder sb = new ValueStringBuilder();
Model\EnglishRobertaTokenizer.cs (1)
771ValueStringBuilder sb = new ValueStringBuilder();
Model\SentencePieceBaseModel.cs (5)
220ValueStringBuilder sb = new(stackalloc char[256]); 336static void FlushBytes(ref int bytesCount, ref byte[]? bytesPoolArray, ref char[]? charPoolArray, ref ValueStringBuilder sb) 356static void EncodeByte(int id, int oneByteUtf8EncodingMaxId, int byteCodeToIdOffset, ref int bytesCount, ref byte[]? bytesPoolArray, ref ValueStringBuilder sb) 370static void AppendTokenWithCheckingPrefix(bool addDummyPrefix, bool treatWhitespaceAsSuffix, string token, char prefixSuffixChar, ref ValueStringBuilder sb, ref bool prefixRemoved, ref int suffixIndex) 399static bool TryDecodeAsSpecialToken(SentencePieceBaseModel model, int id, bool considerSpecialTokens, ref ValueStringBuilder sb)
Model\WordPieceTokenizer.cs (1)
681ValueStringBuilder sb = new ValueStringBuilder();