4 instantiations of ValueStringBuilder
Microsoft.ML.Tokenizers (4)
Model\BPETokenizer.cs (1)
793ValueStringBuilder sb = new ValueStringBuilder();
Model\EnglishRobertaTokenizer.cs (1)
771ValueStringBuilder sb = new ValueStringBuilder();
Model\SentencePieceBaseModel.cs (1)
203ValueStringBuilder sb = new(stackalloc char[256]);
Model\WordPieceTokenizer.cs (1)
681ValueStringBuilder sb = new ValueStringBuilder();
8 references to ValueStringBuilder
Microsoft.ML.Tokenizers (8)
Model\BPETokenizer.cs (1)
793ValueStringBuilder sb = new ValueStringBuilder();
Model\EnglishRobertaTokenizer.cs (1)
771ValueStringBuilder sb = new ValueStringBuilder();
Model\SentencePieceBaseModel.cs (5)
203ValueStringBuilder sb = new(stackalloc char[256]); 319static void FlushBytes(ref int bytesCount, ref byte[]? bytesPoolArray, ref char[]? charPoolArray, ref ValueStringBuilder sb) 339static void EncodeByte(int id, int oneByteUtf8EncodingMaxId, int byteCodeToIdOffset, ref int bytesCount, ref byte[]? bytesPoolArray, ref ValueStringBuilder sb) 353static void AppendTokenWithCheckingPrefix(bool addDummyPrefix, bool treatWhitespaceAsSuffix, string token, char prefixSuffixChar, ref ValueStringBuilder sb, ref bool prefixRemoved, ref int suffixIndex) 382static bool TryDecodeAsSpecialToken(SentencePieceBaseModel model, int id, bool considerSpecialTokens, ref ValueStringBuilder sb)
Model\WordPieceTokenizer.cs (1)
681ValueStringBuilder sb = new ValueStringBuilder();