4 instantiations of ValueStringBuilder
Microsoft.ML.Tokenizers (4)
Model\BPETokenizer.cs (1)
788ValueStringBuilder sb = new ValueStringBuilder();
Model\EnglishRobertaTokenizer.cs (1)
771ValueStringBuilder sb = new ValueStringBuilder();
Model\SentencePieceBaseModel.cs (1)
159ValueStringBuilder sb = new(stackalloc char[256]);
Model\WordPieceTokenizer.cs (1)
681ValueStringBuilder sb = new ValueStringBuilder();
8 references to ValueStringBuilder
Microsoft.ML.Tokenizers (8)
Model\BPETokenizer.cs (1)
788ValueStringBuilder sb = new ValueStringBuilder();
Model\EnglishRobertaTokenizer.cs (1)
771ValueStringBuilder sb = new ValueStringBuilder();
Model\SentencePieceBaseModel.cs (5)
159ValueStringBuilder sb = new(stackalloc char[256]); 275static void FlushBytes(ref int bytesCount, ref byte[]? bytesPoolArray, ref char[]? charPoolArray, ref ValueStringBuilder sb) 295static void EncodeByte(int id, int oneByteUtf8EncodingMaxId, int byteCodeToIdOffset, ref int bytesCount, ref byte[]? bytesPoolArray, ref ValueStringBuilder sb) 309static void AppendTokenWithCheckingPrefix(bool addDummyPrefix, bool treatWhitespaceAsSuffix, string token, char prefixSuffixChar, ref ValueStringBuilder sb, ref bool prefixRemoved, ref int suffixIndex) 338static bool TryDecodeAsSpecialToken(SentencePieceBaseModel model, int id, bool considerSpecialTokens, ref ValueStringBuilder sb)
Model\WordPieceTokenizer.cs (1)
681ValueStringBuilder sb = new ValueStringBuilder();