3 references to GetString
Microsoft.ML.Tokenizers (3)
Model\BPETokenizer.cs (1)
868
string decodedString = Helpers.
GetString
(bytes.AsSpan(0, bytesIndex));
Model\SentencePieceUnigramModel.cs (1)
362
string stringToken = node.Id == UnknownId ? Helpers.
GetString
(normalizationSpan.Slice(node.StartsAt, endsAt - node.StartsAt)) : _vocabReverse[node.Id].Piece;
Model\TiktokenTokenizer.cs (1)
829
return Helpers.
GetString
(utf8Bytes.Slice(0, utf8ByteCount));