4 references to GetString
Microsoft.ML.Tokenizers (4)
Model\BPETokenizer.cs (1)
872
string decodedString = Helpers.
GetString
(bytes.AsSpan(0, bytesIndex));
Model\SentencePieceUnigramModel.cs (1)
637
string stringToken = node.Id == UnknownId ? Helpers.
GetString
(normalizationSpan.Slice(node.StartsAt, endsAt - node.StartsAt)) : _vocabReverse[node.Id].Piece;
Model\TiktokenTokenizer.cs (1)
829
return Helpers.
GetString
(utf8Bytes.Slice(0, utf8ByteCount));
Normalizer\SentencePieceNormalizer.cs (1)
395
string chained = NormalizationChain.Normalize(Helpers.
GetString
(input));