1 type derived from SentencePieceTokenizer
Microsoft.ML.Tokenizers (1)
Model\LlamaTokenizer.cs (1)
17
public sealed class LlamaTokenizer :
SentencePieceTokenizer
11 references to SentencePieceTokenizer
Microsoft.ML.GenAI.Core (2)
Pipeline\CausalLMPipeline.cs (2)
271
SentencePieceTokenizer
bpeTokenizer => bpeTokenizer.Decode(tokenIds.Concat(tokenIds), considerSpecialTokens: true) ?? throw new InvalidOperationException("Failed to decode token ids"),
277
SentencePieceTokenizer
bpeTokenizer => bpeTokenizer.Decode(tokenIds, considerSpecialTokens: true) ?? throw new InvalidOperationException("Failed to decode token ids"),
Microsoft.ML.Tokenizers (2)
Model\SentencePieceTokenizer.cs (2)
1738
static void TryDecodeAsSpecialToken(
SentencePieceTokenizer
tokenizer, int id, bool considerSpecialTokens, ref ValueStringBuilder sb)
1982
static OperationStatus TryDecodeAsSpecialToken(
SentencePieceTokenizer
tokenizer, int id, bool considerSpecialTokens, Span<char> buffer, ref int charsWritten)
Microsoft.ML.Tokenizers.Tests (7)
LlamaTests.cs (5)
69
PropertyInfo? propertyInfo = typeof(
SentencePieceTokenizer
).GetProperty("TreatWhitespaceAsSuffix", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public);
511
SentencePieceTokenizer
sentencePieceBpe = (tokenizer as
SentencePieceTokenizer
)!;
565
SentencePieceTokenizer
sentencePieceBpe = (tokenizer as
SentencePieceTokenizer
)!;
TokenizerTests.cs (2)
155
if (tokenizer is
SentencePieceTokenizer
)
173
if (tokenizer is
SentencePieceTokenizer
)