1 type derived from SentencePieceTokenizer
Microsoft.ML.Tokenizers (1)
Model\LlamaTokenizer.cs (1)
17
public sealed class LlamaTokenizer :
SentencePieceTokenizer
11 references to SentencePieceTokenizer
Microsoft.ML.GenAI.Core (2)
Pipeline\CausalLMPipeline.cs (2)
275
SentencePieceTokenizer
bpeTokenizer => bpeTokenizer.Decode(tokenIds.Concat(tokenIds), considerSpecialTokens: true) ?? throw new InvalidOperationException("Failed to decode token ids"),
281
SentencePieceTokenizer
bpeTokenizer => bpeTokenizer.Decode(tokenIds, considerSpecialTokens: true) ?? throw new InvalidOperationException("Failed to decode token ids"),
Microsoft.ML.Tokenizers (2)
Model\SentencePieceTokenizer.cs (2)
1739
static void TryDecodeAsSpecialToken(
SentencePieceTokenizer
tokenizer, int id, bool considerSpecialTokens, ref ValueStringBuilder sb)
1983
static OperationStatus TryDecodeAsSpecialToken(
SentencePieceTokenizer
tokenizer, int id, bool considerSpecialTokens, Span<char> buffer, ref int charsWritten)
Microsoft.ML.Tokenizers.Tests (7)
LlamaTests.cs (5)
69
PropertyInfo? propertyInfo = typeof(
SentencePieceTokenizer
).GetProperty("TreatWhitespaceAsSuffix", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public);
511
SentencePieceTokenizer
sentencePieceBpe = (tokenizer as
SentencePieceTokenizer
)!;
565
SentencePieceTokenizer
sentencePieceBpe = (tokenizer as
SentencePieceTokenizer
)!;
TokenizerTests.cs (2)
155
if (tokenizer is
SentencePieceTokenizer
)
173
if (tokenizer is
SentencePieceTokenizer
)