1 instantiation of LlamaTokenizer
Microsoft.ML.Tokenizers (1)
Model\LlamaTokenizer.cs (1)
57
return new
LlamaTokenizer
(modelProto, addBeginOfSentence, addEndOfSentence, specialTokens);
20 references to LlamaTokenizer
Microsoft.ML.Tokenizers (1)
Model\LlamaTokenizer.cs (1)
34
public static new
LlamaTokenizer
Create(
Microsoft.ML.Tokenizers.Tests (19)
LlamaTests.cs (19)
33
return
LlamaTokenizer
.Create(remoteStream);
40
return
LlamaTokenizer
.Create(remoteStream);
47
LlamaTokenizer
tokenizer =
LlamaTokenizer
.Create(remoteStream, addBeginOfSentence: true, addEndOfSentence: false,
241
LlamaTokenizer
bpe = (llamaTokenizer as
LlamaTokenizer
)!;
298
private void TestDecodingWithSpan(
LlamaTokenizer
tokenizer, int[] ids, string expectedDecoded)
354
LlamaTokenizer
? bpe = llamaTokenizer as
LlamaTokenizer
;
384
public void TestDecodeSpecialTokenWithSmallId(
LlamaTokenizer
llamaTokenizer)
433
normalizer = new SentencePieceNormalizer(removeExtraWhiteSpaces: false, addDummyPrefix: true, escapeWhiteSpaces: true, treatWhitespaceAsSuffix: false, specialTokens: (_llamaPhi3Tokenizer as
LlamaTokenizer
)!.SpecialTokens);
441
normalizer = new SentencePieceNormalizer(removeExtraWhiteSpaces: false, addDummyPrefix: true, escapeWhiteSpaces: true, treatWhitespaceAsSuffix: true, specialTokens: (_llamaPhi3Tokenizer as
LlamaTokenizer
)!.SpecialTokens);
653
LlamaTokenizer
tokenizer = (_llamaPhi3Tokenizer as
LlamaTokenizer
)!;
834
LlamaTokenizer
tokenizer = (_llamaPhi3Tokenizer as
LlamaTokenizer
)!;
872
LlamaTokenizer
tokenizerWithSuffix = (_llamaPhi3TokenizerWithTreatSpaceSuffix as
LlamaTokenizer
)!;
920
Assert.ThrowsAny<ArgumentException>(() =>
LlamaTokenizer
.Create(null!));