6 references to _llamaPhi3Tokenizer
Microsoft.ML.Tokenizers.Tests (6)
LlamaTests.cs (6)
237
Tokenizer[] tokenizers = tokenizer == _llamaTokenizer ? new[] { tokenizer,
_llamaPhi3Tokenizer
} : new[] { tokenizer };
331
yield return new object[] {
_llamaPhi3Tokenizer
};
433
normalizer = new SentencePieceNormalizer(removeExtraWhiteSpaces: false, addDummyPrefix: true, escapeWhiteSpaces: true, treatWhitespaceAsSuffix: false, specialTokens: (
_llamaPhi3Tokenizer
as LlamaTokenizer)!.SpecialTokens);
441
normalizer = new SentencePieceNormalizer(removeExtraWhiteSpaces: false, addDummyPrefix: true, escapeWhiteSpaces: true, treatWhitespaceAsSuffix: true, specialTokens: (
_llamaPhi3Tokenizer
as LlamaTokenizer)!.SpecialTokens);
653
LlamaTokenizer tokenizer = (
_llamaPhi3Tokenizer
as LlamaTokenizer)!;
834
LlamaTokenizer tokenizer = (
_llamaPhi3Tokenizer
as LlamaTokenizer)!;