2 references to CreateLPhi3Tokenizer
Microsoft.ML.Tokenizers.Tests (2)
LlamaTests.cs (2)
24
private static Tokenizer _llamaPhi3Tokenizer =
CreateLPhi3Tokenizer
();
25
private static Tokenizer _llamaPhi3TokenizerWithTreatSpaceSuffix =
CreateLPhi3Tokenizer
(treatWhitespaceAsSuffix: true);