5 references to CreateCodegen350MMonoTokenizer
Microsoft.ML.Tokenizers.Tests (5)
CodeGenTests.cs (5)
17
private static Tokenizer _codegen350MMonoTokenizer =
CreateCodegen350MMonoTokenizer
();
18
private static Tokenizer _codegen350MMonoTokenizerWithSpace =
CreateCodegen350MMonoTokenizer
(addPrefixSpace: true);
19
private static Tokenizer _codegen350MMonoTokenizerWithBeginningOfSentence =
CreateCodegen350MMonoTokenizer
(bos: true);
20
private static Tokenizer _codegen350MMonoTokenizerWithEndOfSentence =
CreateCodegen350MMonoTokenizer
(eos: true);
21
private static Tokenizer _codegen350MMonoTokenizerWithBeginningAndEndOfSentence =
CreateCodegen350MMonoTokenizer
(bos: true, eos: true);