5 overrides of Normalizer
Microsoft.ML.Tokenizers (5)
Model\BPETokenizer.cs (1)
260
public override Normalizer?
Normalizer
=> _normalizer;
Model\CodeGenTokenizer.cs (1)
254
public override Normalizer?
Normalizer
=> _normalizer;
Model\EnglishRobertaTokenizer.cs (1)
249
public override Normalizer?
Normalizer
=> _normalizer;
Model\SentencePieceBpeTokenizer.cs (1)
163
public override Normalizer?
Normalizer
=> _normalizer;
Model\TiktokenTokenizer.cs (1)
131
public override Normalizer?
Normalizer
=> _normalizer;
6 references to Normalizer
Microsoft.ML.Tokenizers.Tests (6)
LlamaTests.cs (5)
261
Assert.NotNull(llamaTokenizer.
Normalizer
);
262
string normalizedInput = llamaTokenizer.
Normalizer
!.Normalize(input);
356
Assert.NotNull(llamaTokenizer.
Normalizer
);
358
Assert.Equal("▁Hello,▁World!", llamaTokenizer.
Normalizer
.Normalize("Hello, World!"));
497
Assert.NotNull(tokenizer.
Normalizer
);
TokenizerTests.cs (1)
142
Assert.True(tokenizer.
Normalizer
is not null || processedText is null);