6 overrides of Normalizer
Microsoft.ML.Tokenizers (6)
Model\BPETokenizer.cs (1)
305
public override Normalizer?
Normalizer
=> _normalizer;
Model\CodeGenTokenizer.cs (1)
254
public override Normalizer?
Normalizer
=> _normalizer;
Model\EnglishRobertaTokenizer.cs (1)
260
public override Normalizer?
Normalizer
=> _normalizer;
Model\SentencePieceTokenizer.cs (1)
164
public override Normalizer?
Normalizer
=> _normalizer;
Model\TiktokenTokenizer.cs (1)
132
public override Normalizer?
Normalizer
=> _normalizer;
Model\WordPieceTokenizer.cs (1)
248
public override Normalizer?
Normalizer
=> _normalizer;
6 references to Normalizer
Microsoft.ML.Tokenizers.Tests (6)
LlamaTests.cs (5)
261
Assert.NotNull(llamaTokenizer.
Normalizer
);
262
string normalizedInput = llamaTokenizer.
Normalizer
!.Normalize(input);
356
Assert.NotNull(llamaTokenizer.
Normalizer
);
358
Assert.Equal("▁Hello,▁World!", llamaTokenizer.
Normalizer
.Normalize("Hello, World!"));
497
Assert.NotNull(tokenizer.
Normalizer
);
TokenizerTests.cs (1)
142
Assert.True(tokenizer.
Normalizer
is not null || processedText is null);