Normalize

6 overrides of Normalize

Microsoft.ML.Tokenizers (4)

Normalizer\BertNormalizer.cs (1)

29public override string Normalize(string original)

Normalizer\LowerCaseNormalizer.cs (1)

31public override string Normalize(string original) => original.ToLowerInvariant();

Normalizer\SentencePieceNormalizer.cs (1)

61public override string Normalize(string original)

Normalizer\UpperCaseNormalizer.cs (1)

31public override string Normalize(string original) => original.ToUpperInvariant();

Microsoft.ML.Tokenizers.Tests (2)

NormalizerTests.cs (2)

71public override string Normalize(string original) 141public override string Normalize(string original)

6 references to Normalize

Microsoft.ML.Tokenizers (3)

Model\SentencePieceTokenizer.cs (1)

243normalizedText = text is not null ? _normalizer.Normalize(text) : _normalizer.Normalize(textSpan);

Tokenizer.cs (2)

445normalizedText = normalizer.Normalize(textSpan.ToString()); 467normalizedText = normalizer.Normalize(text);

Microsoft.ML.Tokenizers.Tests (3)

LlamaTests.cs (2)

262string normalizedInput = llamaTokenizer.Normalizer!.Normalize(input); 358Assert.Equal("▁Hello,▁World!", llamaTokenizer.Normalizer.Normalize("Hello, World!"));

NormalizerTests.cs (1)

61string? normalizedText = normalizer.Normalize(text);