Base:
method
Normalize
Microsoft.ML.Tokenizers.Normalizer.Normalize(System.ReadOnlySpan<System.Char>)
10 references to Normalize
Microsoft.ML.Tokenizers (1)
Normalizer\SentencePieceNormalizer.cs (1)
68return Normalize(original.AsSpan());
Microsoft.ML.Tokenizers.Tests (9)
LlamaTests.cs (9)
399Assert.Equal("Hello, World!", normalizer.Normalize("Hello, World!".AsSpan())); 403Assert.Equal("Hello, World!", normalizer.Normalize("Hello, World!".AsSpan())); 407Assert.Equal(" Hello, World!", normalizer.Normalize("Hello, World!".AsSpan())); 411Assert.Equal("▁Hello,▁World!", normalizer.Normalize("Hello, World!".AsSpan())); 415Assert.Equal("▁Hello,▁▁▁▁▁▁World!", normalizer.Normalize("Hello, World!".AsSpan())); 419Assert.Equal("Hello,▁World!▁", normalizer.Normalize("Hello, World!".AsSpan())); 423Assert.Equal("Hello,▁World!", normalizer.Normalize("Hello, World!".AsSpan())); 427Assert.Equal("Hello,▁▁▁▁▁▁World!▁", normalizer.Normalize("Hello, World!".AsSpan())); 431Assert.Equal("Hello, World! ", normalizer.Normalize("Hello, World!".AsSpan()));