Base:
method
Normalize
Microsoft.ML.Tokenizers.Normalizer.Normalize(System.String)
21 references to Normalize
Microsoft.ML.Tokenizers.Tests (21)
LlamaTests.cs (21)
398Assert.Equal("Hello, World!", normalizer.Normalize("Hello, World!")); 402Assert.Equal("Hello, World!", normalizer.Normalize("Hello, World!")); 406Assert.Equal(" Hello, World!", normalizer.Normalize("Hello, World!")); 410Assert.Equal("▁Hello,▁World!", normalizer.Normalize("Hello, World!")); 414Assert.Equal("▁Hello,▁▁▁▁▁▁World!", normalizer.Normalize("Hello, World!")); 418Assert.Equal("Hello,▁World!▁", normalizer.Normalize("Hello, World!")); 422Assert.Equal("Hello,▁World!", normalizer.Normalize("Hello, World!")); 426Assert.Equal("Hello,▁▁▁▁▁▁World!▁", normalizer.Normalize("Hello, World!")); 430Assert.Equal("Hello, World! ", normalizer.Normalize("Hello, World!")); 434Assert.Equal("<|user|>", normalizer.Normalize("<|user|>")); 435Assert.Equal("<|user|><|system|><|assistant|><|endoftext|>", normalizer.Normalize("<|user|><|system|><|assistant|><|endoftext|>")); 436Assert.Equal("▁Hello<|user|>", normalizer.Normalize("Hello<|user|>")); 437Assert.Equal("▁Hello,▁<|user|>World", normalizer.Normalize("Hello, <|user|>World")); 438Assert.Equal("<|endoftext|>▁Hello<|user|>", normalizer.Normalize("<|endoftext|>Hello<|user|>")); 439Assert.Equal("", normalizer.Normalize("")); 442Assert.Equal("<|user|>", normalizer.Normalize("<|user|>")); 443Assert.Equal("<|user|><|system|><|assistant|><|endoftext|>", normalizer.Normalize("<|user|><|system|><|assistant|><|endoftext|>")); 444Assert.Equal("Hello▁<|user|>", normalizer.Normalize("Hello<|user|>")); 445Assert.Equal("Hello,▁<|user|>World▁", normalizer.Normalize("Hello, <|user|>World")); 446Assert.Equal("<|endoftext|>Hello▁<|user|>", normalizer.Normalize("<|endoftext|>Hello<|user|>")); 447Assert.Equal("", normalizer.Normalize(""));