9 types derived from SentencePieceNormalizationStep
Microsoft.ML.Tokenizers (9)
Normalizer\SentencePieceNormalizationStep.cs (9)
248private sealed class SequenceStep : SentencePieceNormalizationStep 264private sealed class LowercaseStep : SentencePieceNormalizationStep 271private sealed class StripAccentsStep : SentencePieceNormalizationStep 310private sealed class UnicodeStep : SentencePieceNormalizationStep 317private sealed class PrependStep : SentencePieceNormalizationStep 326private sealed class NmtStep : SentencePieceNormalizationStep 374private sealed class StripStep : SentencePieceNormalizationStep 399private sealed class ReplaceStep : SentencePieceNormalizationStep 471private sealed class PrecompiledStep : SentencePieceNormalizationStep
10 references to SentencePieceNormalizationStep
Microsoft.ML.Tokenizers (10)
Model\SentencePieceTokenizer.cs (4)
674SentencePieceNormalizationStep? chainNormalizer = null; 678if (SentencePieceNormalizationStep.HasRichSteps(normalizerElement)) 680chainNormalizer = SentencePieceNormalizationStep.Build(normalizerElement); 1042return SentencePieceNormalizationStep.DecodePrecompiledCharsMap(base64);
Normalizer\SentencePieceNormalizationStep.cs (5)
127public static SentencePieceNormalizationStep Build(JsonElement normalizer) 139var children = new List<SentencePieceNormalizationStep>(); 250private readonly IReadOnlyList<SentencePieceNormalizationStep> _steps; 252public SequenceStep(IReadOnlyList<SentencePieceNormalizationStep> steps) => _steps = steps; 256foreach (SentencePieceNormalizationStep step in _steps)
Normalizer\SentencePieceNormalizer.cs (1)
33internal SentencePieceNormalizationStep? NormalizationChain { get; set; }