6 types derived from Normalizer
Microsoft.ML.Tokenizers (4)
Normalizer\BertNormalizer.cs (1)
18
internal sealed class BertNormalizer :
Normalizer
Normalizer\LowerCaseNormalizer.cs (1)
14
public sealed class LowerCaseNormalizer :
Normalizer
Normalizer\SentencePieceNormalizer.cs (1)
15
public sealed class SentencePieceNormalizer :
Normalizer
Normalizer\UpperCaseNormalizer.cs (1)
14
public sealed class UpperCaseNormalizer :
Normalizer
Microsoft.ML.Tokenizers.Tests (2)
NormalizerTests.cs (2)
69
public class RemoveQuotesNormalizer :
Normalizer
133
public class UnicodeNormalizer :
Normalizer
44 references to Normalizer
Microsoft.ML.Tokenizers (42)
Model\BPETokenizer.cs (6)
31
private readonly
Normalizer
? _normalizer;
114
Normalizer
? normalizer = null,
164
Normalizer
? normalizer = null,
200
Normalizer
? normalizer = null,
233
Normalizer
? normalizer,
305
public override
Normalizer
? Normalizer => _normalizer;
Model\CodeGenTokenizer.cs (5)
33
private readonly
Normalizer
? _normalizer;
56
Normalizer
? normalizer = null,
88
Normalizer
? normalizer = null,
100
private CodeGenTokenizer(Stream vocabularyStream, Stream mergeStream, PreTokenizer? preTokenizer,
Normalizer
? normalizer, IReadOnlyDictionary<string, int>? specialTokens, bool addPrefixSpace,
254
public override
Normalizer
? Normalizer => _normalizer;
Model\EnglishRobertaTokenizer.cs (7)
29
private readonly
Normalizer
? _normalizer;
68
Normalizer
? normalizer = null,
105
Normalizer
? normalizer = null,
118
internal EnglishRobertaTokenizer(string vocabularyPath, string mergePath, string highestOccurrenceMappingPath, PreTokenizer? preTokenizer = null,
Normalizer
? normalizer = null, bool filterUnsupportedChars = true) :
135
internal EnglishRobertaTokenizer(Stream vocabularyStream, Stream mergeStream, Stream highestOccurrenceMappingStream, PreTokenizer? preTokenizer = null,
Normalizer
? normalizer = null, bool filterUnsupportedChars = true) :
140
private EnglishRobertaTokenizer(Stream vocabularyStream, Stream mergeStream, Stream highestOccurrenceMappingStream, PreTokenizer? preTokenizer,
Normalizer
? normalizer, bool filterUnsupportedChars, bool disposeStream)
260
public override
Normalizer
? Normalizer => _normalizer;
Model\Phi2Tokenizer.cs (2)
38
Normalizer
? normalizer = null,
69
Normalizer
? normalizer = null,
Model\SentencePieceTokenizer.cs (2)
34
private readonly
Normalizer
? _normalizer;
164
public override
Normalizer
? Normalizer => _normalizer;
Model\TiktokenTokenizer.cs (15)
34
private readonly
Normalizer
? _normalizer;
46
internal TiktokenTokenizer(string vocabFilePath, PreTokenizer? preTokenizer, IReadOnlyDictionary<string, int>? specialTokens = null,
Normalizer
? normalizer = null, int cacheSize = LruCache<int[]>.DefaultCacheSize) :
61
internal TiktokenTokenizer(Stream vocabStream, PreTokenizer? preTokenizer, IReadOnlyDictionary<string, int>? specialTokens = null,
Normalizer
? normalizer = null, int cacheSize = LruCache<int[]>.DefaultCacheSize) :
82
Normalizer
? normalizer = null,
102
private TiktokenTokenizer(Stream vocabStream, PreTokenizer? preTokenizer, IReadOnlyDictionary<string, int>? specialTokens,
Normalizer
? normalizer, int cacheSize, bool disposeStream)
132
public override
Normalizer
? Normalizer => _normalizer;
1207
Normalizer
? normalizer = null)
1261
Normalizer
? normalizer,
1281
Normalizer
? normalizer,
1302
Normalizer
? normalizer,
1334
Normalizer
? normalizer,
1362
Normalizer
? normalizer = null)
1401
Normalizer
? normalizer = null,
1433
public static TiktokenTokenizer CreateForModel(string modelName, IReadOnlyDictionary<string, int>? extraSpecialTokens = null,
Normalizer
? normalizer = null)
1443
public static TiktokenTokenizer CreateForEncoding(string encodingName, IReadOnlyDictionary<string, int>? extraSpecialTokens = null,
Normalizer
? normalizer = null)
Model\WordPieceOptions.cs (1)
26
public
Normalizer
? Normalizer { get; set; }
Model\WordPieceTokenizer.cs (2)
28
private readonly
Normalizer
? _normalizer;
248
public override
Normalizer
? Normalizer => _normalizer;
Tokenizer.cs (2)
29
public virtual
Normalizer
? Normalizer => null;
432
Normalizer
? normalizer,
Microsoft.ML.Tokenizers.Tests (2)
BpeTests.cs (1)
550
internal static BpeTokenizer CreateEmptyBpe(PreTokenizer? preTokenizer = null,
Normalizer
? normalizer = null)
NormalizerTests.cs (1)
59
public void TestNormalizer(
Normalizer
normalizer, string text, string normalized)