4 writes to LowerCaseBeforeTokenization
Microsoft.ML.Tokenizers.Tests (4)
BertTokenizerTests.cs (4)
184BertTokenizer[] bertTokenizers = [BertTokenizer.Create(vocabFile, new BertOptions { LowerCaseBeforeTokenization = false }), 185BertTokenizer.Create(vocabStream, new BertOptions { LowerCaseBeforeTokenization = false })]; 252bertTokenizer = await BertTokenizer.CreateAsync(vocabStream, new BertOptions { LowerCaseBeforeTokenization = false }); // no lowercasing and no accent stripping 281bertTokenizer = await BertTokenizer.CreateAsync(vocabStream, new BertOptions { LowerCaseBeforeTokenization = false, RemoveNonSpacingMarks = true }); // no lowercasing and accent stripping
3 references to LowerCaseBeforeTokenization
Microsoft.ML.Tokenizers (3)
Model\BertTokenizer.cs (3)
34LowerCaseBeforeTokenization = options!.LowerCaseBeforeTokenization; 763options.Normalizer ??= options.ApplyBasicTokenization ? new BertNormalizer(options.LowerCaseBeforeTokenization, options.IndividuallyTokenizeCjk, options.RemoveNonSpacingMarks) : null; 768bool lowerCase = options.ApplyBasicTokenization && options.LowerCaseBeforeTokenization;