1 write to IndividuallyTokenizeCjk
Microsoft.ML.Tokenizers.Tests (1)
BertTokenizerTests.cs (1)
240
bertTokenizer = await BertTokenizer.CreateAsync(vocabStream, new BertOptions {
IndividuallyTokenizeCjk
= false }); // do not tokenize Chinese characters
2 references to IndividuallyTokenizeCjk
Microsoft.ML.Tokenizers (2)
Model\BertTokenizer.cs (2)
50
IndividuallyTokenizeCjk = options.
IndividuallyTokenizeCjk
;
763
options.Normalizer ??= options.ApplyBasicTokenization ? new BertNormalizer(options.LowerCaseBeforeTokenization, options.
IndividuallyTokenizeCjk
, options.RemoveNonSpacingMarks) : null;