1 write to _individuallyTokenizeCjk
Microsoft.ML.Tokenizers (1)
Normalizer\BertNormalizer.cs (1)
145
_individuallyTokenizeCjk
= individuallyTokenizeCjk;
1 reference to _individuallyTokenizeCjk
Microsoft.ML.Tokenizers (1)
Normalizer\BertNormalizer.cs (1)
94
if (
_individuallyTokenizeCjk
&& IsCjkChar(codePoint))