2 writes to RemoveExtraWhitespaces
Microsoft.ML.Tokenizers (2)
SentencepieceModel.cs (2)
3256RemoveExtraWhitespaces = other.RemoveExtraWhitespaces; 3336RemoveExtraWhitespaces = input.ReadBool();
7 references to RemoveExtraWhitespaces
Microsoft.ML.Tokenizers (7)
Model\LlamaTokenizer.cs (1)
55modelProto.NormalizerSpec.RemoveExtraWhitespaces,
Model\SentencePieceBpeTokenizer.cs (1)
76_normalizer = new SentencePieceNormalizer(modelProto.NormalizerSpec.RemoveExtraWhitespaces, AddDummyPrefix, EscapeWhiteSpaces, modelProto.TrainerSpec.TreatWhitespaceAsSuffix, specialTokens);
SentencepieceModel.cs (5)
3099if (RemoveExtraWhitespaces != other.RemoveExtraWhitespaces) return false; 3115if (HasRemoveExtraWhitespaces) hash ^= RemoveExtraWhitespaces.GetHashCode(); 3190output.WriteBool(RemoveExtraWhitespaces); 3256RemoveExtraWhitespaces = other.RemoveExtraWhitespaces;