2 writes to TreatWhitespaceAsSuffix
Microsoft.ML.Tokenizers (2)
SentencepieceModel.cs (2)
2316TreatWhitespaceAsSuffix = other.TreatWhitespaceAsSuffix; 2681TreatWhitespaceAsSuffix = input.ReadBool();
8 references to TreatWhitespaceAsSuffix
Microsoft.ML.Tokenizers (8)
Model\LlamaTokenizer.cs (1)
58modelProto.TrainerSpec.TreatWhitespaceAsSuffix,
Model\SentencePieceBpeTokenizer.cs (2)
72TreatWhitespaceAsSuffix = modelProto.TrainerSpec.TreatWhitespaceAsSuffix; 76_normalizer = new SentencePieceNormalizer(modelProto.NormalizerSpec.RemoveExtraWhitespaces, AddDummyPrefix, EscapeWhiteSpaces, modelProto.TrainerSpec.TreatWhitespaceAsSuffix, specialTokens);
SentencepieceModel.cs (5)
1633if (TreatWhitespaceAsSuffix != other.TreatWhitespaceAsSuffix) return false; 1689if (HasTreatWhitespaceAsSuffix) hash ^= TreatWhitespaceAsSuffix.GetHashCode(); 1997output.WriteBool(TreatWhitespaceAsSuffix); 2316TreatWhitespaceAsSuffix = other.TreatWhitespaceAsSuffix;