2 writes to NormalizerSpec
Microsoft.ML.Tokenizers (2)
SentencepieceModel.cs (2)
4151
NormalizerSpec
= new global::Sentencepiece.NormalizerSpec();
4247
NormalizerSpec
= new global::Sentencepiece.NormalizerSpec();
17 references to NormalizerSpec
Microsoft.ML.Tokenizers (17)
Model\LlamaTokenizer.cs (6)
49
if (modelProto.
NormalizerSpec
.Name != "identity" && !string.IsNullOrEmpty(modelProto.
NormalizerSpec
.Name))
51
throw new ArgumentException($"Normalization '{modelProto.
NormalizerSpec
.Name}' is not supported.", nameof(modelProto));
55
modelProto.
NormalizerSpec
.RemoveExtraWhitespaces,
56
modelProto.
NormalizerSpec
.AddDummyPrefix,
57
modelProto.
NormalizerSpec
.EscapeWhitespaces,
Model\SentencePieceBpeTokenizer.cs (3)
70
AddDummyPrefix = modelProto.
NormalizerSpec
.AddDummyPrefix;
71
EscapeWhiteSpaces = modelProto.
NormalizerSpec
.EscapeWhitespaces;
76
_normalizer = new SentencePieceNormalizer(modelProto.
NormalizerSpec
.RemoveExtraWhitespaces, AddDummyPrefix, EscapeWhiteSpaces, modelProto.TrainerSpec.TreatWhitespaceAsSuffix, specialTokens);
SentencepieceModel.cs (8)
4015
if (!object.Equals(
NormalizerSpec
, other.
NormalizerSpec
)) return false;
4030
if (normalizerSpec_ != null) hash ^=
NormalizerSpec
.GetHashCode();
4091
output.WriteMessage(
NormalizerSpec
);
4119
size += 1 + pb::CodedOutputStream.ComputeMessageSize(
NormalizerSpec
);
4153
NormalizerSpec
.MergeFrom(other.
NormalizerSpec
);
4249
input.ReadMessage(
NormalizerSpec
);