2 writes to NormalizerSpec
Microsoft.ML.Tokenizers (2)
SentencepieceModel.cs (2)
4151
NormalizerSpec
= new global::Sentencepiece.NormalizerSpec();
4247
NormalizerSpec
= new global::Sentencepiece.NormalizerSpec();
15 references to NormalizerSpec
Microsoft.ML.Tokenizers (15)
Model\LlamaTokenizer.cs (3)
52
if (modelProto.
NormalizerSpec
.Name != "identity" && !string.IsNullOrEmpty(modelProto.
NormalizerSpec
.Name))
54
throw new ArgumentException($"Normalization '{modelProto.
NormalizerSpec
.Name}' is not supported.", nameof(modelProto));
Model\SentencePieceBaseModel.cs (4)
33
AddDummyPrefix = modelProto.
NormalizerSpec
.AddDummyPrefix;
34
EscapeWhiteSpaces = modelProto.
NormalizerSpec
.EscapeWhitespaces;
55
modelProto.
NormalizerSpec
.PrecompiledCharsmap.Span,
56
modelProto.
NormalizerSpec
.RemoveExtraWhitespaces,
SentencepieceModel.cs (8)
4015
if (!object.Equals(
NormalizerSpec
, other.
NormalizerSpec
)) return false;
4030
if (normalizerSpec_ != null) hash ^=
NormalizerSpec
.GetHashCode();
4091
output.WriteMessage(
NormalizerSpec
);
4119
size += 1 + pb::CodedOutputStream.ComputeMessageSize(
NormalizerSpec
);
4153
NormalizerSpec
.MergeFrom(other.
NormalizerSpec
);
4249
input.ReadMessage(
NormalizerSpec
);