1 write to EndOfWordSuffix
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
314EndOfWordSuffix = endOfWordSuffix;
18 references to EndOfWordSuffix
Microsoft.ML.Tokenizers (18)
Model\BPETokenizer.cs (18)
822if (EndOfWordSuffix is not null) 824sb.RemoveSuffix(EndOfWordSuffix); 826sb.Replace(EndOfWordSuffix, " "); 858if (EndOfWordSuffix is not null && span.EndsWith(EndOfWordSuffix.AsSpan(), StringComparison.Ordinal)) 860span = span.Slice(0, span.Length - EndOfWordSuffix.Length); 920bool endOfWordSuffix = EndOfWordSuffix is not null && EndOfWordSuffix.Length > 0; 955if (endOfWordSuffix && sSpan.EndsWith(EndOfWordSuffix!.AsSpan(), StringComparison.Ordinal)) 957sSpan = sSpan.Slice(0, sSpan.Length - EndOfWordSuffix!.Length); 1034if (EndOfWordSuffix is not null && span.EndsWith(EndOfWordSuffix.AsSpan(), StringComparison.Ordinal)) 1036span = span.Slice(0, span.Length - EndOfWordSuffix.Length); 1235if (i + length >= w.Length && EndOfWordSuffix is not null) 1237if (s.Length + EndOfWordSuffix.Length <= buffer.Length) 1240EndOfWordSuffix.AsSpan().CopyTo(buffer.Slice(s.Length)); 1241s = buffer.Slice(0, s.Length + EndOfWordSuffix.Length); 1246s = $"{s}{EndOfWordSuffix}".AsSpan();