1 write to EndOfWordSuffix
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
227EndOfWordSuffix = endOfWordSuffix;
12 references to EndOfWordSuffix
Microsoft.ML.Tokenizers (12)
Model\BPETokenizer.cs (12)
646if (EndOfWordSuffix is not null) 648sb.RemoveSuffix(EndOfWordSuffix); 650sb.Replace(EndOfWordSuffix, " "); 697bool endOfWordSuffix = EndOfWordSuffix is not null && EndOfWordSuffix.Length > 0; 732if (endOfWordSuffix && sSpan.EndsWith(EndOfWordSuffix!.AsSpan(), StringComparison.Ordinal)) 734sSpan = sSpan.Slice(0, sSpan.Length - EndOfWordSuffix!.Length); 894if (i + length >= w.Length && EndOfWordSuffix is not null) 896if (s.Length + EndOfWordSuffix.Length <= buffer.Length) 899EndOfWordSuffix.AsSpan().CopyTo(buffer.Slice(s.Length)); 900s = buffer.Slice(0, s.Length + EndOfWordSuffix.Length); 905s = $"{s}{EndOfWordSuffix}".AsSpan();