1 write to EndOfWordSuffix
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
242EndOfWordSuffix = endOfWordSuffix;
12 references to EndOfWordSuffix
Microsoft.ML.Tokenizers (12)
Model\BPETokenizer.cs (12)
661if (EndOfWordSuffix is not null) 663sb.RemoveSuffix(EndOfWordSuffix); 665sb.Replace(EndOfWordSuffix, " "); 712bool endOfWordSuffix = EndOfWordSuffix is not null && EndOfWordSuffix.Length > 0; 747if (endOfWordSuffix && sSpan.EndsWith(EndOfWordSuffix!.AsSpan(), StringComparison.Ordinal)) 749sSpan = sSpan.Slice(0, sSpan.Length - EndOfWordSuffix!.Length); 909if (i + length >= w.Length && EndOfWordSuffix is not null) 911if (s.Length + EndOfWordSuffix.Length <= buffer.Length) 914EndOfWordSuffix.AsSpan().CopyTo(buffer.Slice(s.Length)); 915s = buffer.Slice(0, s.Length + EndOfWordSuffix.Length); 920s = $"{s}{EndOfWordSuffix}".AsSpan();