1 write to EndOfWordSuffix
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
309EndOfWordSuffix = endOfWordSuffix;
18 references to EndOfWordSuffix
Microsoft.ML.Tokenizers (18)
Model\BPETokenizer.cs (18)
818if (EndOfWordSuffix is not null) 820sb.RemoveSuffix(EndOfWordSuffix); 822sb.Replace(EndOfWordSuffix, " "); 854if (EndOfWordSuffix is not null && span.EndsWith(EndOfWordSuffix.AsSpan(), StringComparison.Ordinal)) 856span = span.Slice(0, span.Length - EndOfWordSuffix.Length); 916bool endOfWordSuffix = EndOfWordSuffix is not null && EndOfWordSuffix.Length > 0; 951if (endOfWordSuffix && sSpan.EndsWith(EndOfWordSuffix!.AsSpan(), StringComparison.Ordinal)) 953sSpan = sSpan.Slice(0, sSpan.Length - EndOfWordSuffix!.Length); 1030if (EndOfWordSuffix is not null && span.EndsWith(EndOfWordSuffix.AsSpan(), StringComparison.Ordinal)) 1032span = span.Slice(0, span.Length - EndOfWordSuffix.Length); 1231if (i + length >= w.Length && EndOfWordSuffix is not null) 1233if (s.Length + EndOfWordSuffix.Length <= buffer.Length) 1236EndOfWordSuffix.AsSpan().CopyTo(buffer.Slice(s.Length)); 1237s = buffer.Slice(0, s.Length + EndOfWordSuffix.Length); 1242s = $"{s}{EndOfWordSuffix}".AsSpan();