1 write to ContinuingSubwordPrefix
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
308ContinuingSubwordPrefix = continuingSubwordPrefix;
20 references to ContinuingSubwordPrefix
Microsoft.ML.Tokenizers (20)
Model\BPETokenizer.cs (20)
356int prefixLen = ContinuingSubwordPrefix is null ? 0 : ContinuingSubwordPrefix.Length; 825if (ContinuingSubwordPrefix is not null) 827sb.Replace(ContinuingSubwordPrefix, string.Empty); 859if (ContinuingSubwordPrefix is not null && span.StartsWith(ContinuingSubwordPrefix.AsSpan(), StringComparison.Ordinal)) 861span = span.Slice(ContinuingSubwordPrefix.Length); 915bool continuingSubwordPrefix = ContinuingSubwordPrefix is not null && ContinuingSubwordPrefix.Length > 0; 945if (continuingSubwordPrefix && sSpan.StartsWith(ContinuingSubwordPrefix.AsSpan(), StringComparison.Ordinal)) 947sSpan = sSpan.Slice(ContinuingSubwordPrefix!.Length); 1035if (ContinuingSubwordPrefix is not null && span.StartsWith(ContinuingSubwordPrefix.AsSpan(), StringComparison.Ordinal)) 1037span = span.Slice(ContinuingSubwordPrefix.Length); 1211if (i > 0 && ContinuingSubwordPrefix is not null) 1213if (ContinuingSubwordPrefix.Length + s.Length <= buffer.Length) 1215ContinuingSubwordPrefix.AsSpan().CopyTo(buffer); 1216s.CopyTo(buffer.Slice(ContinuingSubwordPrefix.Length)); 1217s = buffer.Slice(0, ContinuingSubwordPrefix.Length + s.Length); 1222s = $"{ContinuingSubwordPrefix}{s}".AsSpan();