1 write to ContinuingSubwordPrefix
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
313ContinuingSubwordPrefix = continuingSubwordPrefix;
20 references to ContinuingSubwordPrefix
Microsoft.ML.Tokenizers (20)
Model\BPETokenizer.cs (20)
361int prefixLen = ContinuingSubwordPrefix is null ? 0 : ContinuingSubwordPrefix.Length; 829if (ContinuingSubwordPrefix is not null) 831sb.Replace(ContinuingSubwordPrefix, string.Empty); 863if (ContinuingSubwordPrefix is not null && span.StartsWith(ContinuingSubwordPrefix.AsSpan(), StringComparison.Ordinal)) 865span = span.Slice(ContinuingSubwordPrefix.Length); 919bool continuingSubwordPrefix = ContinuingSubwordPrefix is not null && ContinuingSubwordPrefix.Length > 0; 949if (continuingSubwordPrefix && sSpan.StartsWith(ContinuingSubwordPrefix.AsSpan(), StringComparison.Ordinal)) 951sSpan = sSpan.Slice(ContinuingSubwordPrefix!.Length); 1039if (ContinuingSubwordPrefix is not null && span.StartsWith(ContinuingSubwordPrefix.AsSpan(), StringComparison.Ordinal)) 1041span = span.Slice(ContinuingSubwordPrefix.Length); 1215if (i > 0 && ContinuingSubwordPrefix is not null) 1217if (ContinuingSubwordPrefix.Length + s.Length <= buffer.Length) 1219ContinuingSubwordPrefix.AsSpan().CopyTo(buffer); 1220s.CopyTo(buffer.Slice(ContinuingSubwordPrefix.Length)); 1221s = buffer.Slice(0, ContinuingSubwordPrefix.Length + s.Length); 1226s = $"{ContinuingSubwordPrefix}{s}".AsSpan();