1 write to ContinuingSubwordPrefix
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
203ContinuingSubwordPrefix = continuingSubwordPrefix;
14 references to ContinuingSubwordPrefix
Microsoft.ML.Tokenizers (14)
Model\BPETokenizer.cs (14)
220int prefixLen = ContinuingSubwordPrefix is null ? 0 : ContinuingSubwordPrefix.Length; 623if (ContinuingSubwordPrefix is not null) 625sb.Replace(ContinuingSubwordPrefix, string.Empty); 666bool continuingSubwordPrefix = ContinuingSubwordPrefix is not null && ContinuingSubwordPrefix.Length > 0; 696if (continuingSubwordPrefix && sSpan.StartsWith(ContinuingSubwordPrefix.AsSpan(), StringComparison.Ordinal)) 698sSpan = sSpan.Slice(ContinuingSubwordPrefix!.Length); 845if (i > 0 && ContinuingSubwordPrefix is not null) 847if (ContinuingSubwordPrefix.Length + s.Length <= buffer.Length) 849ContinuingSubwordPrefix.AsSpan().CopyTo(buffer); 850s.CopyTo(buffer.Slice(ContinuingSubwordPrefix.Length)); 851s = buffer.Slice(0, ContinuingSubwordPrefix.Length + s.Length); 856s = $"{ContinuingSubwordPrefix}{s}".AsSpan();