1 write to ContinuingSubwordPrefix
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
241ContinuingSubwordPrefix = continuingSubwordPrefix;
14 references to ContinuingSubwordPrefix
Microsoft.ML.Tokenizers (14)
Model\BPETokenizer.cs (14)
265int prefixLen = ContinuingSubwordPrefix is null ? 0 : ContinuingSubwordPrefix.Length; 668if (ContinuingSubwordPrefix is not null) 670sb.Replace(ContinuingSubwordPrefix, string.Empty); 711bool continuingSubwordPrefix = ContinuingSubwordPrefix is not null && ContinuingSubwordPrefix.Length > 0; 741if (continuingSubwordPrefix && sSpan.StartsWith(ContinuingSubwordPrefix.AsSpan(), StringComparison.Ordinal)) 743sSpan = sSpan.Slice(ContinuingSubwordPrefix!.Length); 889if (i > 0 && ContinuingSubwordPrefix is not null) 891if (ContinuingSubwordPrefix.Length + s.Length <= buffer.Length) 893ContinuingSubwordPrefix.AsSpan().CopyTo(buffer); 894s.CopyTo(buffer.Slice(ContinuingSubwordPrefix.Length)); 895s = buffer.Slice(0, ContinuingSubwordPrefix.Length + s.Length); 900s = $"{ContinuingSubwordPrefix}{s}".AsSpan();