1 write to ContinuingSubwordPrefix
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
203
ContinuingSubwordPrefix
= continuingSubwordPrefix;
14 references to ContinuingSubwordPrefix
Microsoft.ML.Tokenizers (14)
Model\BPETokenizer.cs (14)
220
int prefixLen =
ContinuingSubwordPrefix
is null ? 0 :
ContinuingSubwordPrefix
.Length;
623
if (
ContinuingSubwordPrefix
is not null)
625
sb.Replace(
ContinuingSubwordPrefix
, string.Empty);
666
bool continuingSubwordPrefix =
ContinuingSubwordPrefix
is not null &&
ContinuingSubwordPrefix
.Length > 0;
696
if (continuingSubwordPrefix && sSpan.StartsWith(
ContinuingSubwordPrefix
.AsSpan(), StringComparison.Ordinal))
698
sSpan = sSpan.Slice(
ContinuingSubwordPrefix
!.Length);
845
if (i > 0 &&
ContinuingSubwordPrefix
is not null)
847
if (
ContinuingSubwordPrefix
.Length + s.Length <= buffer.Length)
849
ContinuingSubwordPrefix
.AsSpan().CopyTo(buffer);
850
s.CopyTo(buffer.Slice(
ContinuingSubwordPrefix
.Length));
851
s = buffer.Slice(0,
ContinuingSubwordPrefix
.Length + s.Length);
856
s = $"{
ContinuingSubwordPrefix
}{s}".AsSpan();