1 write to ContinuingSubwordPrefix
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
313
ContinuingSubwordPrefix
= continuingSubwordPrefix;
20 references to ContinuingSubwordPrefix
Microsoft.ML.Tokenizers (20)
Model\BPETokenizer.cs (20)
361
int prefixLen =
ContinuingSubwordPrefix
is null ? 0 :
ContinuingSubwordPrefix
.Length;
829
if (
ContinuingSubwordPrefix
is not null)
831
sb.Replace(
ContinuingSubwordPrefix
, string.Empty);
863
if (
ContinuingSubwordPrefix
is not null && span.StartsWith(
ContinuingSubwordPrefix
.AsSpan(), StringComparison.Ordinal))
865
span = span.Slice(
ContinuingSubwordPrefix
.Length);
919
bool continuingSubwordPrefix =
ContinuingSubwordPrefix
is not null &&
ContinuingSubwordPrefix
.Length > 0;
949
if (continuingSubwordPrefix && sSpan.StartsWith(
ContinuingSubwordPrefix
.AsSpan(), StringComparison.Ordinal))
951
sSpan = sSpan.Slice(
ContinuingSubwordPrefix
!.Length);
1039
if (
ContinuingSubwordPrefix
is not null && span.StartsWith(
ContinuingSubwordPrefix
.AsSpan(), StringComparison.Ordinal))
1041
span = span.Slice(
ContinuingSubwordPrefix
.Length);
1215
if (i > 0 &&
ContinuingSubwordPrefix
is not null)
1217
if (
ContinuingSubwordPrefix
.Length + s.Length <= buffer.Length)
1219
ContinuingSubwordPrefix
.AsSpan().CopyTo(buffer);
1220
s.CopyTo(buffer.Slice(
ContinuingSubwordPrefix
.Length));
1221
s = buffer.Slice(0,
ContinuingSubwordPrefix
.Length + s.Length);
1226
s = $"{
ContinuingSubwordPrefix
}{s}".AsSpan();