1 write to ContinuingSubwordPrefix
Microsoft.ML.Tokenizers (1)
Model\WordPieceTokenizer.cs (1)
74ContinuingSubwordPrefix = continuingSubwordPrefix;
16 references to ContinuingSubwordPrefix
Microsoft.ML.Tokenizers (15)
Model\WordPieceTokenizer.cs (15)
378int maxLength = MaxInputCharsPerWord + ContinuingSubwordPrefix.Length; 381ContinuingSubwordPrefix.AsSpan().CopyTo(buffer); 400subStr.CopyTo(buffer.Slice(ContinuingSubwordPrefix.Length)); 401subStr = buffer.Slice(0, ContinuingSubwordPrefix.Length + subStr.Length); 518int maxLength = MaxInputCharsPerWord + ContinuingSubwordPrefix.Length; 521ContinuingSubwordPrefix.AsSpan().CopyTo(buffer); 541subStr.CopyTo(buffer.Slice(ContinuingSubwordPrefix.Length)); 542subStr = buffer.Slice(0, ContinuingSubwordPrefix.Length + subStr.Length); 749if (token.StartsWith(ContinuingSubwordPrefix)) 751sb.Append(token.AsSpan().Slice(ContinuingSubwordPrefix.Length)); 808if (token.StartsWith(ContinuingSubwordPrefix, StringComparison.Ordinal)) 810if (token.Length - ContinuingSubwordPrefix.Length > buffer.Length) 814token.AsSpan().Slice(ContinuingSubwordPrefix.Length).CopyTo(buffer); 815buffer = buffer.Slice(token.Length - ContinuingSubwordPrefix.Length); 816charsWritten += token.Length - ContinuingSubwordPrefix.Length;
Microsoft.ML.Tokenizers.Tests (1)
WordPieceTests.cs (1)
42Assert.Equal("##", tokenizer.ContinuingSubwordPrefix);