1 write to ContinuingSubwordPrefix
Microsoft.ML.Tokenizers (1)
Model\WordPieceTokenizer.cs (1)
69ContinuingSubwordPrefix = options.ContinuingSubwordPrefix;
16 references to ContinuingSubwordPrefix
Microsoft.ML.Tokenizers (15)
Model\WordPieceTokenizer.cs (15)
323int maxLength = MaxInputCharsPerWord + ContinuingSubwordPrefix.Length; 326ContinuingSubwordPrefix.AsSpan().CopyTo(buffer); 345subStr.CopyTo(buffer.Slice(ContinuingSubwordPrefix.Length)); 346subStr = buffer.Slice(0, ContinuingSubwordPrefix.Length + subStr.Length); 463int maxLength = MaxInputCharsPerWord + ContinuingSubwordPrefix.Length; 466ContinuingSubwordPrefix.AsSpan().CopyTo(buffer); 486subStr.CopyTo(buffer.Slice(ContinuingSubwordPrefix.Length)); 487subStr = buffer.Slice(0, ContinuingSubwordPrefix.Length + subStr.Length); 694if (token.StartsWith(ContinuingSubwordPrefix)) 696sb.Append(token.AsSpan().Slice(ContinuingSubwordPrefix.Length)); 753if (token.StartsWith(ContinuingSubwordPrefix, StringComparison.Ordinal)) 755if (token.Length - ContinuingSubwordPrefix.Length > buffer.Length) 759token.AsSpan().Slice(ContinuingSubwordPrefix.Length).CopyTo(buffer); 760buffer = buffer.Slice(token.Length - ContinuingSubwordPrefix.Length); 761charsWritten += token.Length - ContinuingSubwordPrefix.Length;
Microsoft.ML.Tokenizers.Tests (1)
WordPieceTests.cs (1)
42Assert.Equal("##", tokenizer.ContinuingSubwordPrefix);