4 references to HasPretokenizationDelimiter
Microsoft.ML.Tokenizers (4)
SentencepieceModel.cs (4)
1692
if (
HasPretokenizationDelimiter
) hash ^= PretokenizationDelimiter.GetHashCode();
2081
if (
HasPretokenizationDelimiter
) {
2179
if (
HasPretokenizationDelimiter
) {
2324
if (other.
HasPretokenizationDelimiter
) {