3 references to ContinuingSubwordPrefix
Microsoft.ML.Tokenizers (3)
Model\WordPieceTokenizer.cs (3)
52
if (options.
ContinuingSubwordPrefix
is null)
54
throw new ArgumentNullException(nameof(options.
ContinuingSubwordPrefix
));
69
ContinuingSubwordPrefix = options.
ContinuingSubwordPrefix
;