3 references to WordPieceTokenizer
Microsoft.ML.Tokenizers (3)
Model\BertTokenizer.cs (1)
40
bool stripAccents) :
base
(vocab, vocabReverse, preTokenizer, normalizer, specialTokens, unknownToken)
Model\WordPieceTokenizer.cs (2)
192
return new
WordPieceTokenizer
(vocab, vocabReverse, preTokenizer, normalizer, specialTokens, unknownToken, continuingSubwordPrefix, maxInputCharsPerWord);
284
return new
WordPieceTokenizer
(vocab, vocabReverse, preTokenizer, normalizer, specialTokens, unknownToken, continuingSubwordPrefix, maxInputCharsPerWord);