4 references to Create
Microsoft.ML.Tokenizers.Tests (4)
WordPieceTests.cs (4)
33
WordPieceTokenizer[] wordPieceTokenizers = [WordPieceTokenizer.
Create
(vocabFile), WordPieceTokenizer.Create(vocabStream)];
58
WordPieceTokenizer tokenizer = WordPieceTokenizer.
Create
(vocabFile);
152
WordPieceTokenizer tokenizer = WordPieceTokenizer.
Create
(vocabFile);
188
WordPieceTokenizer tokenizer = WordPieceTokenizer.
Create
(vocabFile, specialTokens: specialTokens);