1 type derived from WordPieceTokenizer
Microsoft.ML.Tokenizers (1)
Model\BertTokenizer.cs (1)
25public sealed partial class BertTokenizer : WordPieceTokenizer
2 instantiations of WordPieceTokenizer
Microsoft.ML.Tokenizers (2)
Model\WordPieceTokenizer.cs (2)
165return new WordPieceTokenizer(vocab, vocabReverse, options); 229return new WordPieceTokenizer(vocab, vocabReverse, options);
24 references to WordPieceTokenizer
Microsoft.ML.Tokenizers (14)
Model\WordPieceTokenizer.cs (14)
122/// Create a new instance of the <see cref="WordPieceTokenizer"/> class. 126/// <returns>A new instance of the <see cref="WordPieceTokenizer"/> class.</returns> 131public static WordPieceTokenizer Create( 137/// Create a new instance of the <see cref="WordPieceTokenizer"/> class. 141/// <returns>A new instance of the <see cref="WordPieceTokenizer"/> class.</returns> 146public static WordPieceTokenizer Create( 151private static WordPieceTokenizer Create( 177/// Create a new instance of the <see cref="WordPieceTokenizer"/> class asynchronously. 182/// <returns>A new instance of the <see cref="WordPieceTokenizer"/> class.</returns> 187public static async Task<WordPieceTokenizer> CreateAsync( 198/// Create a new instance of the <see cref="WordPieceTokenizer"/> class asynchronously. 203/// <returns>A new instance of the <see cref="WordPieceTokenizer"/> class.</returns> 208public static async Task<WordPieceTokenizer> CreateAsync( 214private static async Task<WordPieceTokenizer> CreateAsync(
Microsoft.ML.Tokenizers.Tests (10)
WordPieceTests.cs (10)
33WordPieceTokenizer[] wordPieceTokenizers = [WordPieceTokenizer.Create(vocabFile), WordPieceTokenizer.Create(vocabStream)]; 35foreach (var tokenizer in wordPieceTokenizers) 58WordPieceTokenizer tokenizer = WordPieceTokenizer.Create(vocabFile); 152WordPieceTokenizer tokenizer = WordPieceTokenizer.Create(vocabFile); 188WordPieceTokenizer tokenizer = WordPieceTokenizer.Create(vocabFile, new WordPieceOptions { SpecialTokens = specialTokens });