1 type derived from WordPieceTokenizer
Microsoft.ML.Tokenizers (1)
Model\BertTokenizer.cs (1)
23public sealed partial class BertTokenizer : WordPieceTokenizer
2 instantiations of WordPieceTokenizer
Microsoft.ML.Tokenizers (2)
Model\WordPieceTokenizer.cs (2)
192return new WordPieceTokenizer(vocab, vocabReverse, preTokenizer, normalizer, specialTokens, unknownToken, continuingSubwordPrefix, maxInputCharsPerWord); 284return new WordPieceTokenizer(vocab, vocabReverse, preTokenizer, normalizer, specialTokens, unknownToken, continuingSubwordPrefix, maxInputCharsPerWord);
24 references to WordPieceTokenizer
Microsoft.ML.Tokenizers (14)
Model\WordPieceTokenizer.cs (14)
127/// Create a new instance of the <see cref="WordPieceTokenizer"/> class. 136/// <returns>A new instance of the <see cref="WordPieceTokenizer"/> class.</returns> 140public static WordPieceTokenizer Create( 151/// Create a new instance of the <see cref="WordPieceTokenizer"/> class. 160/// <returns>A new instance of the <see cref="WordPieceTokenizer"/> class.</returns> 164public static WordPieceTokenizer Create( 173private static WordPieceTokenizer Create( 204/// Create a new instance of the <see cref="WordPieceTokenizer"/> class asynchronously. 214/// <returns>A new instance of the <see cref="WordPieceTokenizer"/> class.</returns> 218public static async Task<WordPieceTokenizer> CreateAsync( 239/// Create a new instance of the <see cref="WordPieceTokenizer"/> class asynchronously. 249/// <returns>A new instance of the <see cref="WordPieceTokenizer"/> class.</returns> 253public static async Task<WordPieceTokenizer> CreateAsync( 264private static async Task<WordPieceTokenizer> CreateAsync(
Microsoft.ML.Tokenizers.Tests (10)
WordPieceTests.cs (10)
33WordPieceTokenizer[] wordPieceTokenizers = [WordPieceTokenizer.Create(vocabFile), WordPieceTokenizer.Create(vocabStream)]; 35foreach (var tokenizer in wordPieceTokenizers) 58WordPieceTokenizer tokenizer = WordPieceTokenizer.Create(vocabFile); 152WordPieceTokenizer tokenizer = WordPieceTokenizer.Create(vocabFile); 188WordPieceTokenizer tokenizer = WordPieceTokenizer.Create(vocabFile, specialTokens: specialTokens);