1 type derived from WordPieceTokenizer
Microsoft.ML.Tokenizers (1)
Model\BertTokenizer.cs (1)
25
public sealed partial class BertTokenizer :
WordPieceTokenizer
2 instantiations of WordPieceTokenizer
Microsoft.ML.Tokenizers (2)
Model\WordPieceTokenizer.cs (2)
165
return new
WordPieceTokenizer
(vocab, vocabReverse, options);
229
return new
WordPieceTokenizer
(vocab, vocabReverse, options);
24 references to WordPieceTokenizer
Microsoft.ML.Tokenizers (14)
Model\WordPieceTokenizer.cs (14)
122
/// Create a new instance of the <see cref="
WordPieceTokenizer
"/> class.
126
/// <returns>A new instance of the <see cref="
WordPieceTokenizer
"/> class.</returns>
131
public static
WordPieceTokenizer
Create(
137
/// Create a new instance of the <see cref="
WordPieceTokenizer
"/> class.
141
/// <returns>A new instance of the <see cref="
WordPieceTokenizer
"/> class.</returns>
146
public static
WordPieceTokenizer
Create(
151
private static
WordPieceTokenizer
Create(
177
/// Create a new instance of the <see cref="
WordPieceTokenizer
"/> class asynchronously.
182
/// <returns>A new instance of the <see cref="
WordPieceTokenizer
"/> class.</returns>
187
public static async Task<
WordPieceTokenizer
> CreateAsync(
198
/// Create a new instance of the <see cref="
WordPieceTokenizer
"/> class asynchronously.
203
/// <returns>A new instance of the <see cref="
WordPieceTokenizer
"/> class.</returns>
208
public static async Task<
WordPieceTokenizer
> CreateAsync(
214
private static async Task<
WordPieceTokenizer
> CreateAsync(
Microsoft.ML.Tokenizers.Tests (10)
WordPieceTests.cs (10)
33
WordPieceTokenizer
[] wordPieceTokenizers = [
WordPieceTokenizer
.Create(vocabFile),
WordPieceTokenizer
.Create(vocabStream)];
35
foreach (
var
tokenizer in wordPieceTokenizers)
58
WordPieceTokenizer
tokenizer =
WordPieceTokenizer
.Create(vocabFile);
152
WordPieceTokenizer
tokenizer =
WordPieceTokenizer
.Create(vocabFile);
188
WordPieceTokenizer
tokenizer =
WordPieceTokenizer
.Create(vocabFile, new WordPieceOptions { SpecialTokens = specialTokens });