1 type derived from WordPieceOptions
Microsoft.ML.Tokenizers (1)
Model\BertOptions.cs (1)
10public sealed class BertOptions : WordPieceOptions
2 instantiations of WordPieceOptions
Microsoft.ML.Tokenizers (1)
Model\WordPieceTokenizer.cs (1)
42options ??= new();
Microsoft.ML.Tokenizers.Tests (1)
WordPieceTests.cs (1)
188WordPieceTokenizer tokenizer = WordPieceTokenizer.Create(vocabFile, new WordPieceOptions { SpecialTokens = specialTokens });
11 references to WordPieceOptions
Microsoft.ML.Tokenizers (11)
Model\WordPieceTokenizer.cs (11)
35WordPieceOptions? options) 128/// If the <see cref="WordPieceOptions.PreTokenizer"/> is null, the whitespace pre-tokenizer will be used. 133WordPieceOptions? options = null) => 143/// If the <see cref="WordPieceOptions.PreTokenizer"/> is null, the whitespace pre-tokenizer will be used. 148WordPieceOptions? options = null) => 153WordPieceOptions? options, 184/// If the <see cref="WordPieceOptions.PreTokenizer"/> is null, the whitespace pre-tokenizer will be used. 189WordPieceOptions? options = null, 205/// If the <see cref="WordPieceOptions.PreTokenizer"/> is null, the whitespace pre-tokenizer will be used. 210WordPieceOptions? options = null, 216WordPieceOptions? options,