8 instantiations of BertOptions
Microsoft.ML.Tokenizers (1)
Model\BertTokenizer.cs (1)
761options ??= new();
Microsoft.ML.Tokenizers.Tests (7)
BertTokenizerTests.cs (7)
34var bertOptions = new BertOptions() 184BertTokenizer[] bertTokenizers = [BertTokenizer.Create(vocabFile, new BertOptions { LowerCaseBeforeTokenization = false }), 185BertTokenizer.Create(vocabStream, new BertOptions { LowerCaseBeforeTokenization = false })]; 252bertTokenizer = await BertTokenizer.CreateAsync(vocabStream, new BertOptions { LowerCaseBeforeTokenization = false }); // no lowercasing and no accent stripping 267bertTokenizer = await BertTokenizer.CreateAsync(vocabStream, new BertOptions { RemoveNonSpacingMarks = true }); // lowercasing and accent stripping 281bertTokenizer = await BertTokenizer.CreateAsync(vocabStream, new BertOptions { LowerCaseBeforeTokenization = false, RemoveNonSpacingMarks = true }); // no lowercasing and accent stripping 329bertTokenizer = await BertTokenizer.CreateAsync(vocabStream, new BertOptions { IndividuallyTokenizeCjk = false }); // do not tokenize Chinese characters
8 references to BertOptions
Microsoft.ML.Tokenizers (7)
Model\BertTokenizer.cs (7)
30BertOptions options) : base(vocab, vocabReverse, options) 663BertOptions? options = null) => 679BertOptions? options = null) => 694BertOptions? options = null, 719BertOptions? options = null, 734private static BertTokenizer Create(Stream vocabStream, BertOptions? options, bool disposeStream) 759BertOptions? options)
Microsoft.ML.Tokenizers.Tests (1)
BertTokenizerTests.cs (1)
34var bertOptions = new BertOptions()