7 references to CreateAsync
Microsoft.ML.Tokenizers (1)
Model\BertTokenizer.cs (1)
726
return await
CreateAsync
(stream, options, cancellationToken).ConfigureAwait(false);
Microsoft.ML.Tokenizers.Tests (6)
BertTokenizerTests.cs (6)
235
BertTokenizer bertTokenizer = await BertTokenizer.
CreateAsync
(vocabStream); // lowercasing and no accent stripping
252
bertTokenizer = await BertTokenizer.
CreateAsync
(vocabStream, new BertOptions { LowerCaseBeforeTokenization = false }); // no lowercasing and no accent stripping
267
bertTokenizer = await BertTokenizer.
CreateAsync
(vocabStream, new BertOptions { RemoveNonSpacingMarks = true }); // lowercasing and accent stripping
281
bertTokenizer = await BertTokenizer.
CreateAsync
(vocabStream, new BertOptions { LowerCaseBeforeTokenization = false, RemoveNonSpacingMarks = true }); // no lowercasing and accent stripping
310
BertTokenizer bertTokenizer = await BertTokenizer.
CreateAsync
(vocabStream); // tokenize Chinese characters
329
bertTokenizer = await BertTokenizer.
CreateAsync
(vocabStream, new BertOptions { IndividuallyTokenizeCjk = false }); // do not tokenize Chinese characters