1 instantiation of RobertaPreTokenizer
Microsoft.ML.Tokenizers (1)
PreTokenizer\RobertaPreTokenizer.cs (1)
18
public static RobertaPreTokenizer Instance { get; } = new
RobertaPreTokenizer
();
9 references to RobertaPreTokenizer
Microsoft.ML.Tokenizers (1)
PreTokenizer\RobertaPreTokenizer.cs (1)
18
public static
RobertaPreTokenizer
Instance { get; } = new RobertaPreTokenizer();
Microsoft.ML.Tokenizers.Tests (7)
EnglishRobertaTests.cs (7)
94
RobertaPreTokenizer
.Instance);
113
Tokenizer tokenizer = EnglishRobertaTokenizer.Create(vocabFile, mergeFile, translationFile,
RobertaPreTokenizer
.Instance);
118
tokenizer = EnglishRobertaTokenizer.Create(vocabFile, mergeFile, translationFile,
RobertaPreTokenizer
.Instance, filterUnsupportedChars: false);
125
tokenizer = EnglishRobertaTokenizer.Create(vocabStream, mergeStream, translationStream,
RobertaPreTokenizer
.Instance);
131
tokenizer = EnglishRobertaTokenizer.Create(vocabFile, mergeFile, translationFile,
RobertaPreTokenizer
.Instance);
134
tokenizer = EnglishRobertaTokenizer.Create(vocabFile, mergeFile, translationFile,
RobertaPreTokenizer
.Instance, filterUnsupportedChars: false);
239
Assert.True(tokenizer.PreTokenizer is
RobertaPreTokenizer
);
Microsoft.ML.TorchSharp (1)
Extensions\TokenizerExtensions.cs (1)
33
RobertaPreTokenizer
.Instance);