1 instantiation of RobertaPreTokenizer
Microsoft.ML.Tokenizers (1)
PreTokenizer\RobertaPreTokenizer.cs (1)
18public static RobertaPreTokenizer Instance { get; } = new RobertaPreTokenizer();
9 references to RobertaPreTokenizer
Microsoft.ML.Tokenizers (1)
PreTokenizer\RobertaPreTokenizer.cs (1)
18public static RobertaPreTokenizer Instance { get; } = new RobertaPreTokenizer();
Microsoft.ML.Tokenizers.Tests (7)
EnglishRobertaTests.cs (7)
94RobertaPreTokenizer.Instance); 113Tokenizer tokenizer = EnglishRobertaTokenizer.Create(vocabFile, mergeFile, translationFile, RobertaPreTokenizer.Instance); 118tokenizer = EnglishRobertaTokenizer.Create(vocabFile, mergeFile, translationFile, RobertaPreTokenizer.Instance, filterUnsupportedChars: false); 125tokenizer = EnglishRobertaTokenizer.Create(vocabStream, mergeStream, translationStream, RobertaPreTokenizer.Instance); 131tokenizer = EnglishRobertaTokenizer.Create(vocabFile, mergeFile, translationFile, RobertaPreTokenizer.Instance); 134tokenizer = EnglishRobertaTokenizer.Create(vocabFile, mergeFile, translationFile, RobertaPreTokenizer.Instance, filterUnsupportedChars: false); 239Assert.True(tokenizer.PreTokenizer is RobertaPreTokenizer);
Microsoft.ML.TorchSharp (1)
Extensions\TokenizerExtensions.cs (1)
33RobertaPreTokenizer.Instance);