4 instantiations of EnglishRobertaTokenizer
Microsoft.ML.Tokenizers (4)
Model\EnglishRobertaTokenizer.cs (4)
46=> new EnglishRobertaTokenizer(vocabularyPath, mergePath, highestOccurrenceMappingPath); 64=> new EnglishRobertaTokenizer(vocabularyPath, mergePath, highestOccurrenceMappingPath, preTokenizer, normalizer, filterUnsupportedChars); 76=> new EnglishRobertaTokenizer(vocabularyStream, mergeStream, highestOccurrenceMappingStream); 95=> new EnglishRobertaTokenizer(vocabularyStream, mergeStream, highestOccurrenceMappingStream, preTokenizer, normalizer, filterUnsupportedChars);
25 references to EnglishRobertaTokenizer
Microsoft.ML.Tokenizers (4)
Model\EnglishRobertaTokenizer.cs (4)
42public static EnglishRobertaTokenizer Create( 57public static EnglishRobertaTokenizer Create( 72public static EnglishRobertaTokenizer Create( 88public static EnglishRobertaTokenizer Create(
Microsoft.ML.Tokenizers.Tests (10)
EnglishRobertaTests.cs (10)
90_robertaTokenizer = EnglishRobertaTokenizer.Create( 113Tokenizer tokenizer = EnglishRobertaTokenizer.Create(vocabFile, mergeFile, translationFile, RobertaPreTokenizer.Instance); 118tokenizer = EnglishRobertaTokenizer.Create(vocabFile, mergeFile, translationFile, RobertaPreTokenizer.Instance, filterUnsupportedChars: false); 125tokenizer = EnglishRobertaTokenizer.Create(vocabStream, mergeStream, translationStream, RobertaPreTokenizer.Instance); 131tokenizer = EnglishRobertaTokenizer.Create(vocabFile, mergeFile, translationFile, RobertaPreTokenizer.Instance); 134tokenizer = EnglishRobertaTokenizer.Create(vocabFile, mergeFile, translationFile, RobertaPreTokenizer.Instance, filterUnsupportedChars: false); 238Assert.True(tokenizer is EnglishRobertaTokenizer); 275EnglishRobertaTokenizer? robertaModel = tokenizer as EnglishRobertaTokenizer; 308private void TestDecodingWithSpan(EnglishRobertaTokenizer tokenizer, int[] ids, string expectedDecoded)
Microsoft.ML.TorchSharp (11)
Extensions\TokenizerExtensions.cs (5)
29_instance = EnglishRobertaTokenizer.Create( 34(_instance as EnglishRobertaTokenizer).AddMaskSymbol(); 41internal static EnglishRobertaTokenizer RobertaModel(this Tokenizer tokenizer) 43EnglishRobertaTokenizer model = tokenizer as EnglishRobertaTokenizer;
NasBert\NasBertTrainer.cs (1)
204EnglishRobertaTokenizer tokenizerModel = Tokenizer.RobertaModel();
NasBert\NerTrainer.cs (1)
330EnglishRobertaTokenizer tokenizerModel = tokenizer.RobertaModel();
NasBert\SentenceSimilarityTrainer.cs (1)
242EnglishRobertaTokenizer tokenizerModel = tokenizer.RobertaModel();
NasBert\TextClassificationTrainer.cs (1)
266EnglishRobertaTokenizer tokenizerModel = tokenizer.RobertaModel();
Roberta\QATrainer.cs (2)
440EnglishRobertaTokenizer robertaModel = Tokenizer as EnglishRobertaTokenizer;