4 instantiations of EnglishRobertaTokenizer
Microsoft.ML.Tokenizers (4)
Model\EnglishRobertaTokenizer.cs (4)
49
=> new
EnglishRobertaTokenizer
(vocabularyPath, mergePath, highestOccurrenceMappingPath);
70
=> new
EnglishRobertaTokenizer
(vocabularyPath, mergePath, highestOccurrenceMappingPath, preTokenizer, normalizer, filterUnsupportedChars);
85
=> new
EnglishRobertaTokenizer
(vocabularyStream, mergeStream, highestOccurrenceMappingStream);
107
=> new
EnglishRobertaTokenizer
(vocabularyStream, mergeStream, highestOccurrenceMappingStream, preTokenizer, normalizer, filterUnsupportedChars);
25 references to EnglishRobertaTokenizer
Microsoft.ML.Tokenizers (4)
Model\EnglishRobertaTokenizer.cs (4)
45
public static
EnglishRobertaTokenizer
Create(
63
public static
EnglishRobertaTokenizer
Create(
81
public static
EnglishRobertaTokenizer
Create(
100
public static
EnglishRobertaTokenizer
Create(
Microsoft.ML.Tokenizers.Tests (10)
EnglishRobertaTests.cs (10)
90
_robertaTokenizer =
EnglishRobertaTokenizer
.Create(
113
Tokenizer tokenizer =
EnglishRobertaTokenizer
.Create(vocabFile, mergeFile, translationFile, RobertaPreTokenizer.Instance);
118
tokenizer =
EnglishRobertaTokenizer
.Create(vocabFile, mergeFile, translationFile, RobertaPreTokenizer.Instance, filterUnsupportedChars: false);
125
tokenizer =
EnglishRobertaTokenizer
.Create(vocabStream, mergeStream, translationStream, RobertaPreTokenizer.Instance);
131
tokenizer =
EnglishRobertaTokenizer
.Create(vocabFile, mergeFile, translationFile, RobertaPreTokenizer.Instance);
134
tokenizer =
EnglishRobertaTokenizer
.Create(vocabFile, mergeFile, translationFile, RobertaPreTokenizer.Instance, filterUnsupportedChars: false);
238
Assert.True(tokenizer is
EnglishRobertaTokenizer
);
275
EnglishRobertaTokenizer
? robertaModel = tokenizer as
EnglishRobertaTokenizer
;
308
private void TestDecodingWithSpan(
EnglishRobertaTokenizer
tokenizer, int[] ids, string expectedDecoded)
Microsoft.ML.TorchSharp (11)
Extensions\TokenizerExtensions.cs (5)
29
_instance =
EnglishRobertaTokenizer
.Create(
34
(_instance as
EnglishRobertaTokenizer
).AddMaskSymbol();
41
internal static
EnglishRobertaTokenizer
RobertaModel(this Tokenizer tokenizer)
43
EnglishRobertaTokenizer
model = tokenizer as
EnglishRobertaTokenizer
;
NasBert\NasBertTrainer.cs (1)
204
EnglishRobertaTokenizer
tokenizerModel = Tokenizer.RobertaModel();
NasBert\NerTrainer.cs (1)
330
EnglishRobertaTokenizer
tokenizerModel = tokenizer.RobertaModel();
NasBert\SentenceSimilarityTrainer.cs (1)
243
EnglishRobertaTokenizer
tokenizerModel = tokenizer.RobertaModel();
NasBert\TextClassificationTrainer.cs (1)
267
EnglishRobertaTokenizer
tokenizerModel = tokenizer.RobertaModel();
Roberta\QATrainer.cs (2)
440
EnglishRobertaTokenizer
robertaModel = Tokenizer as
EnglishRobertaTokenizer
;