2 references to
Microsoft.ML.Tokenizers (2)
Model\EnglishRobertaTokenizer.cs (2)
119
this
(vocabularyPath is null ? throw new ArgumentNullException(nameof(vocabularyPath)) : File.OpenRead(vocabularyPath),
136
this
(vocabularyStream, mergeStream, highestOccurrenceMappingStream, preTokenizer, normalizer, filterUnsupportedChars, disposeStream: false)