2 references to
Microsoft.ML.Tokenizers (2)
Model\EnglishRobertaTokenizer.cs (2)
107
this
(vocabularyPath is null ? throw new ArgumentNullException(nameof(vocabularyPath)) : File.OpenRead(vocabularyPath),
124
this
(vocabularyStream, mergeStream, highestOccurrenceMappingStream, preTokenizer, normalizer, filterUnsupportedChars, disposeStream: false)