2 references to R50kBaseEncodingName
Microsoft.ML.Tokenizers (2)
Model\TiktokenTokenizer.cs (2)
1467
else if (encodingName.Equals(
R50kBaseEncodingName
, StringComparison.OrdinalIgnoreCase))
1473
throw new ArgumentException($"The encoding name '{encodingName}' is not supported. The only supported encoding names are: {TiktokenTokenizer.Cl100kBaseEncodingName}, {TiktokenTokenizer.P50kBaseEncodingName}, {TiktokenTokenizer.P50kEditEncodingName}, and {TiktokenTokenizer.
R50kBaseEncodingName
}.", nameof(encodingName));