2 references to P50kBaseEncodingName
Microsoft.ML.Tokenizers (2)
Model\TiktokenTokenizer.cs (2)
1459
else if (encodingName.Equals(
P50kBaseEncodingName
, StringComparison.OrdinalIgnoreCase))
1473
throw new ArgumentException($"The encoding name '{encodingName}' is not supported. The only supported encoding names are: {TiktokenTokenizer.Cl100kBaseEncodingName}, {TiktokenTokenizer.
P50kBaseEncodingName
}, {TiktokenTokenizer.P50kEditEncodingName}, and {TiktokenTokenizer.R50kBaseEncodingName}.", nameof(encodingName));