2 references to P50kEditEncodingName
Microsoft.ML.Tokenizers (2)
Model\TiktokenTokenizer.cs (2)
1435
else if (encodingName.Equals(
P50kEditEncodingName
, StringComparison.OrdinalIgnoreCase))
1445
throw new ArgumentException($"The encoding name '{encodingName}' is not supported. The only supported encoding names are: {TiktokenTokenizer.Cl100kBaseEncodingName}, {TiktokenTokenizer.P50kBaseEncodingName}, {TiktokenTokenizer.
P50kEditEncodingName
}, and {TiktokenTokenizer.R50kBaseEncodingName}.", nameof(encodingName));