2 references to Cl100kBaseEncodingName
Microsoft.ML.Tokenizers (2)
Model\TiktokenTokenizer.cs (2)
1451
if (encodingName.Equals(
Cl100kBaseEncodingName
, StringComparison.OrdinalIgnoreCase))
1473
throw new ArgumentException($"The encoding name '{encodingName}' is not supported. The only supported encoding names are: {TiktokenTokenizer.
Cl100kBaseEncodingName
}, {TiktokenTokenizer.P50kBaseEncodingName}, {TiktokenTokenizer.P50kEditEncodingName}, and {TiktokenTokenizer.R50kBaseEncodingName}.", nameof(encodingName));