22 references to Cl100kBase
Microsoft.ML.Tokenizers (22)
Model\TiktokenTokenizer.cs (22)
1058( "gpt-4-", ModelEncoding.Cl100kBase ), // e.g., gpt-4-0314, etc., plus gpt-4-32k
1059( "gpt-3.5-", ModelEncoding.Cl100kBase ), // e.g, gpt-3.5-turbo-0301, -0401, etc.
1060( "gpt-35-", ModelEncoding.Cl100kBase ), // Azure deployment name
1065( "ft:gpt-4", ModelEncoding.Cl100kBase ),
1066( "ft:gpt-3.5-turbo", ModelEncoding.Cl100kBase ),
1067( "ft:davinci-002", ModelEncoding.Cl100kBase ),
1068( "ft:babbage-002", ModelEncoding.Cl100kBase ),
1089{ "gpt-4", ModelEncoding.Cl100kBase },
1090{ "gpt-3.5-turbo", ModelEncoding.Cl100kBase },
1091{ "gpt-3.5", ModelEncoding.Cl100kBase },
1092{ "gpt-3.5-turbo-16k", ModelEncoding.Cl100kBase },
1093{ "gpt-35", ModelEncoding.Cl100kBase }, // Azure deployment name
1094{ "gpt-35-turbo", ModelEncoding.Cl100kBase }, // Azure deployment name
1095{ "gpt-35-turbo-16k", ModelEncoding.Cl100kBase }, // Azure deployment name
1098{ "davinci-002", ModelEncoding.Cl100kBase },
1099{ "babbage-002", ModelEncoding.Cl100kBase },
1103{ "text-embedding-ada-002", ModelEncoding.Cl100kBase },
1104{ "text-embedding-3-small", ModelEncoding.Cl100kBase },
1105{ "text-embedding-3-large", ModelEncoding.Cl100kBase },
1150{ Phi4ModelName, ModelEncoding.Cl100kBase },
1207case ModelEncoding.Cl100kBase:
1547modelEncoding = ModelEncoding.Cl100kBase;