1040private static readonly (string Prefix, ModelEncoding Encoding)[] _modelPrefixToEncoding = 1042( "o1-", ModelEncoding.O200kBase ), // e.g. o1-mini 1043( "o3-", ModelEncoding.O200kBase ), // e.g. o3-mini 1044( "o4-mini-", ModelEncoding.O200kBase ), // e.g. o4-mini 1047( "gpt-6-", ModelEncoding.O200kBase ), 1048( "gpt-5.6-", ModelEncoding.O200kBase ), 1049( "gpt-5.5-", ModelEncoding.O200kBase ), 1050( "gpt-5.4-", ModelEncoding.O200kBase ), 1051( "gpt-5.3-", ModelEncoding.O200kBase ), 1052( "gpt-5.2-", ModelEncoding.O200kBase ), 1053( "gpt-5.1-", ModelEncoding.O200kBase ), 1054( "gpt-5-", ModelEncoding.O200kBase ), 1055( "gpt-4.1-", ModelEncoding.O200kBase ), // e.g., gpt-4.1-mini 1056( "gpt-4.5-", ModelEncoding.O200kBase ), // e.g., gpt-4.5 1057( "gpt-4o-", ModelEncoding.O200kBase ), // e.g., gpt-4o-2024-05-13 1058( "chatgpt-4o-", ModelEncoding.O200kBase ), 1059( "gpt-4-", ModelEncoding.Cl100kBase ), // e.g., gpt-4-0314, etc., plus gpt-4-32k 1060( "gpt-3.5-", ModelEncoding.Cl100kBase ), // e.g, gpt-3.5-turbo-0301, -0401, etc. 1061( "gpt-35-", ModelEncoding.Cl100kBase ), // Azure deployment name 1062( "gpt-oss-", ModelEncoding.O200kHarmony ), 1065( "ft:gpt-4o", ModelEncoding.O200kBase ), 1066( "ft:gpt-4", ModelEncoding.Cl100kBase ), 1067( "ft:gpt-3.5-turbo", ModelEncoding.Cl100kBase ), 1068( "ft:davinci-002", ModelEncoding.Cl100kBase ), 1069( "ft:babbage-002", ModelEncoding.Cl100kBase ), 1072private static readonly Dictionary<string, ModelEncoding> _modelToEncoding = 1073new Dictionary<string, ModelEncoding>(StringComparer.OrdinalIgnoreCase) 1076{ "o1", ModelEncoding.O200kBase }, 1077{ "o3", ModelEncoding.O200kBase }, 1078{ "o4-mini", ModelEncoding.O200kBase }, 1081{ "gpt-6", ModelEncoding.O200kBase }, 1082{ "gpt-5.6", ModelEncoding.O200kBase }, 1083{ "gpt-5.5", ModelEncoding.O200kBase }, 1084{ "gpt-5.4", ModelEncoding.O200kBase }, 1085{ "gpt-5.3", ModelEncoding.O200kBase }, 1086{ "gpt-5.2", ModelEncoding.O200kBase }, 1087{ "gpt-5.1", ModelEncoding.O200kBase }, 1088{ "gpt-5", ModelEncoding.O200kBase }, 1089{ "gpt-4.1", ModelEncoding.O200kBase }, 1090{ "gpt-4o", ModelEncoding.O200kBase }, 1091{ "gpt-4", ModelEncoding.Cl100kBase }, 1092{ "gpt-3.5-turbo", ModelEncoding.Cl100kBase }, 1093{ "gpt-3.5", ModelEncoding.Cl100kBase }, 1094{ "gpt-3.5-turbo-16k", ModelEncoding.Cl100kBase }, 1095{ "gpt-35", ModelEncoding.Cl100kBase }, // Azure deployment name 1096{ "gpt-35-turbo", ModelEncoding.Cl100kBase }, // Azure deployment name 1097{ "gpt-35-turbo-16k", ModelEncoding.Cl100kBase }, // Azure deployment name 1100{ "davinci-002", ModelEncoding.Cl100kBase }, 1101{ "babbage-002", ModelEncoding.Cl100kBase }, 1105{ "text-embedding-ada-002", ModelEncoding.Cl100kBase }, 1106{ "text-embedding-3-small", ModelEncoding.Cl100kBase }, 1107{ "text-embedding-3-large", ModelEncoding.Cl100kBase }, 1111{ "text-davinci-003", ModelEncoding.P50kBase }, 1112{ "text-davinci-002", ModelEncoding.P50kBase }, 1113{ "text-davinci-001", ModelEncoding.R50kBase }, 1114{ "text-curie-001", ModelEncoding.R50kBase }, 1115{ "text-babbage-001", ModelEncoding.R50kBase }, 1116{ "text-ada-001", ModelEncoding.R50kBase }, 1117{ "davinci", ModelEncoding.R50kBase }, 1118{ "curie", ModelEncoding.R50kBase }, 1119{ "babbage", ModelEncoding.R50kBase }, 1120{ "ada", ModelEncoding.R50kBase }, 1123{ "code-davinci-002", ModelEncoding.P50kBase }, 1124{ "code-davinci-001", ModelEncoding.P50kBase }, 1125{ "code-cushman-002", ModelEncoding.P50kBase }, 1126{ "code-cushman-001", ModelEncoding.P50kBase }, 1127{ "davinci-codex", ModelEncoding.P50kBase }, 1128{ "cushman-codex", ModelEncoding.P50kBase }, 1131{ "text-davinci-edit-001", ModelEncoding.P50kEdit }, 1132{ "code-davinci-edit-001", ModelEncoding.P50kEdit }, 1136{ "text-similarity-davinci-001", ModelEncoding.R50kBase }, 1137{ "text-similarity-curie-001", ModelEncoding.R50kBase }, 1138{ "text-similarity-babbage-001", ModelEncoding.R50kBase }, 1139{ "text-similarity-ada-001", ModelEncoding.R50kBase }, 1140{ "text-search-davinci-doc-001", ModelEncoding.R50kBase }, 1141{ "text-search-curie-doc-001", ModelEncoding.R50kBase }, 1142{ "text-search-babbage-doc-001", ModelEncoding.R50kBase }, 1143{ "text-search-ada-doc-001", ModelEncoding.R50kBase }, 1144{ "code-search-babbage-code-001", ModelEncoding.R50kBase }, 1145{ "code-search-ada-code-001", ModelEncoding.R50kBase }, 1148{ "gpt2", ModelEncoding.GPT2 }, 1149{ "gpt-2", ModelEncoding.GPT2 }, 1152{ Phi4ModelName, ModelEncoding.Cl100kBase }, 1155private static ModelEncoding GetModelEncoding(string modelName) 1157if (!_modelToEncoding.TryGetValue(modelName, out ModelEncoding encoder)) 1159foreach ((string Prefix, ModelEncoding Encoding) in _modelPrefixToEncoding) 1169if (encoder == ModelEncoding.None) 1205private static (Dictionary<string, int> SpecialTokens, Regex Regex, string VocabFile, Type? DataType, string PackageName) GetTiktokenConfigurations(ModelEncoding modelEncoding, string? modelName = null) 1209case ModelEncoding.Cl100kBase: 1218case ModelEncoding.GPT2: 1221case ModelEncoding.O200kBase: 1224case ModelEncoding.P50kBase: 1227case ModelEncoding.P50kEdit: 1231case ModelEncoding.R50kBase: 1234case ModelEncoding.O200kHarmony: 1300ModelEncoding modelEncoding, 1546ModelEncoding modelEncoding; 1549modelEncoding = ModelEncoding.Cl100kBase; 1553modelEncoding = ModelEncoding.O200kBase; 1557modelEncoding = ModelEncoding.O200kHarmony; 1561modelEncoding = ModelEncoding.P50kBase; 1565modelEncoding = ModelEncoding.P50kEdit; 1569modelEncoding = ModelEncoding.R50kBase;