31 references to O200kBase
Microsoft.ML.Tokenizers (31)
Model\TiktokenTokenizer.cs (31)
1042( "o1-", ModelEncoding.O200kBase ), // e.g. o1-mini 1043( "o3-", ModelEncoding.O200kBase ), // e.g. o3-mini 1044( "o4-mini-", ModelEncoding.O200kBase ), // e.g. o4-mini 1047( "gpt-6-", ModelEncoding.O200kBase ), 1048( "gpt-5.6-", ModelEncoding.O200kBase ), 1049( "gpt-5.5-", ModelEncoding.O200kBase ), 1050( "gpt-5.4-", ModelEncoding.O200kBase ), 1051( "gpt-5.3-", ModelEncoding.O200kBase ), 1052( "gpt-5.2-", ModelEncoding.O200kBase ), 1053( "gpt-5.1-", ModelEncoding.O200kBase ), 1054( "gpt-5-", ModelEncoding.O200kBase ), 1055( "gpt-4.1-", ModelEncoding.O200kBase ), // e.g., gpt-4.1-mini 1056( "gpt-4.5-", ModelEncoding.O200kBase ), // e.g., gpt-4.5 1057( "gpt-4o-", ModelEncoding.O200kBase ), // e.g., gpt-4o-2024-05-13 1058( "chatgpt-4o-", ModelEncoding.O200kBase ), 1065( "ft:gpt-4o", ModelEncoding.O200kBase ), 1076{ "o1", ModelEncoding.O200kBase }, 1077{ "o3", ModelEncoding.O200kBase }, 1078{ "o4-mini", ModelEncoding.O200kBase }, 1081{ "gpt-6", ModelEncoding.O200kBase }, 1082{ "gpt-5.6", ModelEncoding.O200kBase }, 1083{ "gpt-5.5", ModelEncoding.O200kBase }, 1084{ "gpt-5.4", ModelEncoding.O200kBase }, 1085{ "gpt-5.3", ModelEncoding.O200kBase }, 1086{ "gpt-5.2", ModelEncoding.O200kBase }, 1087{ "gpt-5.1", ModelEncoding.O200kBase }, 1088{ "gpt-5", ModelEncoding.O200kBase }, 1089{ "gpt-4.1", ModelEncoding.O200kBase }, 1090{ "gpt-4o", ModelEncoding.O200kBase }, 1221case ModelEncoding.O200kBase: 1553modelEncoding = ModelEncoding.O200kBase;