23 references to O200kBase
Microsoft.ML.Tokenizers (23)
Model\TiktokenTokenizer.cs (23)
1042( "o1-", ModelEncoding.O200kBase ), // e.g. o1-mini
1043( "o3-", ModelEncoding.O200kBase ), // e.g. o3-mini
1044( "o4-mini-", ModelEncoding.O200kBase ), // e.g. o4-mini
1047( "gpt-5.3-", ModelEncoding.O200kBase ),
1048( "gpt-5.2-", ModelEncoding.O200kBase ),
1049( "gpt-5.1-", ModelEncoding.O200kBase ),
1050( "gpt-5-", ModelEncoding.O200kBase ),
1051( "gpt-4.1-", ModelEncoding.O200kBase ), // e.g., gpt-4.1-mini
1052( "gpt-4.5-", ModelEncoding.O200kBase ), // e.g., gpt-4.5
1053( "gpt-4o-", ModelEncoding.O200kBase ), // e.g., gpt-4o-2024-05-13
1054( "chatgpt-4o-", ModelEncoding.O200kBase ),
1061( "ft:gpt-4o", ModelEncoding.O200kBase ),
1072{ "o1", ModelEncoding.O200kBase },
1073{ "o3", ModelEncoding.O200kBase },
1074{ "o4-mini", ModelEncoding.O200kBase },
1077{ "gpt-5.3", ModelEncoding.O200kBase },
1078{ "gpt-5.2", ModelEncoding.O200kBase },
1079{ "gpt-5.1", ModelEncoding.O200kBase },
1080{ "gpt-5", ModelEncoding.O200kBase },
1081{ "gpt-4.1", ModelEncoding.O200kBase },
1082{ "gpt-4o", ModelEncoding.O200kBase },
1213case ModelEncoding.O200kBase:
1545modelEncoding = ModelEncoding.O200kBase;