15 references to Cl100kBase
Microsoft.ML.Tokenizers (15)
Model\TiktokenTokenizer.cs (15)
1036( "gpt-4-", ModelEncoding.Cl100kBase), // e.g., gpt-4-0314, etc., plus gpt-4-32k
1037( "gpt-3.5-", ModelEncoding.Cl100kBase), // e.g, gpt-3.5-turbo-0301, -0401, etc.
1038( "gpt-35-", ModelEncoding.Cl100kBase ) // Azure deployment name
1048{ "gpt-4", ModelEncoding.Cl100kBase },
1049{ "gpt-3.5-turbo", ModelEncoding.Cl100kBase },
1050{ "gpt-3.5-turbo-16k", ModelEncoding.Cl100kBase },
1051{ "gpt-35", ModelEncoding.Cl100kBase }, // Azure deployment name
1052{ "gpt-35-turbo", ModelEncoding.Cl100kBase }, // Azure deployment name
1053{ "gpt-35-turbo-16k", ModelEncoding.Cl100kBase }, // Azure deployment name
1081{ "text-embedding-ada-002", ModelEncoding.Cl100kBase },
1082{ "text-embedding-3-small", ModelEncoding.Cl100kBase },
1083{ "text-embedding-3-large", ModelEncoding.Cl100kBase },
1101{ Phi4ModelName, ModelEncoding.Cl100kBase },
1132case ModelEncoding.Cl100kBase:
1468modelEncoding = ModelEncoding.Cl100kBase;