mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-09 03:18:44 +00:00
[Performance] Use O(1) Set lookups for model routing (#13879)
* o(1) lookups * Revert "o(1) lookups" This reverts commit620d142469. * o(1) lookups * Revert "o(1) lookups" This reverts commit676a9f5bcc. * o(1) lookups * register_model fix * test_aget_valid_models * lambda ai models fix * test_utils.py * test fix vertex ai
This commit is contained in:
Ishaan Jaff
GitHub
parent
bfd5ad032e
commit
e93e266f84
6 changed files with 290 additions and 294 deletions
|
|
@ -467,79 +467,80 @@ BEDROCK_CONVERSE_MODELS = [
|
|||
]
|
||||
|
||||
####### COMPLETION MODELS ###################
|
||||
open_ai_chat_completion_models: List = []
|
||||
open_ai_text_completion_models: List = []
|
||||
cohere_models: List = []
|
||||
cohere_chat_models: List = []
|
||||
mistral_chat_models: List = []
|
||||
text_completion_codestral_models: List = []
|
||||
anthropic_models: List = []
|
||||
openrouter_models: List = []
|
||||
datarobot_models: List = []
|
||||
vertex_language_models: List = []
|
||||
vertex_vision_models: List = []
|
||||
vertex_chat_models: List = []
|
||||
vertex_code_chat_models: List = []
|
||||
vertex_ai_image_models: List = []
|
||||
vertex_text_models: List = []
|
||||
vertex_code_text_models: List = []
|
||||
vertex_embedding_models: List = []
|
||||
vertex_anthropic_models: List = []
|
||||
vertex_llama3_models: List = []
|
||||
vertex_deepseek_models: List = []
|
||||
vertex_ai_ai21_models: List = []
|
||||
vertex_mistral_models: List = []
|
||||
ai21_models: List = []
|
||||
ai21_chat_models: List = []
|
||||
nlp_cloud_models: List = []
|
||||
aleph_alpha_models: List = []
|
||||
bedrock_models: List = []
|
||||
bedrock_converse_models: List = BEDROCK_CONVERSE_MODELS
|
||||
fireworks_ai_models: List = []
|
||||
fireworks_ai_embedding_models: List = []
|
||||
deepinfra_models: List = []
|
||||
perplexity_models: List = []
|
||||
watsonx_models: List = []
|
||||
gemini_models: List = []
|
||||
xai_models: List = []
|
||||
deepseek_models: List = []
|
||||
azure_ai_models: List = []
|
||||
jina_ai_models: List = []
|
||||
voyage_models: List = []
|
||||
infinity_models: List = []
|
||||
databricks_models: List = []
|
||||
cloudflare_models: List = []
|
||||
codestral_models: List = []
|
||||
friendliai_models: List = []
|
||||
featherless_ai_models: List = []
|
||||
palm_models: List = []
|
||||
groq_models: List = []
|
||||
azure_models: List = []
|
||||
azure_text_models: List = []
|
||||
anyscale_models: List = []
|
||||
cerebras_models: List = []
|
||||
galadriel_models: List = []
|
||||
sambanova_models: List = []
|
||||
sambanova_embedding_models: List = []
|
||||
novita_models: List = []
|
||||
assemblyai_models: List = []
|
||||
snowflake_models: List = []
|
||||
gradient_ai_models: List = []
|
||||
llama_models: List = []
|
||||
nscale_models: List = []
|
||||
nebius_models: List = []
|
||||
nebius_embedding_models: List = []
|
||||
deepgram_models: List = []
|
||||
elevenlabs_models: List = []
|
||||
dashscope_models: List = []
|
||||
moonshot_models: List = []
|
||||
v0_models: List = []
|
||||
morph_models: List = []
|
||||
lambda_ai_models: List = []
|
||||
hyperbolic_models: List = []
|
||||
recraft_models: List = []
|
||||
cometapi_models: List = []
|
||||
oci_models: List = []
|
||||
from typing import Set
|
||||
open_ai_chat_completion_models: Set = set()
|
||||
open_ai_text_completion_models: Set = set()
|
||||
cohere_models: Set = set()
|
||||
cohere_chat_models: Set = set()
|
||||
mistral_chat_models: Set = set()
|
||||
text_completion_codestral_models: Set = set()
|
||||
anthropic_models: Set = set()
|
||||
openrouter_models: Set = set()
|
||||
datarobot_models: Set = set()
|
||||
vertex_language_models: Set = set()
|
||||
vertex_vision_models: Set = set()
|
||||
vertex_chat_models: Set = set()
|
||||
vertex_code_chat_models: Set = set()
|
||||
vertex_ai_image_models: Set = set()
|
||||
vertex_text_models: Set = set()
|
||||
vertex_code_text_models: Set = set()
|
||||
vertex_embedding_models: Set = set()
|
||||
vertex_anthropic_models: Set = set()
|
||||
vertex_llama3_models: Set = set()
|
||||
vertex_deepseek_models: Set = set()
|
||||
vertex_ai_ai21_models: Set = set()
|
||||
vertex_mistral_models: Set = set()
|
||||
ai21_models: Set = set()
|
||||
ai21_chat_models: Set = set()
|
||||
nlp_cloud_models: Set = set()
|
||||
aleph_alpha_models: Set = set()
|
||||
bedrock_models: Set = set()
|
||||
bedrock_converse_models: Set = set(BEDROCK_CONVERSE_MODELS)
|
||||
fireworks_ai_models: Set = set()
|
||||
fireworks_ai_embedding_models: Set = set()
|
||||
deepinfra_models: Set = set()
|
||||
perplexity_models: Set = set()
|
||||
watsonx_models: Set = set()
|
||||
gemini_models: Set = set()
|
||||
xai_models: Set = set()
|
||||
deepseek_models: Set = set()
|
||||
azure_ai_models: Set = set()
|
||||
jina_ai_models: Set = set()
|
||||
voyage_models: Set = set()
|
||||
infinity_models: Set = set()
|
||||
databricks_models: Set = set()
|
||||
cloudflare_models: Set = set()
|
||||
codestral_models: Set = set()
|
||||
friendliai_models: Set = set()
|
||||
featherless_ai_models: Set = set()
|
||||
palm_models: Set = set()
|
||||
groq_models: Set = set()
|
||||
azure_models: Set = set()
|
||||
azure_text_models: Set = set()
|
||||
anyscale_models: Set = set()
|
||||
cerebras_models: Set = set()
|
||||
galadriel_models: Set = set()
|
||||
sambanova_models: Set = set()
|
||||
sambanova_embedding_models: Set = set()
|
||||
novita_models: Set = set()
|
||||
assemblyai_models: Set = set()
|
||||
snowflake_models: Set = set()
|
||||
gradient_ai_models: Set = set()
|
||||
llama_models: Set = set()
|
||||
nscale_models: Set = set()
|
||||
nebius_models: Set = set()
|
||||
nebius_embedding_models: Set = set()
|
||||
deepgram_models: Set = set()
|
||||
elevenlabs_models: Set = set()
|
||||
dashscope_models: Set = set()
|
||||
moonshot_models: Set = set()
|
||||
v0_models: Set = set()
|
||||
morph_models: Set = set()
|
||||
lambda_ai_models: Set = set()
|
||||
hyperbolic_models: Set = set()
|
||||
recraft_models: Set = set()
|
||||
cometapi_models: Set = set()
|
||||
oci_models: Set = set()
|
||||
|
||||
|
||||
def is_bedrock_pricing_only_model(key: str) -> bool:
|
||||
|
|
@ -580,166 +581,166 @@ def add_known_models():
|
|||
if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(
|
||||
key
|
||||
):
|
||||
open_ai_chat_completion_models.append(key)
|
||||
open_ai_chat_completion_models.add(key)
|
||||
elif value.get("litellm_provider") == "text-completion-openai":
|
||||
open_ai_text_completion_models.append(key)
|
||||
open_ai_text_completion_models.add(key)
|
||||
elif value.get("litellm_provider") == "azure_text":
|
||||
azure_text_models.append(key)
|
||||
azure_text_models.add(key)
|
||||
elif value.get("litellm_provider") == "cohere":
|
||||
cohere_models.append(key)
|
||||
cohere_models.add(key)
|
||||
elif value.get("litellm_provider") == "cohere_chat":
|
||||
cohere_chat_models.append(key)
|
||||
cohere_chat_models.add(key)
|
||||
elif value.get("litellm_provider") == "mistral":
|
||||
mistral_chat_models.append(key)
|
||||
mistral_chat_models.add(key)
|
||||
elif value.get("litellm_provider") == "anthropic":
|
||||
anthropic_models.append(key)
|
||||
anthropic_models.add(key)
|
||||
elif value.get("litellm_provider") == "empower":
|
||||
empower_models.append(key)
|
||||
empower_models.add(key)
|
||||
elif value.get("litellm_provider") == "openrouter":
|
||||
openrouter_models.append(key)
|
||||
openrouter_models.add(key)
|
||||
elif value.get("litellm_provider") == "datarobot":
|
||||
datarobot_models.append(key)
|
||||
datarobot_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-text-models":
|
||||
vertex_text_models.append(key)
|
||||
vertex_text_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-code-text-models":
|
||||
vertex_code_text_models.append(key)
|
||||
vertex_code_text_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-language-models":
|
||||
vertex_language_models.append(key)
|
||||
vertex_language_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-vision-models":
|
||||
vertex_vision_models.append(key)
|
||||
vertex_vision_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-chat-models":
|
||||
vertex_chat_models.append(key)
|
||||
vertex_chat_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-code-chat-models":
|
||||
vertex_code_chat_models.append(key)
|
||||
vertex_code_chat_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-embedding-models":
|
||||
vertex_embedding_models.append(key)
|
||||
vertex_embedding_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-anthropic_models":
|
||||
key = key.replace("vertex_ai/", "")
|
||||
vertex_anthropic_models.append(key)
|
||||
vertex_anthropic_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-llama_models":
|
||||
key = key.replace("vertex_ai/", "")
|
||||
vertex_llama3_models.append(key)
|
||||
vertex_llama3_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-deepseek_models":
|
||||
key = key.replace("vertex_ai/", "")
|
||||
vertex_deepseek_models.append(key)
|
||||
vertex_deepseek_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-mistral_models":
|
||||
key = key.replace("vertex_ai/", "")
|
||||
vertex_mistral_models.append(key)
|
||||
vertex_mistral_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-ai21_models":
|
||||
key = key.replace("vertex_ai/", "")
|
||||
vertex_ai_ai21_models.append(key)
|
||||
vertex_ai_ai21_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-image-models":
|
||||
key = key.replace("vertex_ai/", "")
|
||||
vertex_ai_image_models.append(key)
|
||||
vertex_ai_image_models.add(key)
|
||||
elif value.get("litellm_provider") == "ai21":
|
||||
if value.get("mode") == "chat":
|
||||
ai21_chat_models.append(key)
|
||||
ai21_chat_models.add(key)
|
||||
else:
|
||||
ai21_models.append(key)
|
||||
ai21_models.add(key)
|
||||
elif value.get("litellm_provider") == "nlp_cloud":
|
||||
nlp_cloud_models.append(key)
|
||||
nlp_cloud_models.add(key)
|
||||
elif value.get("litellm_provider") == "aleph_alpha":
|
||||
aleph_alpha_models.append(key)
|
||||
aleph_alpha_models.add(key)
|
||||
elif value.get(
|
||||
"litellm_provider"
|
||||
) == "bedrock" and not is_bedrock_pricing_only_model(key):
|
||||
bedrock_models.append(key)
|
||||
bedrock_models.add(key)
|
||||
elif value.get("litellm_provider") == "bedrock_converse":
|
||||
bedrock_converse_models.append(key)
|
||||
bedrock_converse_models.add(key)
|
||||
elif value.get("litellm_provider") == "deepinfra":
|
||||
deepinfra_models.append(key)
|
||||
deepinfra_models.add(key)
|
||||
elif value.get("litellm_provider") == "perplexity":
|
||||
perplexity_models.append(key)
|
||||
perplexity_models.add(key)
|
||||
elif value.get("litellm_provider") == "watsonx":
|
||||
watsonx_models.append(key)
|
||||
watsonx_models.add(key)
|
||||
elif value.get("litellm_provider") == "gemini":
|
||||
gemini_models.append(key)
|
||||
gemini_models.add(key)
|
||||
elif value.get("litellm_provider") == "fireworks_ai":
|
||||
# ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
|
||||
if "-to-" not in key and "fireworks-ai-default" not in key:
|
||||
fireworks_ai_models.append(key)
|
||||
fireworks_ai_models.add(key)
|
||||
elif value.get("litellm_provider") == "fireworks_ai-embedding-models":
|
||||
# ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
|
||||
if "-to-" not in key:
|
||||
fireworks_ai_embedding_models.append(key)
|
||||
fireworks_ai_embedding_models.add(key)
|
||||
elif value.get("litellm_provider") == "text-completion-codestral":
|
||||
text_completion_codestral_models.append(key)
|
||||
text_completion_codestral_models.add(key)
|
||||
elif value.get("litellm_provider") == "xai":
|
||||
xai_models.append(key)
|
||||
xai_models.add(key)
|
||||
elif value.get("litellm_provider") == "deepseek":
|
||||
deepseek_models.append(key)
|
||||
deepseek_models.add(key)
|
||||
elif value.get("litellm_provider") == "meta_llama":
|
||||
llama_models.append(key)
|
||||
llama_models.add(key)
|
||||
elif value.get("litellm_provider") == "nscale":
|
||||
nscale_models.append(key)
|
||||
nscale_models.add(key)
|
||||
elif value.get("litellm_provider") == "azure_ai":
|
||||
azure_ai_models.append(key)
|
||||
azure_ai_models.add(key)
|
||||
elif value.get("litellm_provider") == "voyage":
|
||||
voyage_models.append(key)
|
||||
voyage_models.add(key)
|
||||
elif value.get("litellm_provider") == "infinity":
|
||||
infinity_models.append(key)
|
||||
infinity_models.add(key)
|
||||
elif value.get("litellm_provider") == "databricks":
|
||||
databricks_models.append(key)
|
||||
databricks_models.add(key)
|
||||
elif value.get("litellm_provider") == "cloudflare":
|
||||
cloudflare_models.append(key)
|
||||
cloudflare_models.add(key)
|
||||
elif value.get("litellm_provider") == "codestral":
|
||||
codestral_models.append(key)
|
||||
codestral_models.add(key)
|
||||
elif value.get("litellm_provider") == "friendliai":
|
||||
friendliai_models.append(key)
|
||||
friendliai_models.add(key)
|
||||
elif value.get("litellm_provider") == "palm":
|
||||
palm_models.append(key)
|
||||
palm_models.add(key)
|
||||
elif value.get("litellm_provider") == "groq":
|
||||
groq_models.append(key)
|
||||
groq_models.add(key)
|
||||
elif value.get("litellm_provider") == "azure":
|
||||
azure_models.append(key)
|
||||
azure_models.add(key)
|
||||
elif value.get("litellm_provider") == "anyscale":
|
||||
anyscale_models.append(key)
|
||||
anyscale_models.add(key)
|
||||
elif value.get("litellm_provider") == "cerebras":
|
||||
cerebras_models.append(key)
|
||||
cerebras_models.add(key)
|
||||
elif value.get("litellm_provider") == "galadriel":
|
||||
galadriel_models.append(key)
|
||||
galadriel_models.add(key)
|
||||
elif value.get("litellm_provider") == "sambanova":
|
||||
sambanova_models.append(key)
|
||||
sambanova_models.add(key)
|
||||
elif value.get("litellm_provider") == "sambanova-embedding-models":
|
||||
sambanova_embedding_models.append(key)
|
||||
sambanova_embedding_models.add(key)
|
||||
elif value.get("litellm_provider") == "novita":
|
||||
novita_models.append(key)
|
||||
novita_models.add(key)
|
||||
elif value.get("litellm_provider") == "nebius-chat-models":
|
||||
nebius_models.append(key)
|
||||
nebius_models.add(key)
|
||||
elif value.get("litellm_provider") == "nebius-embedding-models":
|
||||
nebius_embedding_models.append(key)
|
||||
nebius_embedding_models.add(key)
|
||||
elif value.get("litellm_provider") == "assemblyai":
|
||||
assemblyai_models.append(key)
|
||||
assemblyai_models.add(key)
|
||||
elif value.get("litellm_provider") == "jina_ai":
|
||||
jina_ai_models.append(key)
|
||||
jina_ai_models.add(key)
|
||||
elif value.get("litellm_provider") == "snowflake":
|
||||
snowflake_models.append(key)
|
||||
snowflake_models.add(key)
|
||||
elif value.get("litellm_provider") == "gradient_ai":
|
||||
gradient_ai_models.append(key)
|
||||
gradient_ai_models.add(key)
|
||||
elif value.get("litellm_provider") == "featherless_ai":
|
||||
featherless_ai_models.append(key)
|
||||
featherless_ai_models.add(key)
|
||||
elif value.get("litellm_provider") == "deepgram":
|
||||
deepgram_models.append(key)
|
||||
deepgram_models.add(key)
|
||||
elif value.get("litellm_provider") == "elevenlabs":
|
||||
elevenlabs_models.append(key)
|
||||
elevenlabs_models.add(key)
|
||||
elif value.get("litellm_provider") == "dashscope":
|
||||
dashscope_models.append(key)
|
||||
dashscope_models.add(key)
|
||||
elif value.get("litellm_provider") == "moonshot":
|
||||
moonshot_models.append(key)
|
||||
moonshot_models.add(key)
|
||||
elif value.get("litellm_provider") == "v0":
|
||||
v0_models.append(key)
|
||||
v0_models.add(key)
|
||||
elif value.get("litellm_provider") == "morph":
|
||||
morph_models.append(key)
|
||||
morph_models.add(key)
|
||||
elif value.get("litellm_provider") == "lambda_ai":
|
||||
lambda_ai_models.append(key)
|
||||
lambda_ai_models.add(key)
|
||||
elif value.get("litellm_provider") == "hyperbolic":
|
||||
hyperbolic_models.append(key)
|
||||
hyperbolic_models.add(key)
|
||||
elif value.get("litellm_provider") == "recraft":
|
||||
recraft_models.append(key)
|
||||
recraft_models.add(key)
|
||||
elif value.get("litellm_provider") == "cometapi":
|
||||
cometapi_models.append(key)
|
||||
cometapi_models.add(key)
|
||||
elif value.get("litellm_provider") == "oci":
|
||||
oci_models.append(key)
|
||||
oci_models.add(key)
|
||||
|
||||
|
||||
add_known_models()
|
||||
|
|
@ -769,68 +770,68 @@ ollama_models = ["llama2"]
|
|||
|
||||
maritalk_models = ["maritalk"]
|
||||
|
||||
model_list = (
|
||||
model_list = list(
|
||||
open_ai_chat_completion_models
|
||||
+ open_ai_text_completion_models
|
||||
+ cohere_models
|
||||
+ cohere_chat_models
|
||||
+ anthropic_models
|
||||
+ replicate_models
|
||||
+ openrouter_models
|
||||
+ datarobot_models
|
||||
+ huggingface_models
|
||||
+ vertex_chat_models
|
||||
+ vertex_text_models
|
||||
+ ai21_models
|
||||
+ ai21_chat_models
|
||||
+ together_ai_models
|
||||
+ baseten_models
|
||||
+ aleph_alpha_models
|
||||
+ nlp_cloud_models
|
||||
+ ollama_models
|
||||
+ bedrock_models
|
||||
+ deepinfra_models
|
||||
+ perplexity_models
|
||||
+ maritalk_models
|
||||
+ vertex_language_models
|
||||
+ watsonx_models
|
||||
+ gemini_models
|
||||
+ text_completion_codestral_models
|
||||
+ xai_models
|
||||
+ deepseek_models
|
||||
+ azure_ai_models
|
||||
+ voyage_models
|
||||
+ infinity_models
|
||||
+ databricks_models
|
||||
+ cloudflare_models
|
||||
+ codestral_models
|
||||
+ friendliai_models
|
||||
+ palm_models
|
||||
+ groq_models
|
||||
+ azure_models
|
||||
+ anyscale_models
|
||||
+ cerebras_models
|
||||
+ galadriel_models
|
||||
+ sambanova_models
|
||||
+ azure_text_models
|
||||
+ novita_models
|
||||
+ assemblyai_models
|
||||
+ jina_ai_models
|
||||
+ snowflake_models
|
||||
+ gradient_ai_models
|
||||
+ llama_models
|
||||
+ featherless_ai_models
|
||||
+ nscale_models
|
||||
+ deepgram_models
|
||||
+ elevenlabs_models
|
||||
+ dashscope_models
|
||||
+ moonshot_models
|
||||
+ v0_models
|
||||
+ morph_models
|
||||
+ lambda_ai_models
|
||||
+ recraft_models
|
||||
+ cometapi_models
|
||||
+ oci_models
|
||||
| open_ai_text_completion_models
|
||||
| cohere_models
|
||||
| cohere_chat_models
|
||||
| anthropic_models
|
||||
| set(replicate_models)
|
||||
| openrouter_models
|
||||
| datarobot_models
|
||||
| set(huggingface_models)
|
||||
| vertex_chat_models
|
||||
| vertex_text_models
|
||||
| ai21_models
|
||||
| ai21_chat_models
|
||||
| set(together_ai_models)
|
||||
| set(baseten_models)
|
||||
| aleph_alpha_models
|
||||
| nlp_cloud_models
|
||||
| set(ollama_models)
|
||||
| bedrock_models
|
||||
| deepinfra_models
|
||||
| perplexity_models
|
||||
| set(maritalk_models)
|
||||
| vertex_language_models
|
||||
| watsonx_models
|
||||
| gemini_models
|
||||
| text_completion_codestral_models
|
||||
| xai_models
|
||||
| deepseek_models
|
||||
| azure_ai_models
|
||||
| voyage_models
|
||||
| infinity_models
|
||||
| databricks_models
|
||||
| cloudflare_models
|
||||
| codestral_models
|
||||
| friendliai_models
|
||||
| palm_models
|
||||
| groq_models
|
||||
| azure_models
|
||||
| anyscale_models
|
||||
| cerebras_models
|
||||
| galadriel_models
|
||||
| sambanova_models
|
||||
| azure_text_models
|
||||
| novita_models
|
||||
| assemblyai_models
|
||||
| jina_ai_models
|
||||
| snowflake_models
|
||||
| gradient_ai_models
|
||||
| llama_models
|
||||
| featherless_ai_models
|
||||
| nscale_models
|
||||
| deepgram_models
|
||||
| elevenlabs_models
|
||||
| dashscope_models
|
||||
| moonshot_models
|
||||
| v0_models
|
||||
| morph_models
|
||||
| lambda_ai_models
|
||||
| recraft_models
|
||||
| cometapi_models
|
||||
| oci_models
|
||||
)
|
||||
|
||||
model_list_set = set(model_list)
|
||||
|
|
@ -839,9 +840,9 @@ provider_list: List[Union[LlmProviders, str]] = list(LlmProviders)
|
|||
|
||||
|
||||
models_by_provider: dict = {
|
||||
"openai": open_ai_chat_completion_models + open_ai_text_completion_models,
|
||||
"openai": open_ai_chat_completion_models | open_ai_text_completion_models,
|
||||
"text-completion-openai": open_ai_text_completion_models,
|
||||
"cohere": cohere_models + cohere_chat_models,
|
||||
"cohere": cohere_models | cohere_chat_models,
|
||||
"cohere_chat": cohere_chat_models,
|
||||
"anthropic": anthropic_models,
|
||||
"replicate": replicate_models,
|
||||
|
|
@ -850,14 +851,9 @@ models_by_provider: dict = {
|
|||
"baseten": baseten_models,
|
||||
"openrouter": openrouter_models,
|
||||
"datarobot": datarobot_models,
|
||||
"vertex_ai": vertex_chat_models
|
||||
+ vertex_text_models
|
||||
+ vertex_anthropic_models
|
||||
+ vertex_vision_models
|
||||
+ vertex_language_models
|
||||
+ vertex_deepseek_models,
|
||||
"vertex_ai": vertex_chat_models | vertex_text_models | vertex_anthropic_models | vertex_vision_models | vertex_language_models | vertex_deepseek_models,
|
||||
"ai21": ai21_models,
|
||||
"bedrock": bedrock_models + bedrock_converse_models,
|
||||
"bedrock": bedrock_models | bedrock_converse_models,
|
||||
"petals": petals_models,
|
||||
"ollama": ollama_models,
|
||||
"ollama_chat": ollama_models,
|
||||
|
|
@ -866,7 +862,7 @@ models_by_provider: dict = {
|
|||
"maritalk": maritalk_models,
|
||||
"watsonx": watsonx_models,
|
||||
"gemini": gemini_models,
|
||||
"fireworks_ai": fireworks_ai_models + fireworks_ai_embedding_models,
|
||||
"fireworks_ai": fireworks_ai_models | fireworks_ai_embedding_models,
|
||||
"aleph_alpha": aleph_alpha_models,
|
||||
"text-completion-codestral": text_completion_codestral_models,
|
||||
"xai": xai_models,
|
||||
|
|
@ -882,14 +878,14 @@ models_by_provider: dict = {
|
|||
"friendliai": friendliai_models,
|
||||
"palm": palm_models,
|
||||
"groq": groq_models,
|
||||
"azure": azure_models + azure_text_models,
|
||||
"azure": azure_models | azure_text_models,
|
||||
"azure_text": azure_text_models,
|
||||
"anyscale": anyscale_models,
|
||||
"cerebras": cerebras_models,
|
||||
"galadriel": galadriel_models,
|
||||
"sambanova": sambanova_models + sambanova_embedding_models,
|
||||
"sambanova": sambanova_models | sambanova_embedding_models,
|
||||
"novita": novita_models,
|
||||
"nebius": nebius_models + nebius_embedding_models,
|
||||
"nebius": nebius_models | nebius_embedding_models,
|
||||
"assemblyai": assemblyai_models,
|
||||
"jina_ai": jina_ai_models,
|
||||
"snowflake": snowflake_models,
|
||||
|
|
@ -936,12 +932,12 @@ longer_context_model_fallback_dict: dict = {
|
|||
|
||||
all_embedding_models = (
|
||||
open_ai_embedding_models
|
||||
+ cohere_embedding_models
|
||||
+ bedrock_embedding_models
|
||||
+ vertex_embedding_models
|
||||
+ fireworks_ai_embedding_models
|
||||
+ nebius_embedding_models
|
||||
+ sambanova_embedding_models
|
||||
| set(cohere_embedding_models)
|
||||
| set(bedrock_embedding_models)
|
||||
| vertex_embedding_models
|
||||
| fireworks_ai_embedding_models
|
||||
| nebius_embedding_models
|
||||
| sambanova_embedding_models
|
||||
)
|
||||
|
||||
####### IMAGE GENERATION MODELS ###################
|
||||
|
|
|
|||
|
|
@ -485,7 +485,7 @@ _openai_like_providers: List = [
|
|||
"watsonx",
|
||||
] # private helper. similar to openai but require some custom auth / endpoint handling, so can't use the openai sdk
|
||||
# well supported replicate llms
|
||||
replicate_models: List = [
|
||||
replicate_models: set = set([
|
||||
# llama replicate supported LLMs
|
||||
"replicate/llama-2-70b-chat:2796ee9483c3fd7aa2e171d38f4ca12251a30609463dcfd4cd76703f22e96cdf",
|
||||
"a16z-infra/llama-2-13b-chat:2a7f981751ec7fdf87b5b91ad4db53683a98082e9ff7bfd12c8cd5ea85980a52",
|
||||
|
|
@ -498,9 +498,9 @@ replicate_models: List = [
|
|||
# Others
|
||||
"replicate/dolly-v2-12b:ef0e1aefc61f8e096ebe4db6b2bacc297daf2ef6899f0f7e001ec445893500e5",
|
||||
"replit/replit-code-v1-3b:b84f4c074b807211cd75e3e8b1589b6399052125b4c27106e43d47189e8415ad",
|
||||
]
|
||||
])
|
||||
|
||||
clarifai_models: List = [
|
||||
clarifai_models: set = set([
|
||||
"clarifai/meta.Llama-3.Llama-3-8B-Instruct",
|
||||
"clarifai/gcp.generate.gemma-1_1-7b-it",
|
||||
"clarifai/mistralai.completion.mixtral-8x22B",
|
||||
|
|
@ -564,10 +564,10 @@ clarifai_models: List = [
|
|||
"clarifai/gcp.generate.gemini-1_5-pro",
|
||||
"clarifai/gcp.generate.imagen-2",
|
||||
"clarifai/salesforce.blip.general-english-image-caption-blip-2",
|
||||
]
|
||||
])
|
||||
|
||||
|
||||
huggingface_models: List = [
|
||||
huggingface_models: set = set([
|
||||
"meta-llama/Llama-2-7b-hf",
|
||||
"meta-llama/Llama-2-7b-chat-hf",
|
||||
"meta-llama/Llama-2-13b-hf",
|
||||
|
|
@ -580,13 +580,13 @@ huggingface_models: List = [
|
|||
"meta-llama/Llama-2-13b-chat",
|
||||
"meta-llama/Llama-2-70b",
|
||||
"meta-llama/Llama-2-70b-chat",
|
||||
] # these have been tested on extensively. But by default all text2text-generation and text-generation models are supported by liteLLM. - https://docs.litellm.ai/docs/providers
|
||||
empower_models = [
|
||||
]) # these have been tested on extensively. But by default all text2text-generation and text-generation models are supported by liteLLM. - https://docs.litellm.ai/docs/providers
|
||||
empower_models = set([
|
||||
"empower/empower-functions",
|
||||
"empower/empower-functions-small",
|
||||
]
|
||||
])
|
||||
|
||||
together_ai_models: List = [
|
||||
together_ai_models: set = set([
|
||||
# llama llms - chat
|
||||
"togethercomputer/llama-2-70b-chat",
|
||||
# llama llms - language / instruct
|
||||
|
|
@ -614,16 +614,17 @@ together_ai_models: List = [
|
|||
"Austism/chronos-hermes-13b",
|
||||
"upstage/SOLAR-0-70b-16bit",
|
||||
"WizardLM/WizardLM-70B-V1.0",
|
||||
] # supports all together ai models, just pass in the model id e.g. completion(model="together_computer/replit_code_3b",...)
|
||||
])
|
||||
# supports all together ai models, just pass in the model id e.g. completion(model="together_computer/replit_code_3b",...)
|
||||
|
||||
|
||||
baseten_models: List = [
|
||||
baseten_models: set = set([
|
||||
"qvv0xeq",
|
||||
"q841o8w",
|
||||
"31dxrj3",
|
||||
] # FALCON 7B # WizardLM # Mosaic ML
|
||||
]) # FALCON 7B # WizardLM # Mosaic ML
|
||||
|
||||
featherless_ai_models: List = [
|
||||
featherless_ai_models: set = set([
|
||||
"featherless-ai/Qwerky-72B",
|
||||
"featherless-ai/Qwerky-QwQ-32B",
|
||||
"Qwen/Qwen2.5-72B-Instruct",
|
||||
|
|
@ -633,9 +634,9 @@ featherless_ai_models: List = [
|
|||
"mistralai/Mistral-Small-24B-Instruct-2501",
|
||||
"mistralai/Mistral-Nemo-Instruct-2407",
|
||||
"ProdeusUnity/Stellar-Odyssey-12b-v0.0",
|
||||
]
|
||||
])
|
||||
|
||||
nebius_models: List = [
|
||||
nebius_models: set = set([
|
||||
"Qwen/Qwen3-235B-A22B",
|
||||
"Qwen/Qwen3-30B-A3B-fast",
|
||||
"Qwen/Qwen3-32B",
|
||||
|
|
@ -648,9 +649,9 @@ nebius_models: List = [
|
|||
"meta-llama/Llama-3.3-70B-Instruct-fast",
|
||||
"Qwen/Qwen2.5-32B-Instruct-fast",
|
||||
"Qwen/Qwen2.5-Coder-32B-Instruct-fast",
|
||||
]
|
||||
])
|
||||
|
||||
dashscope_models: List = [
|
||||
dashscope_models: set = set([
|
||||
"qwen-turbo",
|
||||
"qwen-plus",
|
||||
"qwen-max",
|
||||
|
|
@ -661,13 +662,13 @@ dashscope_models: List = [
|
|||
"qwen3-235b-a22b",
|
||||
"qwen3-32b",
|
||||
"qwen3-30b-a3b",
|
||||
]
|
||||
])
|
||||
|
||||
nebius_embedding_models: List = [
|
||||
nebius_embedding_models: set = set([
|
||||
"BAAI/bge-en-icl",
|
||||
"BAAI/bge-multilingual-gemma2",
|
||||
"intfloat/e5-mistral-7b-instruct",
|
||||
]
|
||||
])
|
||||
|
||||
BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[
|
||||
"cohere",
|
||||
|
|
@ -681,8 +682,8 @@ BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[
|
|||
"deepseek_r1",
|
||||
]
|
||||
|
||||
open_ai_embedding_models: List = ["text-embedding-ada-002"]
|
||||
cohere_embedding_models: List = [
|
||||
open_ai_embedding_models: set = set(["text-embedding-ada-002"])
|
||||
cohere_embedding_models: set = set([
|
||||
"embed-v4.0",
|
||||
"embed-english-v3.0",
|
||||
"embed-english-light-v3.0",
|
||||
|
|
@ -690,12 +691,12 @@ cohere_embedding_models: List = [
|
|||
"embed-english-v2.0",
|
||||
"embed-english-light-v2.0",
|
||||
"embed-multilingual-v2.0",
|
||||
]
|
||||
bedrock_embedding_models: List = [
|
||||
])
|
||||
bedrock_embedding_models: set = set([
|
||||
"amazon.titan-embed-text-v1",
|
||||
"cohere.embed-english-v3",
|
||||
"cohere.embed-multilingual-v3",
|
||||
]
|
||||
])
|
||||
|
||||
known_tokenizer_config = {
|
||||
"mistralai/Mistral-7B-Instruct-v0.1": {
|
||||
|
|
|
|||
|
|
@ -2316,47 +2316,47 @@ def register_model(model_cost: Union[str, dict]): # noqa: PLR0915
|
|||
# add new model names to provider lists
|
||||
if value.get("litellm_provider") == "openai":
|
||||
if key not in litellm.open_ai_chat_completion_models:
|
||||
litellm.open_ai_chat_completion_models.append(key)
|
||||
litellm.open_ai_chat_completion_models.add(key)
|
||||
elif value.get("litellm_provider") == "text-completion-openai":
|
||||
if key not in litellm.open_ai_text_completion_models:
|
||||
litellm.open_ai_text_completion_models.append(key)
|
||||
litellm.open_ai_text_completion_models.add(key)
|
||||
elif value.get("litellm_provider") == "cohere":
|
||||
if key not in litellm.cohere_models:
|
||||
litellm.cohere_models.append(key)
|
||||
litellm.cohere_models.add(key)
|
||||
elif value.get("litellm_provider") == "anthropic":
|
||||
if key not in litellm.anthropic_models:
|
||||
litellm.anthropic_models.append(key)
|
||||
litellm.anthropic_models.add(key)
|
||||
elif value.get("litellm_provider") == "openrouter":
|
||||
split_string = key.split("/", 1)
|
||||
if key not in litellm.openrouter_models:
|
||||
litellm.openrouter_models.append(split_string[1])
|
||||
litellm.openrouter_models.add(split_string[1])
|
||||
elif value.get("litellm_provider") == "vertex_ai-text-models":
|
||||
if key not in litellm.vertex_text_models:
|
||||
litellm.vertex_text_models.append(key)
|
||||
litellm.vertex_text_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-code-text-models":
|
||||
if key not in litellm.vertex_code_text_models:
|
||||
litellm.vertex_code_text_models.append(key)
|
||||
litellm.vertex_code_text_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-chat-models":
|
||||
if key not in litellm.vertex_chat_models:
|
||||
litellm.vertex_chat_models.append(key)
|
||||
litellm.vertex_chat_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-code-chat-models":
|
||||
if key not in litellm.vertex_code_chat_models:
|
||||
litellm.vertex_code_chat_models.append(key)
|
||||
litellm.vertex_code_chat_models.add(key)
|
||||
elif value.get("litellm_provider") == "ai21":
|
||||
if key not in litellm.ai21_models:
|
||||
litellm.ai21_models.append(key)
|
||||
litellm.ai21_models.add(key)
|
||||
elif value.get("litellm_provider") == "nlp_cloud":
|
||||
if key not in litellm.nlp_cloud_models:
|
||||
litellm.nlp_cloud_models.append(key)
|
||||
litellm.nlp_cloud_models.add(key)
|
||||
elif value.get("litellm_provider") == "aleph_alpha":
|
||||
if key not in litellm.aleph_alpha_models:
|
||||
litellm.aleph_alpha_models.append(key)
|
||||
litellm.aleph_alpha_models.add(key)
|
||||
elif value.get("litellm_provider") == "bedrock":
|
||||
if key not in litellm.bedrock_models:
|
||||
litellm.bedrock_models.append(key)
|
||||
litellm.bedrock_models.add(key)
|
||||
elif value.get("litellm_provider") == "novita":
|
||||
if key not in litellm.novita_models:
|
||||
litellm.novita_models.append(key)
|
||||
litellm.novita_models.add(key)
|
||||
return model_cost
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -338,11 +338,9 @@ def test_aget_valid_models():
|
|||
print(valid_models)
|
||||
|
||||
# list of openai supported llms on litellm
|
||||
expected_models = (
|
||||
litellm.open_ai_chat_completion_models + litellm.open_ai_text_completion_models
|
||||
)
|
||||
expected_models = litellm.open_ai_chat_completion_models | litellm.open_ai_text_completion_models
|
||||
|
||||
assert valid_models == expected_models
|
||||
assert set(valid_models) == set(expected_models)
|
||||
|
||||
# reset replicate env key
|
||||
os.environ = old_environ
|
||||
|
|
@ -355,7 +353,7 @@ def test_aget_valid_models():
|
|||
valid_models = get_valid_models()
|
||||
|
||||
print(valid_models)
|
||||
assert valid_models == expected_models
|
||||
assert set(valid_models) == set(expected_models)
|
||||
|
||||
# reset replicate env key
|
||||
os.environ = old_environ
|
||||
|
|
@ -376,7 +374,7 @@ def test_get_valid_models_with_custom_llm_provider(custom_llm_provider):
|
|||
)
|
||||
print(valid_models)
|
||||
assert len(valid_models) > 0
|
||||
assert provider_config.get_models() == valid_models
|
||||
assert set(provider_config.get_models()) == set(valid_models)
|
||||
|
||||
|
||||
# test_get_valid_models()
|
||||
|
|
|
|||
|
|
@ -100,7 +100,7 @@ def test_lambda_ai_models_configuration():
|
|||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
|
||||
# Clear and repopulate lambda_ai_models list after reloading model_cost
|
||||
litellm.lambda_ai_models = []
|
||||
litellm.lambda_ai_models = set()
|
||||
litellm.add_known_models()
|
||||
|
||||
# Some Lambda AI models to test
|
||||
|
|
@ -132,7 +132,7 @@ def test_lambda_ai_model_list_populated():
|
|||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
|
||||
# Clear and repopulate all model lists after reloading model_cost
|
||||
litellm.lambda_ai_models = []
|
||||
litellm.lambda_ai_models = set()
|
||||
litellm.add_known_models()
|
||||
|
||||
# This should be populated by the add_known_models function
|
||||
|
|
|
|||
|
|
@ -291,15 +291,15 @@ def test_avertex_ai():
|
|||
load_vertex_ai_credentials()
|
||||
test_models = (
|
||||
litellm.vertex_chat_models
|
||||
+ litellm.vertex_code_chat_models
|
||||
+ litellm.vertex_text_models
|
||||
+ litellm.vertex_code_text_models
|
||||
| litellm.vertex_code_chat_models
|
||||
| litellm.vertex_text_models
|
||||
| litellm.vertex_code_text_models
|
||||
)
|
||||
litellm.set_verbose = False
|
||||
vertex_ai_project = "pathrise-convert-1606954137718"
|
||||
|
||||
test_models = random.sample(test_models, 1)
|
||||
test_models += litellm.vertex_language_models # always test gemini-pro
|
||||
test_models = random.sample(list(test_models), 1)
|
||||
test_models += list(litellm.vertex_language_models) # always test gemini-pro
|
||||
for model in test_models:
|
||||
try:
|
||||
if model in VERTEX_MODELS_TO_NOT_TEST or (
|
||||
|
|
@ -345,12 +345,12 @@ def test_avertex_ai_stream():
|
|||
|
||||
test_models = (
|
||||
litellm.vertex_chat_models
|
||||
+ litellm.vertex_code_chat_models
|
||||
+ litellm.vertex_text_models
|
||||
+ litellm.vertex_code_text_models
|
||||
| litellm.vertex_code_chat_models
|
||||
| litellm.vertex_text_models
|
||||
| litellm.vertex_code_text_models
|
||||
)
|
||||
test_models = random.sample(test_models, 1)
|
||||
test_models += litellm.vertex_language_models # always test gemini-pro
|
||||
test_models = random.sample(list(test_models), 1)
|
||||
test_models += list(litellm.vertex_language_models) # always test gemini-pro
|
||||
for model in test_models:
|
||||
try:
|
||||
if model in VERTEX_MODELS_TO_NOT_TEST or (
|
||||
|
|
@ -393,12 +393,13 @@ async def test_async_vertexai_response():
|
|||
load_vertex_ai_credentials()
|
||||
test_models = (
|
||||
litellm.vertex_chat_models
|
||||
+ litellm.vertex_code_chat_models
|
||||
+ litellm.vertex_text_models
|
||||
+ litellm.vertex_code_text_models
|
||||
| litellm.vertex_code_chat_models
|
||||
| litellm.vertex_text_models
|
||||
| litellm.vertex_code_text_models
|
||||
)
|
||||
test_models = random.sample(test_models, 1)
|
||||
test_models += litellm.vertex_language_models # always test gemini-pro
|
||||
|
||||
test_models = random.sample(list(test_models), 1)
|
||||
test_models += list(litellm.vertex_language_models) # always test gemini-pro
|
||||
for model in test_models:
|
||||
print(
|
||||
f"model being tested in async call: {model}, litellm.vertex_language_models: {litellm.vertex_language_models}"
|
||||
|
|
@ -450,12 +451,12 @@ async def test_async_vertexai_streaming_response():
|
|||
load_vertex_ai_credentials()
|
||||
test_models = (
|
||||
litellm.vertex_chat_models
|
||||
+ litellm.vertex_code_chat_models
|
||||
+ litellm.vertex_text_models
|
||||
+ litellm.vertex_code_text_models
|
||||
| litellm.vertex_code_chat_models
|
||||
| litellm.vertex_text_models
|
||||
| litellm.vertex_code_text_models
|
||||
)
|
||||
test_models = random.sample(test_models, 1)
|
||||
test_models += litellm.vertex_language_models # always test gemini-pro
|
||||
test_models = random.sample(list(test_models), 1)
|
||||
test_models += list(litellm.vertex_language_models) # always test gemini-pro
|
||||
test_models = ["gemini-2.5-flash"]
|
||||
for model in test_models:
|
||||
if model in VERTEX_MODELS_TO_NOT_TEST or (
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue