From 8590cad7ed2b46b36b3aeb102cabdfa52a807d6e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 18 Oct 2026 11:18:54 -0700 Subject: [PATCH] fix(cost-map): move a model's provider membership when the remote map changes or drops it The import merge only ever added provider set memberships, so a bundled model the remote map moved to another provider kept its old one, and get_llm_provider still answered the old provider for its bare name. A model the remote map dropped also stayed in its bundled set, which sent an azure_ai deployment of a dropped OpenAI model down the azure route. The merge now removes the membership the old entry gave a model once the new one is in place, so lookups match a process that loaded the remote map at import. The provider set chain becomes a function that names a model's set, and the sets are typed as string sets. The raw-fork test moves to the integration suite, since unit tests run without child processes, and the other fork test was already covered there. --- litellm/__init__.py | 771 +++++++++--------- .../litellm_core_utils/get_model_cost_map.py | 10 + .../sdk/test_cost_map_background_fetch_sdk.py | 40 +- .../test_get_model_cost_map.py | 142 ++-- .../test_model_prices_and_context_window.py | 2 +- 5 files changed, 524 insertions(+), 441 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index a7e7ff4ae8a..d7b6319fd2c 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -8,7 +8,7 @@ _apply_process_defaults() import threading import os -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence from typing import ( Any, Callable, @@ -592,125 +592,125 @@ vertex_ai_safety_settings: Optional[dict] = None ####### COMPLETION MODELS ################### from typing import Set -open_ai_chat_completion_models: Set = set() -open_ai_text_completion_models: Set = set() -cohere_models: Set = set() -cohere_chat_models: Set = set() -mistral_chat_models: Set = set() -text_completion_codestral_models: Set = set() -text_completion_inception_models: Set = set() -anthropic_models: Set = set() -openrouter_models: Set = set() -datarobot_models: Set = set() -vertex_language_models: Set = set() -vertex_vision_models: Set = set() -vertex_chat_models: Set = set() -vertex_code_chat_models: Set = set() -vertex_ai_image_models: Set = set() -vertex_ai_video_models: Set = set() -vertex_text_models: Set = set() -vertex_code_text_models: Set = set() -vertex_embedding_models: Set = set() -vertex_anthropic_models: Set = set() -vertex_llama3_models: Set = set() -vertex_deepseek_models: Set = set() -vertex_ai_ai21_models: Set = set() -vertex_mistral_models: Set = set() -vertex_openai_models: Set = set() -vertex_minimax_models: Set = set() -vertex_moonshot_models: Set = set() -vertex_zai_models: Set = set() -ai21_models: Set = set() -ai21_chat_models: Set = set() -nlp_cloud_models: Set = set() -aleph_alpha_models: Set = set() -bedrock_models: Set = set() -bedrock_converse_models: Set = set(BEDROCK_CONVERSE_MODELS) -fal_ai_models: Set = set() -fireworks_ai_models: Set = set() -fireworks_ai_embedding_models: Set = set() -deepinfra_models: Set = set() -perplexity_models: Set = set() -watsonx_models: Set = set() -gemini_models: Set = set() -xai_models: Set = set() -zai_models: Set = set() -deepseek_models: Set = set() -tencent_models: Set = set() -runwayml_models: Set = set() -azure_ai_models: Set = set() -jina_ai_models: Set = set() -voyage_models: Set = set() -infinity_models: Set = set() -heroku_models: Set = set() -databricks_models: Set = set() -cloudflare_models: Set = set() -codestral_models: Set = set() -friendliai_models: Set = set() -featherless_ai_models: Set = set() -palm_models: Set = set() -groq_models: Set = set() -azure_models: Set = set() -azure_anthropic_models: Set = set() -azure_text_models: Set = set() -anyscale_models: Set = set() -cerebras_models: Set = set() -nadir_models: Set = set() -galadriel_models: Set = set() -nvidia_nim_models: Set = set() -nvidia_riva_models: Set = set() -soniox_models: Set = set() -sambanova_models: Set = set() +open_ai_chat_completion_models: Set[str] = set() +open_ai_text_completion_models: Set[str] = set() +cohere_models: Set[str] = set() +cohere_chat_models: Set[str] = set() +mistral_chat_models: Set[str] = set() +text_completion_codestral_models: Set[str] = set() +text_completion_inception_models: Set[str] = set() +anthropic_models: Set[str] = set() +openrouter_models: Set[str] = set() +datarobot_models: Set[str] = set() +vertex_language_models: Set[str] = set() +vertex_vision_models: Set[str] = set() +vertex_chat_models: Set[str] = set() +vertex_code_chat_models: Set[str] = set() +vertex_ai_image_models: Set[str] = set() +vertex_ai_video_models: Set[str] = set() +vertex_text_models: Set[str] = set() +vertex_code_text_models: Set[str] = set() +vertex_embedding_models: Set[str] = set() +vertex_anthropic_models: Set[str] = set() +vertex_llama3_models: Set[str] = set() +vertex_deepseek_models: Set[str] = set() +vertex_ai_ai21_models: Set[str] = set() +vertex_mistral_models: Set[str] = set() +vertex_openai_models: Set[str] = set() +vertex_minimax_models: Set[str] = set() +vertex_moonshot_models: Set[str] = set() +vertex_zai_models: Set[str] = set() +ai21_models: Set[str] = set() +ai21_chat_models: Set[str] = set() +nlp_cloud_models: Set[str] = set() +aleph_alpha_models: Set[str] = set() +bedrock_models: Set[str] = set() +bedrock_converse_models: Set[str] = set(BEDROCK_CONVERSE_MODELS) +fal_ai_models: Set[str] = set() +fireworks_ai_models: Set[str] = set() +fireworks_ai_embedding_models: Set[str] = set() +deepinfra_models: Set[str] = set() +perplexity_models: Set[str] = set() +watsonx_models: Set[str] = set() +gemini_models: Set[str] = set() +xai_models: Set[str] = set() +zai_models: Set[str] = set() +deepseek_models: Set[str] = set() +tencent_models: Set[str] = set() +runwayml_models: Set[str] = set() +azure_ai_models: Set[str] = set() +jina_ai_models: Set[str] = set() +voyage_models: Set[str] = set() +infinity_models: Set[str] = set() +heroku_models: Set[str] = set() +databricks_models: Set[str] = set() +cloudflare_models: Set[str] = set() +codestral_models: Set[str] = set() +friendliai_models: Set[str] = set() +featherless_ai_models: Set[str] = set() +palm_models: Set[str] = set() +groq_models: Set[str] = set() +azure_models: Set[str] = set() +azure_anthropic_models: Set[str] = set() +azure_text_models: Set[str] = set() +anyscale_models: Set[str] = set() +cerebras_models: Set[str] = set() +nadir_models: Set[str] = set() +galadriel_models: Set[str] = set() +nvidia_nim_models: Set[str] = set() +nvidia_riva_models: Set[str] = set() +soniox_models: Set[str] = set() +sambanova_models: Set[str] = set() scaledown_models: Final[Set[str]] = set() # Price reloads update this registry through existing references. -sambanova_embedding_models: Set = set() -novita_models: Set = set() -assemblyai_models: Set = set() -snowflake_models: Set = set() -gradient_ai_models: Set = set() -llama_models: Set = set() -nscale_models: Set = set() -nebius_models: Set = set() -nebius_embedding_models: Set = set() -aiml_models: Set = set() -deepgram_models: Set = set() -elevenlabs_models: Set = set() -dashscope_models: Set = set() -qwencloud_models: Set = set() -qwen_ai_platform_models: Set = set() -moonshot_models: Set = set() -publicai_models: Set = set() -darkbloom_models: Set = set() +sambanova_embedding_models: Set[str] = set() +novita_models: Set[str] = set() +assemblyai_models: Set[str] = set() +snowflake_models: Set[str] = set() +gradient_ai_models: Set[str] = set() +llama_models: Set[str] = set() +nscale_models: Set[str] = set() +nebius_models: Set[str] = set() +nebius_embedding_models: Set[str] = set() +aiml_models: Set[str] = set() +deepgram_models: Set[str] = set() +elevenlabs_models: Set[str] = set() +dashscope_models: Set[str] = set() +qwencloud_models: Set[str] = set() +qwen_ai_platform_models: Set[str] = set() +moonshot_models: Set[str] = set() +publicai_models: Set[str] = set() +darkbloom_models: Set[str] = set() openinfer_models: Final[Set[str]] = set() -v0_models: Set = set() -morph_models: Set = set() -lambda_ai_models: Set = set() -inception_models: Set = set() -hyperbolic_models: Set = set() -black_forest_labs_models: Set = set() -recraft_models: Set = set() -cometapi_models: Set = set() -oci_models: Set = set() -vercel_ai_gateway_models: Set = set() -edenai_models: Set = set() -volcengine_models: Set = set() -wandb_models: Set = set(WANDB_MODELS) -ovhcloud_models: Set = set() -ovhcloud_embedding_models: Set = set() -lemonade_models: Set = set() -docker_model_runner_models: Set = set() -amazon_nova_models: Set = set() -stability_models: Set = set() -github_copilot_models: Set = set() +v0_models: Set[str] = set() +morph_models: Set[str] = set() +lambda_ai_models: Set[str] = set() +inception_models: Set[str] = set() +hyperbolic_models: Set[str] = set() +black_forest_labs_models: Set[str] = set() +recraft_models: Set[str] = set() +cometapi_models: Set[str] = set() +oci_models: Set[str] = set() +vercel_ai_gateway_models: Set[str] = set() +edenai_models: Set[str] = set() +volcengine_models: Set[str] = set() +wandb_models: Set[str] = set(WANDB_MODELS) +ovhcloud_models: Set[str] = set() +ovhcloud_embedding_models: Set[str] = set() +lemonade_models: Set[str] = set() +docker_model_runner_models: Set[str] = set() +amazon_nova_models: Set[str] = set() +stability_models: Set[str] = set() +github_copilot_models: Set[str] = set() microsoft_365_copilot_models: Set[str] = set() -chatgpt_models: Set = set() -minimax_models: Set = set() -aws_polly_models: Set = set() -transcribe_models: Set = set() -gigachat_models: Set = set() -llamagate_models: Set = set() +chatgpt_models: Set[str] = set() +minimax_models: Set[str] = set() +aws_polly_models: Set[str] = set() +transcribe_models: Set[str] = set() +gigachat_models: Set[str] = set() +llamagate_models: Set[str] = set() coralbricks_models: Set[str] = set() -reducto_models: Set = set() -bedrock_mantle_models: Set = set() +reducto_models: Set[str] = set() +bedrock_mantle_models: Set[str] = set() def is_bedrock_pricing_only_model(key: str) -> bool: @@ -746,268 +746,264 @@ def is_openai_finetune_model(key: str) -> bool: return key.startswith("ft:") and not key.count(":") > 1 +def _provider_model_set(key: str, value: Mapping[str, object]) -> Optional[Tuple[Set[str], str]]: + provider: Final = value.get("litellm_provider") + if provider == "openai" and not is_openai_finetune_model(key): + return open_ai_chat_completion_models, key + if provider == "text-completion-openai": + return open_ai_text_completion_models, key + if provider == "azure_text": + return azure_text_models, key + if provider == "cohere": + return cohere_models, key + if provider == "cohere_chat": + return cohere_chat_models, key + if provider == "mistral": + return mistral_chat_models, key + if provider == "anthropic": + return anthropic_models, key + if provider == "empower": + return empower_models, key + if provider == "openrouter": + return openrouter_models, key + if provider == "vercel_ai_gateway": + return vercel_ai_gateway_models, key + if provider == "edenai": + return edenai_models, key + if provider == "datarobot": + return datarobot_models, key + if provider == "vertex_ai-text-models": + return vertex_text_models, key + if provider == "vertex_ai-code-text-models": + return vertex_code_text_models, key + if provider == "vertex_ai-language-models": + return vertex_language_models, key + if provider == "vertex_ai-vision-models": + return vertex_vision_models, key + if provider == "vertex_ai-chat-models": + return vertex_chat_models, key + if provider == "vertex_ai-code-chat-models": + return vertex_code_chat_models, key + if provider == "vertex_ai-embedding-models": + return vertex_embedding_models, key + if provider == "vertex_ai-anthropic_models": + return vertex_anthropic_models, key.replace("vertex_ai/", "") + if provider == "vertex_ai-llama_models": + return vertex_llama3_models, key.replace("vertex_ai/", "") + if provider == "vertex_ai-deepseek_models": + return vertex_deepseek_models, key.replace("vertex_ai/", "") + if provider == "vertex_ai-mistral_models": + return vertex_mistral_models, key.replace("vertex_ai/", "") + if provider == "vertex_ai-ai21_models": + return vertex_ai_ai21_models, key.replace("vertex_ai/", "") + if provider == "vertex_ai-image-models": + return vertex_ai_image_models, key.replace("vertex_ai/", "") + if provider == "vertex_ai-video-models": + return vertex_ai_video_models, key.replace("vertex_ai/", "") + if provider == "vertex_ai-openai_models": + return vertex_openai_models, key.replace("vertex_ai/", "") + if provider == "vertex_ai-minimax_models": + return vertex_minimax_models, key.replace("vertex_ai/", "") + if provider == "vertex_ai-moonshot_models": + return vertex_moonshot_models, key.replace("vertex_ai/", "") + if provider == "vertex_ai-zai_models": + return vertex_zai_models, key.replace("vertex_ai/", "") + if provider == "ai21": + if value.get("mode") == "chat": + return ai21_chat_models, key + else: + return ai21_models, key + if provider == "nlp_cloud": + return nlp_cloud_models, key + if provider == "aleph_alpha": + return aleph_alpha_models, key + if provider == "bedrock" and value.get("mode") == "guardrail": + return None + if provider == "bedrock" and not is_bedrock_pricing_only_model(key): + return bedrock_models, key + if provider == "bedrock_converse": + return bedrock_converse_models, key + if provider == "deepinfra": + return deepinfra_models, key + if provider == "perplexity": + return perplexity_models, key + if provider == "watsonx": + return watsonx_models, key + if provider == "gemini": + return gemini_models, key + if provider == "fireworks_ai": + # ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params. + if "-to-" not in key and "fireworks-ai-default" not in key: + return fireworks_ai_models, key + if provider == "fireworks_ai-embedding-models": + # ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params. + if "-to-" not in key: + return fireworks_ai_embedding_models, key + if provider == "text-completion-codestral": + return text_completion_codestral_models, key + if provider == "text-completion-inception": + return text_completion_inception_models, key + if provider == "xai": + return xai_models, key + if provider == "zai": + return zai_models, key + if provider == "fal_ai": + return fal_ai_models, key + if provider == "deepseek": + return deepseek_models, key + if provider == "tencent": + return tencent_models, key + if provider == "runwayml": + return runwayml_models, key + if provider == "meta_llama": + return llama_models, key + if provider == "nscale": + return nscale_models, key + if provider == "azure_ai": + return azure_ai_models, key + if provider == "voyage": + return voyage_models, key + if provider == "infinity": + return infinity_models, key + if provider == "databricks": + return databricks_models, key + if provider == "cloudflare": + return cloudflare_models, key + if provider == "codestral": + return codestral_models, key + if provider == "friendliai": + return friendliai_models, key + if provider == "palm": + return palm_models, key + if provider == "groq": + return groq_models, key + if provider == "azure": + return azure_models, key + if provider == "azure_anthropic": + return azure_anthropic_models, key + if provider == "anyscale": + return anyscale_models, key + if provider == "cerebras": + return cerebras_models, key + if provider == "nadir": + return nadir_models, key + if provider == "galadriel": + return galadriel_models, key + if provider == "nvidia_nim": + return nvidia_nim_models, key + if provider == "nvidia_riva": + return nvidia_riva_models, key + if provider == "soniox": + return soniox_models, key + if provider == "sambanova": + return sambanova_models, key + if provider == "scaledown": + return scaledown_models, key + if provider == "sambanova-embedding-models": + return sambanova_embedding_models, key + if provider == "novita": + return novita_models, key + if provider == "nebius-chat-models": + return nebius_models, key + if provider == "nebius-embedding-models": + return nebius_embedding_models, key + if provider == "aiml": + return aiml_models, key + if provider == "assemblyai": + return assemblyai_models, key + if provider == "jina_ai": + return jina_ai_models, key + if provider == "snowflake": + return snowflake_models, key + if provider == "gradient_ai": + return gradient_ai_models, key + if provider == "featherless_ai": + return featherless_ai_models, key + if provider == "deepgram": + return deepgram_models, key + if provider == "elevenlabs": + return elevenlabs_models, key + if provider == "heroku": + return heroku_models, key + if provider == "dashscope": + return dashscope_models, key + if provider == "qwencloud": + return qwencloud_models, key + if provider == "qwen_ai_platform": + return qwen_ai_platform_models, key + if provider == "modelscope": + return modelscope_models, key + if provider == "moonshot": + return moonshot_models, key + if provider == "publicai": + return publicai_models, key + if provider == "darkbloom": + return darkbloom_models, key + if provider == "openinfer": + return openinfer_models, key + if provider == "v0": + return v0_models, key + if provider == "morph": + return morph_models, key + if provider == "lambda_ai": + return lambda_ai_models, key + if provider == "inception": + return inception_models, key + if provider == "hyperbolic": + return hyperbolic_models, key + if provider == "black_forest_labs": + return black_forest_labs_models, key + if provider == "recraft": + return recraft_models, key + if provider == "cometapi": + return cometapi_models, key + if provider == "oci": + return oci_models, key + if provider == "volcengine": + return volcengine_models, key + if provider == "wandb": + return wandb_models, key + if provider == "ovhcloud": + return ovhcloud_models, key + if provider == "ovhcloud-embedding-models": + return ovhcloud_embedding_models, key + if provider == "lemonade": + return lemonade_models, key + if provider == "docker_model_runner": + return docker_model_runner_models, key + if provider == "amazon_nova": + return amazon_nova_models, key + if provider == "stability": + return stability_models, key + if provider == "github_copilot": + return github_copilot_models, key + if provider == "microsoft_365_copilot": + return microsoft_365_copilot_models, key + if provider == "chatgpt": + return chatgpt_models, key + if provider == "minimax": + return minimax_models, key + if provider == "aws_polly": + return aws_polly_models, key + if provider == "transcribe": + return transcribe_models, key + if provider == "gigachat": + return gigachat_models, key + if provider == "llamagate": + return llamagate_models, key + if provider == "coralbricks": + return coralbricks_models, key + if provider == "reducto": + return reducto_models, key + if provider == "bedrock_mantle": + return bedrock_mantle_models, key + return None + + def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, object]]) -> None: for key, value in tuple(model_cost_map.items()): - if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(key): - open_ai_chat_completion_models.add(key) - elif value.get("litellm_provider") == "text-completion-openai": - open_ai_text_completion_models.add(key) - elif value.get("litellm_provider") == "azure_text": - azure_text_models.add(key) - elif value.get("litellm_provider") == "cohere": - cohere_models.add(key) - elif value.get("litellm_provider") == "cohere_chat": - cohere_chat_models.add(key) - elif value.get("litellm_provider") == "mistral": - mistral_chat_models.add(key) - elif value.get("litellm_provider") == "anthropic": - anthropic_models.add(key) - elif value.get("litellm_provider") == "empower": - empower_models.add(key) - elif value.get("litellm_provider") == "openrouter": - openrouter_models.add(key) - elif value.get("litellm_provider") == "vercel_ai_gateway": - vercel_ai_gateway_models.add(key) - elif value.get("litellm_provider") == "edenai": - edenai_models.add(key) - elif value.get("litellm_provider") == "datarobot": - datarobot_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-text-models": - vertex_text_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-code-text-models": - vertex_code_text_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-language-models": - vertex_language_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-vision-models": - vertex_vision_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-chat-models": - vertex_chat_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-code-chat-models": - vertex_code_chat_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-embedding-models": - vertex_embedding_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-anthropic_models": - key = key.replace("vertex_ai/", "") - vertex_anthropic_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-llama_models": - key = key.replace("vertex_ai/", "") - vertex_llama3_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-deepseek_models": - key = key.replace("vertex_ai/", "") - vertex_deepseek_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-mistral_models": - key = key.replace("vertex_ai/", "") - vertex_mistral_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-ai21_models": - key = key.replace("vertex_ai/", "") - vertex_ai_ai21_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-image-models": - key = key.replace("vertex_ai/", "") - vertex_ai_image_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-video-models": - key = key.replace("vertex_ai/", "") - vertex_ai_video_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-openai_models": - key = key.replace("vertex_ai/", "") - vertex_openai_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-minimax_models": - key = key.replace("vertex_ai/", "") - vertex_minimax_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-moonshot_models": - key = key.replace("vertex_ai/", "") - vertex_moonshot_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-zai_models": - key = key.replace("vertex_ai/", "") - vertex_zai_models.add(key) - elif value.get("litellm_provider") == "ai21": - if value.get("mode") == "chat": - ai21_chat_models.add(key) - else: - ai21_models.add(key) - elif value.get("litellm_provider") == "nlp_cloud": - nlp_cloud_models.add(key) - elif value.get("litellm_provider") == "aleph_alpha": - aleph_alpha_models.add(key) - elif value.get("litellm_provider") == "bedrock" and value.get("mode") == "guardrail": - pass - elif value.get("litellm_provider") == "bedrock" and not is_bedrock_pricing_only_model(key): - bedrock_models.add(key) - elif value.get("litellm_provider") == "bedrock_converse": - bedrock_converse_models.add(key) - elif value.get("litellm_provider") == "deepinfra": - deepinfra_models.add(key) - elif value.get("litellm_provider") == "perplexity": - perplexity_models.add(key) - elif value.get("litellm_provider") == "watsonx": - watsonx_models.add(key) - elif value.get("litellm_provider") == "gemini": - gemini_models.add(key) - elif value.get("litellm_provider") == "fireworks_ai": - # ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params. - if "-to-" not in key and "fireworks-ai-default" not in key: - fireworks_ai_models.add(key) - elif value.get("litellm_provider") == "fireworks_ai-embedding-models": - # ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params. - if "-to-" not in key: - fireworks_ai_embedding_models.add(key) - elif value.get("litellm_provider") == "text-completion-codestral": - text_completion_codestral_models.add(key) - elif value.get("litellm_provider") == "text-completion-inception": - text_completion_inception_models.add(key) - elif value.get("litellm_provider") == "xai": - xai_models.add(key) - elif value.get("litellm_provider") == "zai": - zai_models.add(key) - elif value.get("litellm_provider") == "fal_ai": - fal_ai_models.add(key) - elif value.get("litellm_provider") == "deepseek": - deepseek_models.add(key) - elif value.get("litellm_provider") == "tencent": - tencent_models.add(key) - elif value.get("litellm_provider") == "runwayml": - runwayml_models.add(key) - elif value.get("litellm_provider") == "meta_llama": - llama_models.add(key) - elif value.get("litellm_provider") == "nscale": - nscale_models.add(key) - elif value.get("litellm_provider") == "azure_ai": - azure_ai_models.add(key) - elif value.get("litellm_provider") == "voyage": - voyage_models.add(key) - elif value.get("litellm_provider") == "infinity": - infinity_models.add(key) - elif value.get("litellm_provider") == "databricks": - databricks_models.add(key) - elif value.get("litellm_provider") == "cloudflare": - cloudflare_models.add(key) - elif value.get("litellm_provider") == "codestral": - codestral_models.add(key) - elif value.get("litellm_provider") == "friendliai": - friendliai_models.add(key) - elif value.get("litellm_provider") == "palm": - palm_models.add(key) - elif value.get("litellm_provider") == "groq": - groq_models.add(key) - elif value.get("litellm_provider") == "azure": - azure_models.add(key) - elif value.get("litellm_provider") == "azure_anthropic": - azure_anthropic_models.add(key) - elif value.get("litellm_provider") == "anyscale": - anyscale_models.add(key) - elif value.get("litellm_provider") == "cerebras": - cerebras_models.add(key) - elif value.get("litellm_provider") == "nadir": - nadir_models.add(key) - elif value.get("litellm_provider") == "galadriel": - galadriel_models.add(key) - elif value.get("litellm_provider") == "nvidia_nim": - nvidia_nim_models.add(key) - elif value.get("litellm_provider") == "nvidia_riva": - nvidia_riva_models.add(key) - elif value.get("litellm_provider") == "soniox": - soniox_models.add(key) - elif value.get("litellm_provider") == "sambanova": - sambanova_models.add(key) - elif value.get("litellm_provider") == "scaledown": - scaledown_models.add(key) - elif value.get("litellm_provider") == "sambanova-embedding-models": - sambanova_embedding_models.add(key) - elif value.get("litellm_provider") == "novita": - novita_models.add(key) - elif value.get("litellm_provider") == "nebius-chat-models": - nebius_models.add(key) - elif value.get("litellm_provider") == "nebius-embedding-models": - nebius_embedding_models.add(key) - elif value.get("litellm_provider") == "aiml": - aiml_models.add(key) - elif value.get("litellm_provider") == "assemblyai": - assemblyai_models.add(key) - elif value.get("litellm_provider") == "jina_ai": - jina_ai_models.add(key) - elif value.get("litellm_provider") == "snowflake": - snowflake_models.add(key) - elif value.get("litellm_provider") == "gradient_ai": - gradient_ai_models.add(key) - elif value.get("litellm_provider") == "featherless_ai": - featherless_ai_models.add(key) - elif value.get("litellm_provider") == "deepgram": - deepgram_models.add(key) - elif value.get("litellm_provider") == "elevenlabs": - elevenlabs_models.add(key) - elif value.get("litellm_provider") == "heroku": - heroku_models.add(key) - elif value.get("litellm_provider") == "dashscope": - dashscope_models.add(key) - elif value.get("litellm_provider") == "qwencloud": - qwencloud_models.add(key) - elif value.get("litellm_provider") == "qwen_ai_platform": - qwen_ai_platform_models.add(key) - elif value.get("litellm_provider") == "modelscope": - modelscope_models.add(key) - elif value.get("litellm_provider") == "moonshot": - moonshot_models.add(key) - elif value.get("litellm_provider") == "publicai": - publicai_models.add(key) - elif value.get("litellm_provider") == "darkbloom": - darkbloom_models.add(key) - elif value.get("litellm_provider") == "openinfer": - openinfer_models.add(key) - elif value.get("litellm_provider") == "v0": - v0_models.add(key) - elif value.get("litellm_provider") == "morph": - morph_models.add(key) - elif value.get("litellm_provider") == "lambda_ai": - lambda_ai_models.add(key) - elif value.get("litellm_provider") == "inception": - inception_models.add(key) - elif value.get("litellm_provider") == "hyperbolic": - hyperbolic_models.add(key) - elif value.get("litellm_provider") == "black_forest_labs": - black_forest_labs_models.add(key) - elif value.get("litellm_provider") == "recraft": - recraft_models.add(key) - elif value.get("litellm_provider") == "cometapi": - cometapi_models.add(key) - elif value.get("litellm_provider") == "oci": - oci_models.add(key) - elif value.get("litellm_provider") == "volcengine": - volcengine_models.add(key) - elif value.get("litellm_provider") == "wandb": - wandb_models.add(key) - elif value.get("litellm_provider") == "ovhcloud": - ovhcloud_models.add(key) - elif value.get("litellm_provider") == "ovhcloud-embedding-models": - ovhcloud_embedding_models.add(key) - elif value.get("litellm_provider") == "lemonade": - lemonade_models.add(key) - elif value.get("litellm_provider") == "docker_model_runner": - docker_model_runner_models.add(key) - elif value.get("litellm_provider") == "amazon_nova": - amazon_nova_models.add(key) - elif value.get("litellm_provider") == "stability": - stability_models.add(key) - elif value.get("litellm_provider") == "github_copilot": - github_copilot_models.add(key) - elif value.get("litellm_provider") == "microsoft_365_copilot": - microsoft_365_copilot_models.add(key) - elif value.get("litellm_provider") == "chatgpt": - chatgpt_models.add(key) - elif value.get("litellm_provider") == "minimax": - minimax_models.add(key) - elif value.get("litellm_provider") == "aws_polly": - aws_polly_models.add(key) - elif value.get("litellm_provider") == "transcribe": - transcribe_models.add(key) - elif value.get("litellm_provider") == "gigachat": - gigachat_models.add(key) - elif value.get("litellm_provider") == "llamagate": - llamagate_models.add(key) - elif value.get("litellm_provider") == "coralbricks": - coralbricks_models.add(key) - elif value.get("litellm_provider") == "reducto": - reducto_models.add(key) - elif value.get("litellm_provider") == "bedrock_mantle": - bedrock_mantle_models.add(key) + if (membership := _provider_model_set(key, value)) is not None: + membership[0].add(membership[1]) def add_known_models(model_cost_map: Optional[Dict] = None): @@ -1028,6 +1024,29 @@ def add_known_models_to_model_list() -> None: model_list_set.update(added) +def _provider_model_set_left( + key: str, before: Mapping[str, object], after: Mapping[str, object] +) -> Optional[Tuple[Set[str], str]]: + old: Final = _provider_model_set(key, before) + new: Final = _provider_model_set(key, after) + if old is None or (new is not None and old[0] is new[0] and old[1] == new[1]): + return None + return old + + +def drop_stale_known_models(changes: Iterable[Tuple[str, Mapping[str, object], Mapping[str, object]]]) -> None: + left: Final = tuple( + membership + for key, before, after in changes + if (membership := _provider_model_set_left(key, before, after)) is not None + ) + if not left: + return + for model_set, name in left: + model_set.discard(name) + models_by_provider.update(_build_models_by_provider()) + + _populate_provider_model_sets(model_cost) # known openai compatible endpoints - we'll eventually move this list to the model_prices_and_context_window.json dictionary diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index 69a19ffe6bf..4dcdf6e75cb 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -844,6 +844,11 @@ def _merge_into_import_map(fetch: _ImportFetch, finalized: Mapping[str, object]) if entry_id not in reshaped and isinstance(live := shared_once[entry_id][0], MutableMapping) ) replaced: Final = {key: reshaped[id(value)] for key, value in tuple(fetch.bundled.items()) if id(value) in reshaped} + moved: Final = tuple( + (key, MappingProxyType(dict(value)), changed[id(value)]) + for key, value in tuple(fetch.bundled.items()) + if id(value) in changed and _is_cost_entry(value) + ) kept_for_remote: Final = { id(entry): reshaped.get(entry_id, current) for entry_id, (current, _, entry) in shared_once.items() } @@ -862,6 +867,10 @@ def _merge_into_import_map(fetch: _ImportFetch, finalized: Mapping[str, object]) for key in fetch.loaded.keys() - finalized.keys() if _untouched_since_import(fetch.bundled.get(key), fetch.loaded[key], parsed.get(key, _NO_FIELDS)) ) + regrouped: Final = ( + *moved, + *((key, entry, _NO_FIELDS) for key in dropped if _is_cost_entry(entry := fetch.bundled.get(key))), + ) for live_entry, entry in updated_in_place: live_entry.update(entry) fetch.bundled.update({**replaced, **added, **added_since_import}) @@ -873,6 +882,7 @@ def _merge_into_import_map(fetch: _ImportFetch, finalized: Mapping[str, object]) _record_remote_adoption() utils._invalidate_model_cost_lowercase_map() # pyright: ignore[reportPrivateUsage] # required cache invalidation litellm.add_known_models_to_model_list() + litellm.drop_stale_known_models(regrouped) utils.reapply_runtime_model_cost_registrations() _PendingImportFetch.adoptions += 1 diff --git a/tests/integration/sdk/test_cost_map_background_fetch_sdk.py b/tests/integration/sdk/test_cost_map_background_fetch_sdk.py index 6e4fba0f7a4..f2b295c0289 100644 --- a/tests/integration/sdk/test_cost_map_background_fetch_sdk.py +++ b/tests/integration/sdk/test_cost_map_background_fetch_sdk.py @@ -49,6 +49,17 @@ _SCRIPT: Final = textwrap.dedent( join_fetch() os._exit(0 if model in litellm.model_cost else 1) print(os.waitstatus_to_exitcode(os.waitpid(pid, 0)[1]), flush=True) + elif variant == "raw_fork": + import ctypes, signal + pid = ctypes.PyDLL(None).fork() + if pid == 0: + signal.alarm(60) + try: + print(litellm.cost_per_token(model="gpt-4o", prompt_tokens=1000, completion_tokens=0)[0], flush=True) + os._exit(0) + finally: + os._exit(1) + print(os.waitstatus_to_exitcode(os.waitpid(pid, 0)[1]), flush=True) elif variant == "forks_during_retry_wait": while get_model_cost_map_source_info()["fallback_reason"] == "Remote fetch in progress": time.sleep(0.01) @@ -67,9 +78,23 @@ _SCRIPT: Final = textwrap.dedent( ) +def _bundled_map() -> dict: + return json.loads(files("litellm").joinpath("model_prices_and_context_window_backup.json").read_text()) + + +def _remote_gpt_4o_input_cost() -> float: + return 3 * _bundled_map()["gpt-4o"]["input_cost_per_token"] + + def _remote_map() -> bytes: - bundled: Final = json.loads(files("litellm").joinpath("model_prices_and_context_window_backup.json").read_text()) - return json.dumps({**bundled, _REMOTE_ONLY_MODEL: {"litellm_provider": "openai", "mode": "chat"}}).encode() + bundled: Final = _bundled_map() + return json.dumps( + { + **bundled, + "gpt-4o": {**bundled["gpt-4o"], "input_cost_per_token": _remote_gpt_4o_input_cost()}, + _REMOTE_ONLY_MODEL: {"litellm_provider": "openai", "mode": "chat"}, + } + ).encode() @dataclass(frozen=True, slots=True) @@ -160,6 +185,17 @@ def test_a_worker_forked_while_the_remote_map_is_in_flight_fetches_it_itself( assert child.rest == ("0",) +@pytest.mark.skipif(not hasattr(os, "fork"), reason="needs os.fork") +def test_a_worker_forked_without_the_fork_hooks_while_the_remote_map_is_in_flight_prices_from_it( + cost_map_host: _CostMapHost, tmp_path: Path +) -> None: + child: Final = _run_child(cost_map_host, "raw_fork", tmp_path, answer_after_import=False) + + assert child.after_import == "local Remote fetch in progress False" + assert float(child.rest[0]) == pytest.approx(1000 * _remote_gpt_4o_input_cost()) + assert child.rest[1:] == ("0",) + + @pytest.mark.skipif(not hasattr(os, "fork"), reason="needs os.fork") def test_workers_forked_during_a_rate_limited_retry_wait_spread_their_retries(tmp_path: Path) -> None: body: Final = _remote_map() diff --git a/tests/unit/litellm_core_utils/test_get_model_cost_map.py b/tests/unit/litellm_core_utils/test_get_model_cost_map.py index cc62fbf4eae..eae34fd8011 100644 --- a/tests/unit/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/unit/litellm_core_utils/test_get_model_cost_map.py @@ -5,10 +5,8 @@ count actual model entries, not reserved meta keys) and the extraction of the """ import asyncio, importlib, importlib.resources as importlib_get_model, json, litellm -import ctypes import logging import os -import signal import sys import threading import time @@ -1334,6 +1332,86 @@ def test_a_router_built_before_the_remote_map_lands_serves_a_bare_model_only_the assert litellm.get_model_info("claude-remote-only-test")["litellm_provider"] == "anthropic" +def _bundled_chat_model_of(provider: str) -> str: + return min( + key + for key, entry in _load_root_cost_map().items() + if entry.get("litellm_provider") == provider and entry.get("mode") == "chat" + ) + + +def _remote_map_editing(model: str, fields: dict) -> bytes: + remote_map = _load_root_cost_map() + remote_map[model] = {**remote_map[model], **fields} + return json.dumps(remote_map).encode() + + +def _land_remote_map(monkeypatch, body: bytes) -> None: + import litellm + from litellm.litellm_core_utils.get_model_cost_map import load_model_cost_map_at_import + + client, retry_started, release = _rate_limited_then_held_client(body) + try: + monkeypatch.setattr( + litellm, "model_cost", load_model_cost_map_at_import(url=_URL, client=client, sleep=lambda _: None) + ) + assert retry_started.wait(timeout=10) + finally: + _finish_fetch(release) + + +@pytest.mark.parametrize( + "remote_fields, provider_after", + ( + ({"litellm_provider": "text-completion-openai"}, "text-completion-openai"), + ({"input_cost_per_token": 1.0}, "openai"), + ), + ids=("moved", "repriced"), +) +def test_a_bare_model_the_remote_map_edits_resolves_to_the_remote_provider( + isolated_litellm_cost_state, monkeypatch, remote_fields, provider_after +): + import litellm + + model = _bundled_chat_model_of("openai") + assert litellm.get_llm_provider(model)[1] == "openai" + + _land_remote_map(monkeypatch, _remote_map_editing(model, remote_fields)) + + assert litellm.get_llm_provider(model)[1] == provider_after + + +def test_an_azure_ai_deployment_of_a_model_the_remote_map_drops_stays_on_azure_ai( + isolated_litellm_cost_state, monkeypatch +): + import litellm + + model = _bundled_chat_model_of("openai") + deployment = f"azure_ai/{model}" + api_base = "https://example.invalid" + assert litellm.get_llm_provider(deployment, api_base=api_base)[1] == "azure" + remote_map = _load_root_cost_map() + del remote_map[model] + + _land_remote_map(monkeypatch, json.dumps(remote_map).encode()) + + assert litellm.get_llm_provider(deployment, api_base=api_base)[1] == "azure_ai" + + +def test_a_model_the_remote_map_moves_to_another_provider_leaves_the_old_provider_wildcard_list( + isolated_litellm_cost_state, monkeypatch +): + import litellm + + model = _bundled_chat_model_of("azure") + assert model in litellm.models_by_provider["azure"] + + _land_remote_map(monkeypatch, _remote_map_editing(model, {"litellm_provider": "openai"})) + + assert model not in litellm.models_by_provider["azure"] + assert model in litellm.models_by_provider["openai"] + + def test_a_partial_edit_made_before_the_remote_map_lands_keeps_only_the_fields_it_changed( isolated_litellm_cost_state, monkeypatch ): @@ -1761,66 +1839,6 @@ def test_models_listed_while_the_remote_map_is_in_flight_include_a_model_only_th assert "claude-remote-only-test" in valid_models -@pytest.mark.skipif(not hasattr(os, "fork"), reason="needs os.fork") -def test_a_process_forked_while_the_remote_map_is_in_flight_inherits_it_once_adopted( - isolated_litellm_cost_state, monkeypatch, release_when_a_lookup_waits -): - from litellm.litellm_core_utils import get_model_cost_map as module - - client, started, release = _held_client(httpx.Response(200, content=_remote_map_with("claude-remote-only-test"))) - release_when_a_lookup_waits(release) - try: - monkeypatch.setattr(litellm, "model_cost", module.load_model_cost_map_at_import(url=_URL, client=client)) - assert started.wait(timeout=10) - child = os.fork() - if child == 0: - inherited = "claude-remote-only-test" in litellm.model_cost and module._PendingImportFetch.fetch is None - os._exit(0 if inherited else 1) - _, wait_status = os.waitpid(child, 0) - finally: - _finish_fetch(release) - - assert os.waitstatus_to_exitcode(wait_status) == 0 - - -@pytest.mark.skipif(not hasattr(os, "fork"), reason="needs fork") -def test_a_process_forked_without_the_fork_hooks_while_the_remote_map_is_in_flight_prices_from_it( - isolated_litellm_cost_state, monkeypatch -): - from litellm.litellm_core_utils.get_model_cost_map import load_model_cost_map_at_import - - body, remote_price = _remote_map_repricing_gpt_4o() - first_request_started = threading.Event() - release = threading.Event() - served = [] - - def handler(request): - served.append(request) - if len(served) == 1: - first_request_started.set() - release.wait(timeout=10) - return httpx.Response(200, content=body) - - client = httpx.Client(transport=httpx.MockTransport(handler)) - try: - monkeypatch.setattr(litellm, "model_cost", load_model_cost_map_at_import(url=_URL, client=client)) - assert first_request_started.wait(timeout=10) - child = ctypes.PyDLL(None).fork() - if child == 0: - signal.signal(signal.SIGALRM, signal.SIG_DFL) - signal.alarm(10) - try: - cost = litellm.cost_per_token(model="gpt-4o", prompt_tokens=1000, completion_tokens=0)[0] - os._exit(0 if cost == pytest.approx(1000 * remote_price) else 1) - finally: - os._exit(2) - _, wait_status = os.waitpid(child, 0) - finally: - _finish_fetch(release) - - assert os.waitstatus_to_exitcode(wait_status) == 0 - - def test_import_load_with_the_local_env_override_starts_no_fetch(monkeypatch): from litellm.litellm_core_utils.get_model_cost_map import load_model_cost_map_at_import diff --git a/tests/unit/test_model_prices_and_context_window.py b/tests/unit/test_model_prices_and_context_window.py index 395fe8c9750..3775d35d8c2 100644 --- a/tests/unit/test_model_prices_and_context_window.py +++ b/tests/unit/test_model_prices_and_context_window.py @@ -697,7 +697,7 @@ def unregistered_providers(rows: Mapping[str, object]) -> list[str]: def test_every_cost_map_provider_is_registered(path: Path): assert unregistered_providers(json.loads(path.read_text())) == [], ( f"{path.name} carries a litellm_provider whose models a `/*` grant does not list. A new provider " - "needs a `_models` set in litellm/__init__.py, filled in _populate_provider_model_sets and listed " + "needs a `_models` set in litellm/__init__.py, returned by _provider_model_set and listed " "in _build_models_by_provider. A new `-` label needs its rows added to a set that " "`models_by_provider[]` includes" )