From f79943548423f8464e8a75f7008891a8ac8e27de Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 28 Oct 2026 05:24:47 -0700 Subject: [PATCH] refactor(model-cost): derive the provider model sets from the live map [12/12] The 121 provider sets and models_by_provider are read-only views over the catalog's index, grouped by raw litellm_provider, so a direct write, reload or registration shows on the next read and a moved or dropped key leaves its old set. The populate chain and the models_by_provider builder are gone, and every provider in the map is listed Co-Authored-By: Claude Opus 5.5 --- .../python-bridge/src/model_registry.rs | 26 +- .../src/model_registry/name_index.rs | 19 +- .../src/model_registry/resolve.rs | 4 + litellm/__init__.py | 765 +++--------------- litellm/_catalog/provider_sets.py | 413 ++++++++++ .../litellm_core_utils/get_model_cost_map.py | 2 - litellm/rust_bridge/_native.pyi | 2 + litellm/utils.py | 86 +- tests/unit/_catalog/test_provider_sets.py | 70 ++ tests/unit/conftest.py | 34 +- .../test_get_llm_provider_logic.py | 6 +- ...test_azure_gpt_reasoning_transformation.py | 2 +- .../response/test_azure_transformation.py | 2 +- .../baseten/chat/test_baseten_completions.py | 10 +- ..._cross_region_inference_profile_mapping.py | 14 +- .../test_bedrock_mantle_transformation.py | 4 - .../chat/test_edenai_chat_transformation.py | 8 +- ...github_copilot_responses_transformation.py | 2 +- .../test_gpt_reasoning_transformation.py | 2 +- tests/unit/proxy/auth/test_auth_checks.py | 13 +- tests/unit/proxy/auth/test_model_checks.py | 13 +- tests/unit/test_cost_calculator.py | 1 - tests/unit/test_main.py | 6 +- tests/unit/test_router/test_router.py | 2 +- .../unit/test_router_model_cost_isolation.py | 5 +- tests/unit/test_utils.py | 40 +- 26 files changed, 739 insertions(+), 812 deletions(-) create mode 100644 litellm/_catalog/provider_sets.py create mode 100644 tests/unit/_catalog/test_provider_sets.py diff --git a/litellm-rust/crates/python-bridge/src/model_registry.rs b/litellm-rust/crates/python-bridge/src/model_registry.rs index 03de4c6ddc3..f26c5fa9f3e 100644 --- a/litellm-rust/crates/python-bridge/src/model_registry.rs +++ b/litellm-rust/crates/python-bridge/src/model_registry.rs @@ -18,7 +18,7 @@ use litellm_model_catalog::{BUNDLED_REGISTRY, Catalog, Keys, Layers, Located}; use litellm_model_catalog_refresh::{Source, git_blob_id}; use pyo3::exceptions::PyTypeError; use pyo3::prelude::*; -use pyo3::types::{PyDict, PyModule}; +use pyo3::types::{PyDict, PyFrozenSet, PyModule}; use serde::Serialize; pub(crate) use fetch::{Bundled, File, Provenance, Remote}; @@ -333,6 +333,30 @@ impl CatalogHandle { Ok(keys.provider_keys(provider)) } + /// The keys whose live row names this raw `litellm_provider`, empty when none (decision 18) + fn models_for<'py>( + &self, + py: Python<'py>, + provider: &str, + ) -> PyResult> { + let dict = self.bound_dict(py)?; + let keys = resolve::LiveKeys { + dict: &dict, + index: &self.index, + }; + PyFrozenSet::new(py, keys.provider_keys(provider)) + } + + /// Every raw `litellm_provider` string a live row names, sorted + fn providers(&self, py: Python<'_>) -> PyResult> { + let dict = self.bound_dict(py)?; + let keys = resolve::LiveKeys { + dict: &dict, + index: &self.index, + }; + Ok(keys.providers()) + } + /// main's 133-field copy of a row Python built, a generalization match or a fill-missing merge #[staticmethod] #[pyo3(signature = (key, row, provider = None))] diff --git a/litellm-rust/crates/python-bridge/src/model_registry/name_index.rs b/litellm-rust/crates/python-bridge/src/model_registry/name_index.rs index dd3c0ba0cb4..7f8db435ab8 100644 --- a/litellm-rust/crates/python-bridge/src/model_registry/name_index.rs +++ b/litellm-rust/crates/python-bridge/src/model_registry/name_index.rs @@ -4,10 +4,11 @@ use pyo3::prelude::*; use pyo3::types::{PyDict, PyString}; /// The one name index of decision 18. A normalized name to the actual key, and the keys each raw `litellm_provider` -/// names. Never a row, since rows are read live, so only adding or removing keys can make it stale +/// names. Never a row, since rows are read live, so only adding or removing keys can make it stale. It holds the dict it +/// indexed, so a freed dict's id can never be taken for it by a new dict of the same length #[derive(Debug, Default)] pub(crate) struct NameIndex { - dict_id: usize, + dict: Option>, len: usize, by_lower: HashMap, by_provider: HashMap>, @@ -32,7 +33,7 @@ impl NameIndex { by_lower.insert(key.to_lowercase(), key); } Self { - dict_id: dict.as_ptr() as usize, + dict: Some(dict.clone().unbind()), len: dict.len(), by_lower, by_provider, @@ -40,13 +41,23 @@ impl NameIndex { } pub(crate) fn is_current(&self, dict: &Bound<'_, PyDict>) -> bool { - self.dict_id == dict.as_ptr() as usize && self.len == dict.len() + self.dict + .as_ref() + .is_some_and(|indexed| indexed.as_ptr() == dict.as_ptr()) + && self.len == dict.len() } pub(crate) fn by_lower(&self, name: &str) -> Option<&str> { self.by_lower.get(&name.to_lowercase()).map(String::as_str) } + /// Every raw `litellm_provider` string some live row names, sorted + pub(crate) fn providers(&self) -> Vec { + let mut providers: Vec = self.by_provider.keys().cloned().collect(); + providers.sort(); + providers + } + pub(crate) fn provider_keys(&self, provider: &str) -> &[String] { self.by_provider.get(provider).map_or(&[], Vec::as_slice) } diff --git a/litellm-rust/crates/python-bridge/src/model_registry/resolve.rs b/litellm-rust/crates/python-bridge/src/model_registry/resolve.rs index e86b87c3877..2913cfbfd89 100644 --- a/litellm-rust/crates/python-bridge/src/model_registry/resolve.rs +++ b/litellm-rust/crates/python-bridge/src/model_registry/resolve.rs @@ -106,6 +106,10 @@ impl LiveKeys<'_, '_> { Some(NameIndex::build(self.dict)); } + pub(crate) fn providers(&self) -> Vec { + self.indexed(NameIndex::providers) + } + pub(crate) fn provider_keys(&self, provider: &str) -> Vec { self.indexed(|index| index.provider_keys(provider).to_vec()) } diff --git a/litellm/__init__.py b/litellm/__init__.py index a948e9eb40d..bc26f06f82b 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -7,7 +7,6 @@ _apply_process_defaults() ### INIT VARIABLES ######################### import threading import os -import datetime from collections.abc import Mapping, Sequence from typing import ( @@ -39,7 +38,6 @@ from litellm._logging import ( turn_on_json, log_level, ) -import re from litellm.constants import ( DEFAULT_BATCH_SIZE, DEFAULT_FLUSH_INTERVAL_SECONDS, @@ -61,9 +59,9 @@ from litellm.constants import ( clarifai_models, huggingface_models, modelscope_models, - empower_models, + empower_models as empower_models, together_ai_models, - WANDB_MODELS, + WANDB_MODELS as WANDB_MODELS, REPEATED_STREAMING_CHUNK_LIMIT, request_timeout, request_timeout_explicitly_set as request_timeout_explicitly_set, @@ -73,7 +71,7 @@ from litellm.constants import ( known_tokenizer_config, BEDROCK_INVOKE_PROVIDERS_LITERAL, BEDROCK_EMBEDDING_PROVIDERS_LITERAL, - BEDROCK_CONVERSE_MODELS, + BEDROCK_CONVERSE_MODELS as BEDROCK_CONVERSE_MODELS, DEFAULT_MAX_TOKENS, DEFAULT_SOFT_BUDGET, DEFAULT_ALLOWED_FAILS, @@ -548,6 +546,14 @@ _key_management_system: Optional["KeyManagementSystem"] = None #### PII MASKING #### output_parse_pii: bool = False ############################################# +from litellm._catalog.provider_sets import ( + LEGACY_NAMES as _LEGACY_PROVIDER_SETS, + ModelsByProvider, + _is_past_deprecation_date as _is_past_deprecation_date, # pyright: ignore[reportPrivateUsage] # main's name + is_bedrock_pricing_only_model as is_bedrock_pricing_only_model, + is_openai_finetune_model as is_openai_finetune_model, + legacy_set as _provider_set, +) from litellm.litellm_core_utils.get_model_cost_map import ( get_model_cost_map as get_model_cost_map, load_bundled_model_cost_map_at_import, @@ -594,455 +600,19 @@ config_path = None vertex_ai_safety_settings: Optional[dict] = None ####### COMPLETION MODELS ################### -from typing import Set - -open_ai_chat_completion_models: Set = set() -open_ai_text_completion_models: Set = set() -cohere_models: Set = set() -cohere_chat_models: Set = set() -mistral_chat_models: Set = set() -text_completion_codestral_models: Set = set() -text_completion_inception_models: Set = set() -anthropic_models: Set = set() -openrouter_models: Set = set() -datarobot_models: Set = set() -vertex_language_models: Set = set() -vertex_vision_models: Set = set() -vertex_chat_models: Set = set() -vertex_code_chat_models: Set = set() -vertex_ai_image_models: Set = set() -vertex_ai_video_models: Set = set() -vertex_text_models: Set = set() -vertex_code_text_models: Set = set() -vertex_embedding_models: Set = set() -vertex_anthropic_models: Set = set() -vertex_llama3_models: Set = set() -vertex_deepseek_models: Set = set() -vertex_ai_ai21_models: Set = set() -vertex_mistral_models: Set = set() -vertex_openai_models: Set = set() -vertex_minimax_models: Set = set() -vertex_moonshot_models: Set = set() -vertex_zai_models: Set = set() -ai21_models: Set = set() -ai21_chat_models: Set = set() -nlp_cloud_models: Set = set() -aleph_alpha_models: Set = set() -bedrock_models: Set = set() -bedrock_converse_models: Set = set(BEDROCK_CONVERSE_MODELS) -fal_ai_models: Set = set() -fireworks_ai_models: Set = set() -baseten_models: Set[str] = set() -fireworks_ai_embedding_models: Set = set() -deepinfra_models: Set = set() -perplexity_models: Set = set() -watsonx_models: Set = set() -gemini_models: Set = set() -xai_models: Set = set() -zai_models: Set = set() -deepseek_models: Set = set() -tencent_models: Set = set() -runwayml_models: Set = set() -azure_ai_models: Set = set() -jina_ai_models: Set = set() -voyage_models: Set = set() -infinity_models: Set = set() -heroku_models: Set = set() -databricks_models: Set = set() -cloudflare_models: Set = set() -codestral_models: Set = set() -friendliai_models: Set = set() -featherless_ai_models: Set = set() -palm_models: Set = set() -groq_models: Set = set() -azure_models: Set = set() -azure_anthropic_models: Set = set() -azure_text_models: Set = set() -anyscale_models: Set = set() -cerebras_models: Set = set() -nadir_models: Set = set() -galadriel_models: Set = set() -nvidia_nim_models: Set = set() -nvidia_riva_models: Set = set() -soniox_models: Set = set() -sambanova_models: Set = set() -scaledown_models: Final[Set[str]] = set() # Price reloads update this registry through existing references. -sambanova_embedding_models: Set = set() -novita_models: Set = set() -assemblyai_models: Set = set() -snowflake_models: Set = set() -gradient_ai_models: Set = set() -llama_models: Set = set() -nscale_models: Set = set() -nebius_models: Set = set() -nebius_embedding_models: Set = set() -aiml_models: Set = set() -deepgram_models: Set = set() -elevenlabs_models: Set = set() -dashscope_models: Set = set() -qwencloud_models: Set = set() -qwen_ai_platform_models: Set = set() -moonshot_models: Set = set() -publicai_models: Set = set() -darkbloom_models: Set = set() -openinfer_models: Final[Set[str]] = set() -v0_models: Set = set() -morph_models: Set = set() -opencode_go_models: Final[set[str]] = set() # mutable-ok: populated when the shared model catalog loads -lambda_ai_models: Set = set() -inception_models: Set = set() -hyperbolic_models: Set = set() -black_forest_labs_models: Set = set() -recraft_models: Set = set() -cometapi_models: Set = set() -oci_models: Set = set() -vercel_ai_gateway_models: Set = set() -edenai_models: Set = set() -volcengine_models: Set = set() -wandb_models: Set = set(WANDB_MODELS) -ovhcloud_models: Set = set() -ovhcloud_embedding_models: Set = set() -lemonade_models: Set = set() -docker_model_runner_models: Set = set() -amazon_nova_models: Set = set() -stability_models: Set = set() -github_copilot_models: Set = set() -microsoft_365_copilot_models: Set[str] = set() -chatgpt_models: Set = set() -minimax_models: Set = set() -aws_polly_models: Set = set() -transcribe_models: Set = set() -gigachat_models: Set = set() -llamagate_models: Set = set() -coralbricks_models: Set[str] = set() -reducto_models: Set = set() -bedrock_mantle_models: Set = set() -def is_bedrock_pricing_only_model(key: str) -> bool: - """ - Excludes keys with the pattern 'bedrock//'. These are in the model_prices_and_context_window.json file for pricing purposes only. - - Args: - key (str): A key to filter. - - Returns: - bool: True if the key matches the Bedrock pattern, False otherwise. - """ - # Regex to match 'bedrock//' - bedrock_pattern: Final = re.compile(r"^bedrock/[a-zA-Z0-9_-]+/.+$") - - if "month-commitment" in key: - return True - - is_match: Final = bedrock_pattern.match(key) - return is_match is not None -def is_openai_finetune_model(key: str) -> bool: - """ - Excludes model cost keys with the pattern 'ft:'. These are in the model_prices_and_context_window.json file for pricing purposes only. - - Args: - key (str): A key to filter. - - Returns: - bool: True if the key matches the OpenAI finetune pattern, False otherwise. - """ - return key.startswith("ft:") and not key.count(":") > 1 -def _is_past_deprecation_date(value: Mapping[str, object]) -> bool: - deprecation_date: Final = value.get("deprecation_date") - if not isinstance(deprecation_date, str): - return False - try: - return datetime.date.fromisoformat(deprecation_date) <= datetime.date.today() - except ValueError: - return False -def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, object]]) -> None: - for key, value in tuple(model_cost_map.items()): - if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(key): - open_ai_chat_completion_models.add(key) - elif value.get("litellm_provider") == "text-completion-openai": - open_ai_text_completion_models.add(key) - elif value.get("litellm_provider") == "azure_text": - azure_text_models.add(key) - elif value.get("litellm_provider") == "cohere": - cohere_models.add(key) - elif value.get("litellm_provider") == "cohere_chat": - cohere_chat_models.add(key) - elif value.get("litellm_provider") == "mistral": - mistral_chat_models.add(key) - elif value.get("litellm_provider") == "anthropic": - anthropic_models.add(key) - elif value.get("litellm_provider") == "empower": - empower_models.add(key) - elif value.get("litellm_provider") == "openrouter": - openrouter_models.add(key) - elif value.get("litellm_provider") == "vercel_ai_gateway": - vercel_ai_gateway_models.add(key) - elif value.get("litellm_provider") == "edenai": - edenai_models.add(key) - elif value.get("litellm_provider") == "datarobot": - datarobot_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-text-models": - vertex_text_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-code-text-models": - vertex_code_text_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-language-models": - vertex_language_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-vision-models": - vertex_vision_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-chat-models": - vertex_chat_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-code-chat-models": - vertex_code_chat_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-embedding-models": - vertex_embedding_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-anthropic_models": - key = key.replace("vertex_ai/", "") - vertex_anthropic_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-llama_models": - key = key.replace("vertex_ai/", "") - vertex_llama3_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-deepseek_models": - key = key.replace("vertex_ai/", "") - vertex_deepseek_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-mistral_models": - key = key.replace("vertex_ai/", "") - vertex_mistral_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-ai21_models": - key = key.replace("vertex_ai/", "") - vertex_ai_ai21_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-image-models": - key = key.replace("vertex_ai/", "") - vertex_ai_image_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-video-models": - key = key.replace("vertex_ai/", "") - vertex_ai_video_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-openai_models": - key = key.replace("vertex_ai/", "") - vertex_openai_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-minimax_models": - key = key.replace("vertex_ai/", "") - vertex_minimax_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-moonshot_models": - key = key.replace("vertex_ai/", "") - vertex_moonshot_models.add(key) - elif value.get("litellm_provider") == "vertex_ai-zai_models": - key = key.replace("vertex_ai/", "") - vertex_zai_models.add(key) - elif value.get("litellm_provider") == "ai21": - if value.get("mode") == "chat": - ai21_chat_models.add(key) - else: - ai21_models.add(key) - elif value.get("litellm_provider") == "nlp_cloud": - nlp_cloud_models.add(key) - elif value.get("litellm_provider") == "aleph_alpha": - aleph_alpha_models.add(key) - elif value.get("litellm_provider") == "bedrock" and value.get("mode") == "guardrail": - pass - elif value.get("litellm_provider") == "bedrock" and not is_bedrock_pricing_only_model(key): - bedrock_models.add(key) - elif value.get("litellm_provider") == "bedrock_converse": - bedrock_converse_models.add(key) - elif value.get("litellm_provider") == "deepinfra": - deepinfra_models.add(key) - elif value.get("litellm_provider") == "perplexity": - perplexity_models.add(key) - elif value.get("litellm_provider") == "watsonx": - watsonx_models.add(key) - elif value.get("litellm_provider") == "gemini": - gemini_models.add(key) - elif value.get("litellm_provider") == "fireworks_ai": - # ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params. - if "-to-" not in key and "fireworks-ai-default" not in key: - fireworks_ai_models.add(key) - elif value.get("litellm_provider") == "baseten": - if _is_past_deprecation_date(value): - baseten_models.discard(key) - else: - baseten_models.add(key) - elif value.get("litellm_provider") == "fireworks_ai-embedding-models": - # ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params. - if "-to-" not in key: - fireworks_ai_embedding_models.add(key) - elif value.get("litellm_provider") == "text-completion-codestral": - text_completion_codestral_models.add(key) - elif value.get("litellm_provider") == "text-completion-inception": - text_completion_inception_models.add(key) - elif value.get("litellm_provider") == "xai": - xai_models.add(key) - elif value.get("litellm_provider") == "zai": - zai_models.add(key) - elif value.get("litellm_provider") == "fal_ai": - fal_ai_models.add(key) - elif value.get("litellm_provider") == "deepseek": - deepseek_models.add(key) - elif value.get("litellm_provider") == "tencent": - tencent_models.add(key) - elif value.get("litellm_provider") == "runwayml": - runwayml_models.add(key) - elif value.get("litellm_provider") == "meta_llama": - llama_models.add(key) - elif value.get("litellm_provider") == "nscale": - nscale_models.add(key) - elif value.get("litellm_provider") == "azure_ai": - azure_ai_models.add(key) - elif value.get("litellm_provider") == "voyage": - voyage_models.add(key) - elif value.get("litellm_provider") == "infinity": - infinity_models.add(key) - elif value.get("litellm_provider") == "databricks": - databricks_models.add(key) - elif value.get("litellm_provider") == "cloudflare": - cloudflare_models.add(key) - elif value.get("litellm_provider") == "codestral": - codestral_models.add(key) - elif value.get("litellm_provider") == "friendliai": - friendliai_models.add(key) - elif value.get("litellm_provider") == "palm": - palm_models.add(key) - elif value.get("litellm_provider") == "groq": - groq_models.add(key) - elif value.get("litellm_provider") == "azure": - azure_models.add(key) - elif value.get("litellm_provider") == "azure_anthropic": - azure_anthropic_models.add(key) - elif value.get("litellm_provider") == "anyscale": - anyscale_models.add(key) - elif value.get("litellm_provider") == "cerebras": - cerebras_models.add(key) - elif value.get("litellm_provider") == "nadir": - nadir_models.add(key) - elif value.get("litellm_provider") == "galadriel": - galadriel_models.add(key) - elif value.get("litellm_provider") == "nvidia_nim": - nvidia_nim_models.add(key) - elif value.get("litellm_provider") == "nvidia_riva": - nvidia_riva_models.add(key) - elif value.get("litellm_provider") == "soniox": - soniox_models.add(key) - elif value.get("litellm_provider") == "sambanova": - sambanova_models.add(key) - elif value.get("litellm_provider") == "scaledown": - scaledown_models.add(key) - elif value.get("litellm_provider") == "sambanova-embedding-models": - sambanova_embedding_models.add(key) - elif value.get("litellm_provider") == "novita": - novita_models.add(key) - elif value.get("litellm_provider") == "nebius-chat-models": - nebius_models.add(key) - elif value.get("litellm_provider") == "nebius-embedding-models": - nebius_embedding_models.add(key) - elif value.get("litellm_provider") == "aiml": - aiml_models.add(key) - elif value.get("litellm_provider") == "assemblyai": - assemblyai_models.add(key) - elif value.get("litellm_provider") == "jina_ai": - jina_ai_models.add(key) - elif value.get("litellm_provider") == "snowflake": - snowflake_models.add(key) - elif value.get("litellm_provider") == "gradient_ai": - gradient_ai_models.add(key) - elif value.get("litellm_provider") == "featherless_ai": - featherless_ai_models.add(key) - elif value.get("litellm_provider") == "deepgram": - deepgram_models.add(key) - elif value.get("litellm_provider") == "elevenlabs": - elevenlabs_models.add(key) - elif value.get("litellm_provider") == "heroku": - heroku_models.add(key) - elif value.get("litellm_provider") == "dashscope": - dashscope_models.add(key) - elif value.get("litellm_provider") == "qwencloud": - qwencloud_models.add(key) - elif value.get("litellm_provider") == "qwen_ai_platform": - qwen_ai_platform_models.add(key) - elif value.get("litellm_provider") == "modelscope": - modelscope_models.add(key) - elif value.get("litellm_provider") == "moonshot": - moonshot_models.add(key) - elif value.get("litellm_provider") == "publicai": - publicai_models.add(key) - elif value.get("litellm_provider") == "darkbloom": - darkbloom_models.add(key) - elif value.get("litellm_provider") == "openinfer": - openinfer_models.add(key) - elif value.get("litellm_provider") == "v0": - v0_models.add(key) - elif value.get("litellm_provider") == "morph": - morph_models.add(key) - elif value.get("litellm_provider") == "opencode_go": - opencode_go_models.add(key) - elif value.get("litellm_provider") == "lambda_ai": - lambda_ai_models.add(key) - elif value.get("litellm_provider") == "inception": - inception_models.add(key) - elif value.get("litellm_provider") == "hyperbolic": - hyperbolic_models.add(key) - elif value.get("litellm_provider") == "black_forest_labs": - black_forest_labs_models.add(key) - elif value.get("litellm_provider") == "recraft": - recraft_models.add(key) - elif value.get("litellm_provider") == "cometapi": - cometapi_models.add(key) - elif value.get("litellm_provider") == "oci": - oci_models.add(key) - elif value.get("litellm_provider") == "volcengine": - volcengine_models.add(key) - elif value.get("litellm_provider") == "wandb": - wandb_models.add(key) - elif value.get("litellm_provider") == "ovhcloud": - ovhcloud_models.add(key) - elif value.get("litellm_provider") == "ovhcloud-embedding-models": - ovhcloud_embedding_models.add(key) - elif value.get("litellm_provider") == "lemonade": - lemonade_models.add(key) - elif value.get("litellm_provider") == "docker_model_runner": - docker_model_runner_models.add(key) - elif value.get("litellm_provider") == "amazon_nova": - amazon_nova_models.add(key) - elif value.get("litellm_provider") == "stability": - stability_models.add(key) - elif value.get("litellm_provider") == "github_copilot": - github_copilot_models.add(key) - elif value.get("litellm_provider") == "microsoft_365_copilot": - microsoft_365_copilot_models.add(key) - elif value.get("litellm_provider") == "chatgpt": - chatgpt_models.add(key) - elif value.get("litellm_provider") == "minimax": - minimax_models.add(key) - elif value.get("litellm_provider") == "aws_polly": - aws_polly_models.add(key) - elif value.get("litellm_provider") == "transcribe": - transcribe_models.add(key) - elif value.get("litellm_provider") == "gigachat": - gigachat_models.add(key) - elif value.get("litellm_provider") == "llamagate": - llamagate_models.add(key) - elif value.get("litellm_provider") == "coralbricks": - coralbricks_models.add(key) - elif value.get("litellm_provider") == "reducto": - reducto_models.add(key) - elif value.get("litellm_provider") == "bedrock_mantle": - bedrock_mantle_models.add(key) -def add_known_models(model_cost_map: Optional[Dict] = None): - """Fold `model_cost_map` (defaults to `litellm.model_cost`) into the per-provider model sets, - then refresh `models_by_provider` from those sets so the additions reach wildcard expansion. - The refresh updates the dict in place, so references captured before a reload stay live. - """ - _populate_provider_model_sets(model_cost_map if model_cost_map is not None else model_cost) - models_by_provider.update(_build_models_by_provider()) -_populate_provider_model_sets(model_cost) + # known openai compatible endpoints - we'll eventually move this list to the model_prices_and_context_window.json dictionary # this is maintained for Exception Mapping @@ -1073,97 +643,97 @@ ollama_models = ["llama2"] maritalk_models = ["maritalk"] model_list = list( - open_ai_chat_completion_models - | open_ai_text_completion_models - | cohere_models - | cohere_chat_models - | anthropic_models + _provider_set("open_ai_chat_completion_models") + | _provider_set("open_ai_text_completion_models") + | _provider_set("cohere_models") + | _provider_set("cohere_chat_models") + | _provider_set("anthropic_models") | set(replicate_models) - | openrouter_models - | datarobot_models + | _provider_set("openrouter_models") + | _provider_set("datarobot_models") | set(huggingface_models) - | vertex_chat_models - | vertex_text_models - | ai21_models - | ai21_chat_models + | _provider_set("vertex_chat_models") + | _provider_set("vertex_text_models") + | _provider_set("ai21_models") + | _provider_set("ai21_chat_models") | set(together_ai_models) - | set(baseten_models) - | aleph_alpha_models - | nlp_cloud_models + | set(_provider_set("baseten_models")) + | _provider_set("aleph_alpha_models") + | _provider_set("nlp_cloud_models") | set(ollama_models) - | bedrock_models - | deepinfra_models - | perplexity_models + | _provider_set("bedrock_models") + | _provider_set("deepinfra_models") + | _provider_set("perplexity_models") | set(maritalk_models) - | runwayml_models - | vertex_language_models - | watsonx_models - | gemini_models - | text_completion_codestral_models - | text_completion_inception_models - | xai_models - | zai_models - | fal_ai_models - | deepseek_models + | _provider_set("runwayml_models") + | _provider_set("vertex_language_models") + | _provider_set("watsonx_models") + | _provider_set("gemini_models") + | _provider_set("text_completion_codestral_models") + | _provider_set("text_completion_inception_models") + | _provider_set("xai_models") + | _provider_set("zai_models") + | _provider_set("fal_ai_models") + | _provider_set("deepseek_models") | modelscope_models - | azure_ai_models - | voyage_models - | infinity_models - | databricks_models - | cloudflare_models - | codestral_models - | friendliai_models - | palm_models - | groq_models - | azure_models - | azure_anthropic_models - | anyscale_models - | cerebras_models - | nadir_models - | galadriel_models - | nvidia_nim_models - | nvidia_riva_models - | soniox_models - | sambanova_models - | scaledown_models - | azure_text_models - | novita_models - | assemblyai_models - | jina_ai_models - | snowflake_models - | gradient_ai_models - | llama_models - | featherless_ai_models - | nscale_models - | deepgram_models - | elevenlabs_models - | dashscope_models - | qwencloud_models - | qwen_ai_platform_models - | moonshot_models - | publicai_models - | darkbloom_models - | openinfer_models - | v0_models - | morph_models - | opencode_go_models - | lambda_ai_models - | inception_models - | black_forest_labs_models - | recraft_models - | cometapi_models - | oci_models - | heroku_models - | vercel_ai_gateway_models - | edenai_models - | volcengine_models - | wandb_models - | ovhcloud_models - | lemonade_models - | docker_model_runner_models - | reducto_models - | bedrock_mantle_models - | coralbricks_models + | _provider_set("azure_ai_models") + | _provider_set("voyage_models") + | _provider_set("infinity_models") + | _provider_set("databricks_models") + | _provider_set("cloudflare_models") + | _provider_set("codestral_models") + | _provider_set("friendliai_models") + | _provider_set("palm_models") + | _provider_set("groq_models") + | _provider_set("azure_models") + | _provider_set("azure_anthropic_models") + | _provider_set("anyscale_models") + | _provider_set("cerebras_models") + | _provider_set("nadir_models") + | _provider_set("galadriel_models") + | _provider_set("nvidia_nim_models") + | _provider_set("nvidia_riva_models") + | _provider_set("soniox_models") + | _provider_set("sambanova_models") + | _provider_set("scaledown_models") + | _provider_set("azure_text_models") + | _provider_set("novita_models") + | _provider_set("assemblyai_models") + | _provider_set("jina_ai_models") + | _provider_set("snowflake_models") + | _provider_set("gradient_ai_models") + | _provider_set("llama_models") + | _provider_set("featherless_ai_models") + | _provider_set("nscale_models") + | _provider_set("deepgram_models") + | _provider_set("elevenlabs_models") + | _provider_set("dashscope_models") + | _provider_set("qwencloud_models") + | _provider_set("qwen_ai_platform_models") + | _provider_set("moonshot_models") + | _provider_set("publicai_models") + | _provider_set("darkbloom_models") + | _provider_set("openinfer_models") + | _provider_set("v0_models") + | _provider_set("morph_models") + | _provider_set("opencode_go_models") + | _provider_set("lambda_ai_models") + | _provider_set("inception_models") + | _provider_set("black_forest_labs_models") + | _provider_set("recraft_models") + | _provider_set("cometapi_models") + | _provider_set("oci_models") + | _provider_set("heroku_models") + | _provider_set("vercel_ai_gateway_models") + | _provider_set("edenai_models") + | _provider_set("volcengine_models") + | _provider_set("wandb_models") + | _provider_set("ovhcloud_models") + | _provider_set("lemonade_models") + | _provider_set("docker_model_runner_models") + | _provider_set("reducto_models") + | _provider_set("bedrock_mantle_models") + | _provider_set("coralbricks_models") | set(clarifai_models) ) @@ -1172,128 +742,13 @@ model_list_set = set(model_list) # provider_list is lazy-loaded via __getattr__ to avoid importing LlmProviders at import time -def _build_models_by_provider() -> dict: - return { - "openai": open_ai_chat_completion_models | open_ai_text_completion_models, - "text-completion-openai": open_ai_text_completion_models, - "cohere": cohere_models | cohere_chat_models, - "cohere_chat": cohere_chat_models, - "anthropic": anthropic_models, - "replicate": replicate_models, - "huggingface": huggingface_models, - "together_ai": together_ai_models, - "baseten": baseten_models, - "openrouter": openrouter_models, - "vercel_ai_gateway": vercel_ai_gateway_models, - "edenai": edenai_models, - "datarobot": datarobot_models, - "vertex_ai": vertex_chat_models - | vertex_code_chat_models - | vertex_text_models - | vertex_code_text_models - | vertex_anthropic_models - | vertex_vision_models - | vertex_language_models - | vertex_deepseek_models - | vertex_minimax_models - | vertex_moonshot_models - | vertex_zai_models, - "ai21": ai21_models | ai21_chat_models, - "bedrock": bedrock_models | bedrock_converse_models, - "petals": petals_models, - "ollama": ollama_models, - "ollama_chat": ollama_models, - "deepinfra": deepinfra_models, - "perplexity": perplexity_models, - "maritalk": maritalk_models, - "watsonx": watsonx_models, - "gemini": gemini_models, - "fireworks_ai": fireworks_ai_models | fireworks_ai_embedding_models, - "aleph_alpha": aleph_alpha_models, - "text-completion-codestral": text_completion_codestral_models, - "text-completion-inception": text_completion_inception_models, - "xai": xai_models, - "zai": zai_models, - "fal_ai": fal_ai_models, - "deepseek": deepseek_models, - "tencent": tencent_models, - "runwayml": runwayml_models, - "mistral": mistral_chat_models, - "azure_ai": azure_ai_models, - "voyage": voyage_models, - "infinity": infinity_models, - "databricks": databricks_models, - "cloudflare": cloudflare_models, - "codestral": codestral_models, - "nlp_cloud": nlp_cloud_models, - "friendliai": friendliai_models, - "palm": palm_models, - "groq": groq_models, - "azure": azure_models | azure_text_models, - "azure_anthropic": azure_anthropic_models, - "azure_text": azure_text_models, - "anyscale": anyscale_models, - "cerebras": cerebras_models, - "nadir": nadir_models, - "galadriel": galadriel_models, - "nvidia_nim": nvidia_nim_models, - "nvidia_riva": nvidia_riva_models, - "soniox": soniox_models, - "sambanova": sambanova_models | sambanova_embedding_models, - "scaledown": scaledown_models, - "novita": novita_models, - "nebius": nebius_models | nebius_embedding_models, - "aiml": aiml_models, - "assemblyai": assemblyai_models, - "jina_ai": jina_ai_models, - "snowflake": snowflake_models, - "gradient_ai": gradient_ai_models, - "meta_llama": llama_models, - "nscale": nscale_models, - "featherless_ai": featherless_ai_models, - "deepgram": deepgram_models, - "elevenlabs": elevenlabs_models, - "heroku": heroku_models, - "dashscope": dashscope_models, - "qwencloud": qwencloud_models, - "qwen_ai_platform": qwen_ai_platform_models, - "modelscope": modelscope_models, - "moonshot": moonshot_models, - "publicai": publicai_models, - "darkbloom": darkbloom_models, - "openinfer": openinfer_models, - "v0": v0_models, - "morph": morph_models, - "opencode_go": opencode_go_models, - "lambda_ai": lambda_ai_models, - "inception": inception_models, - "hyperbolic": hyperbolic_models, - "black_forest_labs": black_forest_labs_models, - "recraft": recraft_models, - "cometapi": cometapi_models, - "oci": oci_models, - "volcengine": volcengine_models, - "wandb": wandb_models, - "ovhcloud": ovhcloud_models | ovhcloud_embedding_models, - "lemonade": lemonade_models, - "clarifai": clarifai_models, - "amazon_nova": amazon_nova_models, - "stability": stability_models, - "github_copilot": github_copilot_models, - "microsoft_365_copilot": microsoft_365_copilot_models, - "chatgpt": chatgpt_models, - "minimax": minimax_models, - "aws_polly": aws_polly_models, - "transcribe": transcribe_models, - "gigachat": gigachat_models, - "llamagate": llamagate_models, - "coralbricks": coralbricks_models, - "reducto": reducto_models, - "bedrock_mantle": bedrock_mantle_models, - } -models_by_provider: dict = _build_models_by_provider() +models_by_provider: Final = ModelsByProvider() + + +def add_known_models(model_cost_map: Optional[Dict] = None) -> None: + """Kept for callers. The provider sets are derived from the live map on read (decision 18)""" # mapping for those models which have larger equivalents longer_context_model_fallback_dict: dict = { @@ -1323,11 +778,11 @@ all_embedding_models = ( open_ai_embedding_models | set(cohere_embedding_models) | set(bedrock_embedding_models) - | vertex_embedding_models - | fireworks_ai_embedding_models - | nebius_embedding_models - | sambanova_embedding_models - | ovhcloud_embedding_models + | _provider_set("vertex_embedding_models") + | _provider_set("fireworks_ai_embedding_models") + | _provider_set("nebius_embedding_models") + | _provider_set("sambanova_embedding_models") + | _provider_set("ovhcloud_embedding_models") ) ####### IMAGE GENERATION MODELS ################### @@ -2429,6 +1884,12 @@ if os.getenv("LITELLM_DISABLE_LAZY_LOADING", "").lower() in ("1", "true", "yes", def __getattr__(name: str) -> Any: + if name in _LEGACY_PROVIDER_SETS: + return _provider_set(name) + return _lazy_getattr(name) + + +def _lazy_getattr(name: str) -> Any: """Lazy import handler with cached registry for improved performance.""" global _async_client_cleanup_registered # Register async client cleanup on first access (only once) diff --git a/litellm/_catalog/provider_sets.py b/litellm/_catalog/provider_sets.py new file mode 100644 index 00000000000..bd5a761843f --- /dev/null +++ b/litellm/_catalog/provider_sets.py @@ -0,0 +1,413 @@ +"""The provider model sets, derived from the live map on read (decision 18). The only place that knows provider names. +``LEGACY_NAMES`` names each of main's module-level sets and the raw ``litellm_provider`` strings it holds, ``FILTERS`` +and ``BARE_KEYS`` carry the rules main's branch chain applied, and ``ALIASES`` is main's ``models_by_provider``""" + +import datetime +import re +import sys +from collections.abc import Callable, Iterator, Mapping +from types import MappingProxyType +from typing import Final + +from litellm._catalog import CATALOG + +LEGACY_NAMES: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType( + { + "open_ai_chat_completion_models": ("openai",), + "open_ai_text_completion_models": ("text-completion-openai",), + "cohere_models": ("cohere",), + "cohere_chat_models": ("cohere_chat",), + "mistral_chat_models": ("mistral",), + "text_completion_codestral_models": ("text-completion-codestral",), + "text_completion_inception_models": ("text-completion-inception",), + "anthropic_models": ("anthropic",), + "openrouter_models": ("openrouter",), + "datarobot_models": ("datarobot",), + "vertex_language_models": ("vertex_ai-language-models",), + "vertex_vision_models": ("vertex_ai-vision-models",), + "vertex_chat_models": ("vertex_ai-chat-models",), + "vertex_code_chat_models": ("vertex_ai-code-chat-models",), + "vertex_ai_image_models": ("vertex_ai-image-models",), + "vertex_ai_video_models": ("vertex_ai-video-models",), + "vertex_text_models": ("vertex_ai-text-models",), + "vertex_code_text_models": ("vertex_ai-code-text-models",), + "vertex_embedding_models": ("vertex_ai-embedding-models",), + "vertex_anthropic_models": ("vertex_ai-anthropic_models",), + "vertex_llama3_models": ("vertex_ai-llama_models",), + "vertex_deepseek_models": ("vertex_ai-deepseek_models",), + "vertex_ai_ai21_models": ("vertex_ai-ai21_models",), + "vertex_mistral_models": ("vertex_ai-mistral_models",), + "vertex_openai_models": ("vertex_ai-openai_models",), + "vertex_minimax_models": ("vertex_ai-minimax_models",), + "vertex_moonshot_models": ("vertex_ai-moonshot_models",), + "vertex_zai_models": ("vertex_ai-zai_models",), + "ai21_models": ("ai21",), + "ai21_chat_models": ("ai21",), + "nlp_cloud_models": ("nlp_cloud",), + "aleph_alpha_models": ("aleph_alpha",), + "bedrock_models": ("bedrock",), + "bedrock_converse_models": ("bedrock_converse",), + "fal_ai_models": ("fal_ai",), + "fireworks_ai_models": ("fireworks_ai",), + "baseten_models": ("baseten",), + "fireworks_ai_embedding_models": ("fireworks_ai-embedding-models",), + "deepinfra_models": ("deepinfra",), + "perplexity_models": ("perplexity",), + "watsonx_models": ("watsonx",), + "gemini_models": ("gemini",), + "xai_models": ("xai",), + "zai_models": ("zai",), + "deepseek_models": ("deepseek",), + "tencent_models": ("tencent",), + "runwayml_models": ("runwayml",), + "azure_ai_models": ("azure_ai",), + "jina_ai_models": ("jina_ai",), + "voyage_models": ("voyage",), + "infinity_models": ("infinity",), + "heroku_models": ("heroku",), + "databricks_models": ("databricks",), + "cloudflare_models": ("cloudflare",), + "codestral_models": ("codestral",), + "friendliai_models": ("friendliai",), + "featherless_ai_models": ("featherless_ai",), + "palm_models": ("palm",), + "groq_models": ("groq",), + "azure_models": ("azure",), + "azure_anthropic_models": ("azure_anthropic",), + "azure_text_models": ("azure_text",), + "anyscale_models": ("anyscale",), + "cerebras_models": ("cerebras",), + "nadir_models": ("nadir",), + "galadriel_models": ("galadriel",), + "nvidia_nim_models": ("nvidia_nim",), + "nvidia_riva_models": ("nvidia_riva",), + "soniox_models": ("soniox",), + "sambanova_models": ("sambanova",), + "scaledown_models": ("scaledown",), + "sambanova_embedding_models": ("sambanova-embedding-models",), + "novita_models": ("novita",), + "assemblyai_models": ("assemblyai",), + "snowflake_models": ("snowflake",), + "gradient_ai_models": ("gradient_ai",), + "llama_models": ("meta_llama",), + "nscale_models": ("nscale",), + "nebius_models": ("nebius-chat-models",), + "nebius_embedding_models": ("nebius-embedding-models",), + "aiml_models": ("aiml",), + "deepgram_models": ("deepgram",), + "elevenlabs_models": ("elevenlabs",), + "dashscope_models": ("dashscope",), + "qwencloud_models": ("qwencloud",), + "qwen_ai_platform_models": ("qwen_ai_platform",), + "moonshot_models": ("moonshot",), + "publicai_models": ("publicai",), + "darkbloom_models": ("darkbloom",), + "openinfer_models": ("openinfer",), + "v0_models": ("v0",), + "morph_models": ("morph",), + "opencode_go_models": ("opencode_go",), + "lambda_ai_models": ("lambda_ai",), + "inception_models": ("inception",), + "hyperbolic_models": ("hyperbolic",), + "black_forest_labs_models": ("black_forest_labs",), + "recraft_models": ("recraft",), + "cometapi_models": ("cometapi",), + "oci_models": ("oci",), + "vercel_ai_gateway_models": ("vercel_ai_gateway",), + "edenai_models": ("edenai",), + "volcengine_models": ("volcengine",), + "wandb_models": ("wandb",), + "ovhcloud_models": ("ovhcloud",), + "ovhcloud_embedding_models": ("ovhcloud-embedding-models",), + "lemonade_models": ("lemonade",), + "docker_model_runner_models": ("docker_model_runner",), + "amazon_nova_models": ("amazon_nova",), + "stability_models": ("stability",), + "github_copilot_models": ("github_copilot",), + "microsoft_365_copilot_models": ("microsoft_365_copilot",), + "chatgpt_models": ("chatgpt",), + "minimax_models": ("minimax",), + "aws_polly_models": ("aws_polly",), + "transcribe_models": ("transcribe",), + "gigachat_models": ("gigachat",), + "llamagate_models": ("llamagate",), + "coralbricks_models": ("coralbricks",), + "reducto_models": ("reducto",), + "bedrock_mantle_models": ("bedrock_mantle",), + } +) + +_BEDROCK_PRICING_ONLY: Final = re.compile(r"^bedrock/[a-zA-Z0-9_-]+/.+$") + + +def is_bedrock_pricing_only_model(key: str) -> bool: + """Keys of the pattern ``bedrock//`` and commitment keys, priced for lookup only""" + return "month-commitment" in key or _BEDROCK_PRICING_ONLY.match(key) is not None + + +def is_openai_finetune_model(key: str) -> bool: + """``ft:`` keys, priced for lookup only""" + return key.startswith("ft:") and not key.count(":") > 1 + + +def _is_past_deprecation_date(row: Mapping[str, object]) -> bool: + deprecation_date: Final = row.get("deprecation_date") + if not isinstance(deprecation_date, str): + return False + try: + return datetime.date.fromisoformat(deprecation_date) <= datetime.datetime.now(datetime.timezone.utc).astimezone().date() + except ValueError: + return False + + +FILTERS: Final[Mapping[str, Callable[[str, Mapping[str, object]], bool]]] = MappingProxyType( + { + "open_ai_chat_completion_models": lambda key, row: not is_openai_finetune_model(key), + "ai21_chat_models": lambda key, row: row.get("mode") == "chat", + "ai21_models": lambda key, row: row.get("mode") != "chat", + "bedrock_models": lambda key, row: row.get("mode") != "guardrail" and not is_bedrock_pricing_only_model(key), + "fireworks_ai_models": lambda key, row: "-to-" not in key and "fireworks-ai-default" not in key, + "fireworks_ai_embedding_models": lambda key, row: "-to-" not in key, + "baseten_models": lambda key, row: not _is_past_deprecation_date(row), + } +) + +BARE_KEYS: Final = frozenset( + { + "vertex_ai_ai21_models", + "vertex_ai_image_models", + "vertex_ai_video_models", + "vertex_anthropic_models", + "vertex_deepseek_models", + "vertex_llama3_models", + "vertex_minimax_models", + "vertex_mistral_models", + "vertex_moonshot_models", + "vertex_openai_models", + "vertex_zai_models", + } +) +"""Sets main filled with the ``vertex_ai/`` prefix stripped from each key""" + +STATIC: Final = frozenset( + { + "clarifai_models", + "huggingface_models", + "maritalk_models", + "ollama_models", + "petals_models", + "replicate_models", + "together_ai_models", + } +) +"""Hand lists of models the map has no rows for, read from ``litellm`` as they are""" + +ALIASES: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType( + { + "openai": ("open_ai_chat_completion_models", "open_ai_text_completion_models"), + "text-completion-openai": ("open_ai_text_completion_models",), + "cohere": ("cohere_models", "cohere_chat_models"), + "cohere_chat": ("cohere_chat_models",), + "anthropic": ("anthropic_models",), + "replicate": ("replicate_models",), + "huggingface": ("huggingface_models",), + "together_ai": ("together_ai_models",), + "baseten": ("baseten_models",), + "openrouter": ("openrouter_models",), + "vercel_ai_gateway": ("vercel_ai_gateway_models",), + "edenai": ("edenai_models",), + "datarobot": ("datarobot_models",), + "vertex_ai": ( + "vertex_chat_models", + "vertex_code_chat_models", + "vertex_text_models", + "vertex_code_text_models", + "vertex_anthropic_models", + "vertex_vision_models", + "vertex_language_models", + "vertex_deepseek_models", + "vertex_minimax_models", + "vertex_moonshot_models", + "vertex_zai_models", + ), + "ai21": ("ai21_models", "ai21_chat_models"), + "bedrock": ("bedrock_models", "bedrock_converse_models"), + "petals": ("petals_models",), + "ollama": ("ollama_models",), + "ollama_chat": ("ollama_models",), + "deepinfra": ("deepinfra_models",), + "perplexity": ("perplexity_models",), + "maritalk": ("maritalk_models",), + "watsonx": ("watsonx_models",), + "gemini": ("gemini_models",), + "fireworks_ai": ("fireworks_ai_models", "fireworks_ai_embedding_models"), + "aleph_alpha": ("aleph_alpha_models",), + "text-completion-codestral": ("text_completion_codestral_models",), + "text-completion-inception": ("text_completion_inception_models",), + "xai": ("xai_models",), + "zai": ("zai_models",), + "fal_ai": ("fal_ai_models",), + "deepseek": ("deepseek_models",), + "tencent": ("tencent_models",), + "runwayml": ("runwayml_models",), + "mistral": ("mistral_chat_models",), + "azure_ai": ("azure_ai_models",), + "voyage": ("voyage_models",), + "infinity": ("infinity_models",), + "databricks": ("databricks_models",), + "cloudflare": ("cloudflare_models",), + "codestral": ("codestral_models",), + "nlp_cloud": ("nlp_cloud_models",), + "friendliai": ("friendliai_models",), + "palm": ("palm_models",), + "groq": ("groq_models",), + "azure": ("azure_models", "azure_text_models"), + "azure_anthropic": ("azure_anthropic_models",), + "azure_text": ("azure_text_models",), + "anyscale": ("anyscale_models",), + "cerebras": ("cerebras_models",), + "nadir": ("nadir_models",), + "galadriel": ("galadriel_models",), + "nvidia_nim": ("nvidia_nim_models",), + "nvidia_riva": ("nvidia_riva_models",), + "soniox": ("soniox_models",), + "sambanova": ("sambanova_models", "sambanova_embedding_models"), + "scaledown": ("scaledown_models",), + "novita": ("novita_models",), + "nebius": ("nebius_models", "nebius_embedding_models"), + "aiml": ("aiml_models",), + "assemblyai": ("assemblyai_models",), + "jina_ai": ("jina_ai_models",), + "snowflake": ("snowflake_models",), + "gradient_ai": ("gradient_ai_models",), + "meta_llama": ("llama_models",), + "nscale": ("nscale_models",), + "featherless_ai": ("featherless_ai_models",), + "deepgram": ("deepgram_models",), + "elevenlabs": ("elevenlabs_models",), + "heroku": ("heroku_models",), + "dashscope": ("dashscope_models",), + "qwencloud": ("qwencloud_models",), + "qwen_ai_platform": ("qwen_ai_platform_models",), + "modelscope": ("modelscope_models",), + "moonshot": ("moonshot_models",), + "publicai": ("publicai_models",), + "darkbloom": ("darkbloom_models",), + "openinfer": ("openinfer_models",), + "v0": ("v0_models",), + "morph": ("morph_models",), + "opencode_go": ("opencode_go_models",), + "lambda_ai": ("lambda_ai_models",), + "inception": ("inception_models",), + "hyperbolic": ("hyperbolic_models",), + "black_forest_labs": ("black_forest_labs_models",), + "recraft": ("recraft_models",), + "cometapi": ("cometapi_models",), + "oci": ("oci_models",), + "volcengine": ("volcengine_models",), + "wandb": ("wandb_models",), + "ovhcloud": ("ovhcloud_models", "ovhcloud_embedding_models"), + "lemonade": ("lemonade_models",), + "clarifai": ("clarifai_models",), + "amazon_nova": ("amazon_nova_models",), + "stability": ("stability_models",), + "github_copilot": ("github_copilot_models",), + "microsoft_365_copilot": ("microsoft_365_copilot_models",), + "chatgpt": ("chatgpt_models",), + "minimax": ("minimax_models",), + "aws_polly": ("aws_polly_models",), + "transcribe": ("transcribe_models",), + "gigachat": ("gigachat_models",), + "llamagate": ("llamagate_models",), + "coralbricks": ("coralbricks_models",), + "reducto": ("reducto_models",), + "bedrock_mantle": ("bedrock_mantle_models",), + } +) +"""main's ``models_by_provider``, each provider the union of the named sets""" + +_SEEDS: Final[Mapping[str, str]] = MappingProxyType( + {"bedrock_converse_models": "BEDROCK_CONVERSE_MODELS", "wandb_models": "WANDB_MODELS"} +) + + +OPAQUE_KEYS: Final[set[str]] = set() # mutable-ok: register_model adds router deployment ids, never removed +"""Keys registered as opaque ids (``add_to_provider_model_sets=False``), which no provider set lists""" + + +def _litellm() -> object: + return sys.modules["litellm"] + + +def _rows() -> Mapping[str, Mapping[str, object]]: + import litellm + + return litellm.model_cost + + +def _derived(name: str) -> frozenset[str]: + rows: Final = _rows() + keep: Final = FILTERS.get(name, lambda key, row: True) + bare: Final = name in BARE_KEYS + keys: Final = ( + key + for provider in LEGACY_NAMES[name] + for key in CATALOG.models_for(provider) # comprehension-ok: a set's providers, then each provider's keys + if keep(key, rows.get(key) or {}) and key not in OPAQUE_KEYS + ) + seed: Final = _SEEDS.get(name) + seeded: Final = frozenset(getattr(_litellm(), seed)) if seed is not None else frozenset() + return frozenset(key.replace("vertex_ai/", "") if bare else key for key in keys) | seeded + + +def legacy_set(name: str) -> frozenset[str]: + """One of main's 121 module-level provider sets, as the live map has it now. Read only""" + return _derived(name) + + +def contains(name: str, model: str) -> bool: + """``model in litellm.`` without building the set, for the request path. The same rows, filters, bare keys + and seeds as ``legacy_set``. A set assigned to ``litellm.`` answers instead""" + assigned: Final = vars(_litellm()).get(name) + if assigned is not None: + return model in assigned + rows: Final = _rows() + keep: Final = FILTERS.get(name, lambda key, row: True) + bare: Final = name in BARE_KEYS + if bare and "vertex_ai/" in model: + return False + keys: Final = (model, f"vertex_ai/{model}") if bare else (model,) + in_map: Final = any( + isinstance(row := rows.get(key), Mapping) + and row.get("litellm_provider") in LEGACY_NAMES[name] + and key not in OPAQUE_KEYS + and keep(key, row) + for key in keys + ) + seed: Final = _SEEDS.get(name) + return in_map or (seed is not None and model in getattr(_litellm(), seed)) + + +def models_for(provider: str) -> frozenset[str]: + """The models of ``provider`` as ``models_by_provider`` lists them""" + names: Final = ALIASES.get(provider) + if names is None: + return CATALOG.models_for(provider) - OPAQUE_KEYS + return frozenset().union( + *(legacy_set(name) if name in LEGACY_NAMES else frozenset(getattr(_litellm(), name)) for name in names) + ) + + +class ModelsByProvider(Mapping[str, frozenset[str]]): + """``litellm.models_by_provider``, read only. Main's providers, then every raw provider string in the map""" + + def __getitem__(self, provider: str) -> frozenset[str]: + if provider in ALIASES or provider in CATALOG.providers(): + return models_for(provider) + raise KeyError(provider) + + def __iter__(self) -> Iterator[str]: + return iter((*ALIASES, *(provider for provider in CATALOG.providers() if provider not in ALIASES))) + + def __len__(self) -> int: + return len(ALIASES) + sum(1 for provider in CATALOG.providers() if provider not in ALIASES) diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index 145d27c1e62..f0e2930c921 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -160,11 +160,9 @@ def _install( def _refresh_derived_state() -> None: - import litellm from litellm import utils utils._invalidate_model_cost_lowercase_map() # pyright: ignore[reportPrivateUsage] # required cache invalidation - litellm.add_known_models(model_cost_map=litellm.model_cost) def reload_model_cost_map(source: Source, catalog: Catalog = CATALOG) -> Installed | Unavailable: diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index e8e4ec6fc7c..93fb5e50e97 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -248,6 +248,8 @@ class Catalog: def resolve(self, model: str, provider: str | None = None) -> Resolved | Missing | Invalid: ... def lookup_key(self, name: str) -> str | None: ... def provider_keys(self, provider: str) -> list[str]: ... + def models_for(self, provider: str) -> frozenset[str]: ... + def providers(self) -> list[str]: ... def mark_index_stale(self) -> None: ... def register( self, rows: Mapping[str, Mapping[str, object]], *, persist: bool, provider: str | None = None diff --git a/litellm/utils.py b/litellm/utils.py index c8f36564cf4..585a0b65c41 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -51,6 +51,7 @@ import litellm.litellm_core_utils # audio_utils.utils is lazy-loaded - only imported when needed for transcription calls import litellm.litellm_core_utils.json_validation_rule +from litellm._catalog.provider_sets import contains as _in_provider_set from litellm._internal_context import is_internal_call from litellm._lazy_imports import ( _get_modified_max_tokens, @@ -2635,7 +2636,7 @@ def _return_openai_tokenizer(model: str) -> SelectTokenizerResponse: def uses_anthropic_tokenizer(model: str) -> bool: - return model in litellm.anthropic_models and "claude-3" not in model + return _in_provider_set("anthropic_models", model) and "claude-3" not in model HuggingFaceTokenizerKind = Literal["cohere", "anthropic", "deepseek", "llama2", "llama3"] @@ -2651,7 +2652,7 @@ _BUNDLED_TOKENIZER_LOAD_LOCKS: Final[Mapping[HuggingFaceTokenizerKind, threading def huggingface_tokenizer_kind(model: str) -> HuggingFaceTokenizerKind | None: """Which HuggingFace tokenizer `token_counter` selects for a model; `None` means tiktoken.""" - if model in litellm.cohere_models and "command-r" in model: + if _in_provider_set("cohere_models", model) and "command-r" in model: return "cohere" if uses_anthropic_tokenizer(model): return "anthropic" @@ -3656,9 +3657,10 @@ def register_model( that provider, so a deployment id that happens to equal another provider's catalog key stays its own provider-less entry instead of merging into it. - ``add_to_provider_model_sets`` adds the keys to the per-provider model sets - behind provider resolution and wildcard model listing. Pass False when the - keys are opaque ids rather than model names (e.g. router deployment ids). + The provider model sets are derived from ``litellm.model_cost`` on read + (decision 18). Pass ``add_to_provider_model_sets=False`` when the keys are + opaque ids rather than model names (e.g. router deployment ids), so no set + lists them. """ loaded_model_cost = {} @@ -3685,12 +3687,12 @@ def register_model( "register_model: model=%s has custom pricing but not in built-in cost map and no prefix/region variant matched; cache_creation_input_token_cost and cache_read_input_token_cost will default to 0 for this model (input/output cost tracking is unaffected). To track cache cost, add them to model_info", warning_display_name or key, ) - for key, value in loaded_model_cost.items(): + for key in loaded_model_cost: verbose_logger.debug("added/updated model=%s in litellm.model_cost", key) - if add_to_provider_model_sets and value.get("litellm_provider") == "openrouter": - litellm.openrouter_models.add(str(key).split("/", 1)[-1]) - if add_to_provider_model_sets: - litellm.add_known_models(loaded_model_cost) + if not add_to_provider_model_sets: + from litellm._catalog.provider_sets import OPAQUE_KEYS + + OPAQUE_KEYS.update(str(key) for key in loaded_model_cost) return model_cost @@ -4864,12 +4866,12 @@ def get_optional_params( drop_params=bool(drop_params), ) elif custom_llm_provider == "vertex_ai" and ( - model in litellm.vertex_chat_models - or model in litellm.vertex_code_chat_models - or model in litellm.vertex_text_models - or model in litellm.vertex_code_text_models - or model in litellm.vertex_language_models - or model in litellm.vertex_vision_models + _in_provider_set("vertex_chat_models", model) + or _in_provider_set("vertex_code_chat_models", model) + or _in_provider_set("vertex_text_models", model) + or _in_provider_set("vertex_code_text_models", model) + or _in_provider_set("vertex_language_models", model) + or _in_provider_set("vertex_vision_models", model) ): optional_params = litellm.VertexGeminiConfig().map_openai_params( non_default_params=non_default_params, @@ -4900,7 +4902,7 @@ def get_optional_params( drop_params=bool(drop_params), ) elif custom_llm_provider == "vertex_ai": - if model in litellm.vertex_mistral_models: + if _in_provider_set("vertex_mistral_models", model): if "codestral" in model: optional_params = litellm.CodestralTextCompletionConfig().map_openai_params( model=model, @@ -4915,7 +4917,7 @@ def get_optional_params( optional_params=optional_params, drop_params=bool(drop_params), ) - elif model in litellm.vertex_ai_ai21_models: + elif _in_provider_set("vertex_ai_ai21_models", model): optional_params = litellm.VertexAIAi21Config().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, @@ -6109,10 +6111,10 @@ def get_model_info_helper( if custom_llm_provider is not None and custom_llm_provider == "vertex_ai_beta": custom_llm_provider = "vertex_ai" if custom_llm_provider is not None and custom_llm_provider == "vertex_ai": - if "meta/" + model in litellm.vertex_llama3_models: + if _in_provider_set("vertex_llama3_models", "meta/" + model): model = "meta/" + model - elif ( - model + "@latest" in litellm.vertex_mistral_models or model + "@latest" in litellm.vertex_ai_ai21_models + elif _in_provider_set("vertex_mistral_models", model + "@latest") or _in_provider_set( + "vertex_ai_ai21_models", model + "@latest" ): model = model + "@latest" ########################## @@ -6845,8 +6847,8 @@ def validate_environment( else: ## openai - chatcompletion + text completion if ( - model in litellm.open_ai_chat_completion_models - or model in litellm.open_ai_text_completion_models + _in_provider_set("open_ai_chat_completion_models", model) + or _in_provider_set("open_ai_text_completion_models", model) or model in litellm.open_ai_embedding_models or model in litellm.openai_image_generation_models or model.startswith("gpt-image") @@ -6856,13 +6858,13 @@ def validate_environment( else: missing_keys.append("OPENAI_API_KEY") ## anthropic - elif model in litellm.anthropic_models: + elif _in_provider_set("anthropic_models", model): if "ANTHROPIC_API_KEY" in os.environ or "ANTHROPIC_AUTH_TOKEN" in os.environ: keys_in_environment = True else: missing_keys.append("ANTHROPIC_API_KEY") ## cohere - elif model in litellm.cohere_models: + elif _in_provider_set("cohere_models", model): if "COHERE_API_KEY" in os.environ: keys_in_environment = True else: @@ -6874,33 +6876,33 @@ def validate_environment( else: missing_keys.append("REPLICATE_API_KEY") ## openrouter - elif model in litellm.openrouter_models: + elif _in_provider_set("openrouter_models", model): if "OPENROUTER_API_KEY" in os.environ: keys_in_environment = True else: missing_keys.append("OPENROUTER_API_KEY") ## vercel_ai_gateway - elif model in litellm.vercel_ai_gateway_models: + elif _in_provider_set("vercel_ai_gateway_models", model): if "VERCEL_AI_GATEWAY_API_KEY" in os.environ: keys_in_environment = True else: missing_keys.append("VERCEL_AI_GATEWAY_API_KEY") ## edenai - elif model in litellm.edenai_models: + elif _in_provider_set("edenai_models", model): if "EDENAI_API_KEY" in os.environ: keys_in_environment = True else: missing_keys.append("EDENAI_API_KEY") ## datarobot - elif model in litellm.datarobot_models: + elif _in_provider_set("datarobot_models", model): if "DATAROBOT_API_TOKEN" in os.environ: keys_in_environment = True else: missing_keys.append("DATAROBOT_API_TOKEN") ## vertex - text + chat models elif ( - model in litellm.vertex_chat_models - or model in litellm.vertex_text_models + _in_provider_set("vertex_chat_models", model) + or _in_provider_set("vertex_text_models", model) or model in litellm.models_by_provider["vertex_ai"] ): if "VERTEXAI_PROJECT" in os.environ and "VERTEXAI_LOCATION" in os.environ: @@ -6914,7 +6916,7 @@ def validate_environment( else: missing_keys.append("HUGGINGFACE_API_KEY") ## ai21 - elif model in litellm.ai21_models: + elif _in_provider_set("ai21_models", model): if "AI21_API_KEY" in os.environ: keys_in_environment = True else: @@ -6926,34 +6928,34 @@ def validate_environment( else: missing_keys.append("TOGETHERAI_API_KEY") ## aleph_alpha - elif model in litellm.aleph_alpha_models: + elif _in_provider_set("aleph_alpha_models", model): if "ALEPH_ALPHA_API_KEY" in os.environ: keys_in_environment = True else: missing_keys.append("ALEPH_ALPHA_API_KEY") ## baseten - elif model in litellm.baseten_models: + elif _in_provider_set("baseten_models", model): if "BASETEN_API_KEY" in os.environ: keys_in_environment = True else: missing_keys.append("BASETEN_API_KEY") ## nlp_cloud - elif model in litellm.nlp_cloud_models: + elif _in_provider_set("nlp_cloud_models", model): if "NLP_CLOUD_API_KEY" in os.environ: keys_in_environment = True else: missing_keys.append("NLP_CLOUD_API_KEY") - elif model in litellm.novita_models: + elif _in_provider_set("novita_models", model): if "NOVITA_API_KEY" in os.environ: keys_in_environment = True else: missing_keys.append("NOVITA_API_KEY") - elif model in litellm.nebius_models: + elif _in_provider_set("nebius_models", model): if "NEBIUS_API_KEY" in os.environ: keys_in_environment = True else: missing_keys.append("NEBIUS_API_KEY") - elif model in litellm.wandb_models: + elif _in_provider_set("wandb_models", model): if "WANDB_API_KEY" in os.environ: keys_in_environment = True else: @@ -6985,7 +6987,9 @@ def acreate(*args, **kwargs): ## Thin client to handle the acreate langchain ca def valid_model(model): try: # for a given model name, check if the user has the right permissions to access the model - if model in litellm.open_ai_chat_completion_models or model in litellm.open_ai_text_completion_models: + if _in_provider_set("open_ai_chat_completion_models", model) or _in_provider_set( + "open_ai_text_completion_models", model + ): openai.models.retrieve(model) else: messages: Final = [{"role": "user", "content": "Hello World"}] @@ -8494,11 +8498,11 @@ class ProviderConfigManager: ) return VertexAIGPTOSSTransformation() - elif model in litellm.vertex_mistral_models: + elif _in_provider_set("vertex_mistral_models", model): if "codestral" in model: return litellm.CodestralTextCompletionConfig() return litellm.VertexAIMistralConfig() - elif model in litellm.vertex_ai_ai21_models: + elif _in_provider_set("vertex_ai_ai21_models", model): return litellm.VertexAIAi21Config() else: return litellm.VertexAILlama3Config() diff --git a/tests/unit/_catalog/test_provider_sets.py b/tests/unit/_catalog/test_provider_sets.py new file mode 100644 index 00000000000..1a72a4d610c --- /dev/null +++ b/tests/unit/_catalog/test_provider_sets.py @@ -0,0 +1,70 @@ +from typing import Final + +import pytest + +import litellm +from litellm._catalog import CATALOG +from litellm._catalog.provider_sets import models_for + + +@pytest.fixture +def live_map(monkeypatch: pytest.MonkeyPatch) -> dict[str, dict[str, object]]: + rows: Final = dict(CATALOG.bundled().served()) + monkeypatch.setattr(litellm, "model_cost", rows) + return rows + + +def test_a_row_added_dropped_or_moved_shows_on_the_next_read(live_map: dict[str, dict[str, object]]) -> None: + """provider-sets-live. On main a moved key stayed in its old set and a direct write reached none""" + live_map["acme/new-chat"] = {"litellm_provider": "cohere", "mode": "chat"} + assert "acme/new-chat" in litellm.cohere_models + assert "acme/new-chat" in litellm.models_by_provider["cohere"] + + live_map["acme/new-chat"] = {"litellm_provider": "mistral", "mode": "chat"} + live_map["acme/len-change"] = {} + assert "acme/new-chat" not in litellm.cohere_models + assert "acme/new-chat" in litellm.mistral_chat_models + + del live_map["acme/new-chat"] + assert "acme/new-chat" not in litellm.models_by_provider["mistral"] + + +def test_the_provider_sets_are_read_only() -> None: + """provider-sets-readonly""" + with pytest.raises(AttributeError): + litellm.anthropic_models.add("x") # pyright: ignore[reportAttributeAccessIssue] # the mutation under test + with pytest.raises(TypeError): + litellm.models_by_provider["openai"] = frozenset() # pyright: ignore[reportIndexIssue] # the mutation under test + + +@pytest.mark.usefixtures("live_map") +def test_every_provider_in_the_map_lists_the_rows_that_name_it() -> None: + providers: Final = frozenset( + row["litellm_provider"] for row in litellm.model_cost.values() if isinstance(row.get("litellm_provider"), str) + ) + for provider in providers: + assert provider in litellm.models_by_provider + assert "together_ai" in litellm.models_by_provider + assert all(litellm.model_cost[key]["litellm_provider"] == "aihubmix" for key in models_for("aihubmix")) + + +def test_registering_an_openrouter_model_adds_no_stripped_name(live_map: dict[str, dict[str, object]]) -> None: + litellm.register_model({"openrouter/acme/model": {"litellm_provider": "openrouter", "mode": "chat"}}) + + assert "openrouter/acme/model" in litellm.openrouter_models + assert "acme/model" not in litellm.openrouter_models + + +@pytest.mark.usefixtures("live_map") +def test_the_request_path_membership_check_agrees_with_every_derived_set() -> None: + from litellm._catalog.provider_sets import LEGACY_NAMES, contains, legacy_set + + names: Final = frozenset(litellm.model_cost) | {key.replace("vertex_ai/", "") for key in litellm.model_cost} + disagreements: Final = [ + (set_name, model) + for set_name in LEGACY_NAMES + for model in names # comprehension-ok: every set against every candidate name + if contains(set_name, model) != (model in legacy_set(set_name)) + ] + + assert disagreements == [] diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 2879911ea6b..a7a033df7e0 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -44,18 +44,27 @@ os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" import litellm # noqa: E402 # litellm reads LITELLM_LOCAL_MODEL_COST_MAP at import import litellm.router as litellm_router_module # noqa: E402 # same import-time dependency import litellm.utils as litellm_utils_module # noqa: E402 # same import-time dependency +from litellm._catalog.provider_sets import LEGACY_NAMES as LEGACY_PROVIDER_SETS # noqa: E402 # after litellm from litellm._logging import ALL_LOGGERS # noqa: E402 # same import-time dependency +from litellm.anthropic_beta_headers_manager import ( + reload_beta_headers_config, # noqa: E402 # same import-time dependency +) from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER # noqa: E402 # same import-time dependency -from litellm.anthropic_beta_headers_manager import reload_beta_headers_config # noqa: E402 # same import-time dependency -from litellm.litellm_core_utils.prompt_templates import factory as prompt_factory_module # noqa: E402 # same import-time dependency +from litellm.litellm_core_utils.prompt_templates import ( + factory as prompt_factory_module, # noqa: E402 # same import-time dependency +) from litellm.litellm_core_utils.prompt_templates import ( # noqa: E402 # same import-time dependency image_handling as image_handling_module, ) -from litellm.llms.gemini.chat import transformation as gemini_chat_transformation_module # noqa: E402 # same import-time dependency from litellm.llms.custom_httpx.async_client_cleanup import ( # noqa: E402 # same import-time dependency close_litellm_async_clients, ) -from litellm.proxy.db import tool_registry_writer as tool_registry_writer_module # noqa: E402 # same import-time dependency +from litellm.llms.gemini.chat import ( + transformation as gemini_chat_transformation_module, # noqa: E402 # same import-time dependency +) +from litellm.proxy.db import ( + tool_registry_writer as tool_registry_writer_module, # noqa: E402 # same import-time dependency +) LOOPBACK_HOSTS: Final = ["127.0.0.1", "::1", "localhost"] AMBIENT_AZURE_CREDENTIAL_ENV_VARS: Final = ( @@ -124,8 +133,6 @@ RESTORED_GLOBALS: Final = ( "cost_discount_config", "disable_hf_tokenizer_download", "disable_copilot_system_to_assistant", - "cohere_models", - "anthropic_models", "token_counter", "initialized_langfuse_clients", ) @@ -309,26 +316,15 @@ def isolate_router_model_cost_state() -> Iterator[None]: litellm.get_model_info.cache_clear() -def _provider_model_sets() -> tuple[set[str], ...]: - module_globals: Final = tuple(vars(litellm).items()) - return tuple(models for name, models in module_globals if name.endswith("_models") and isinstance(models, set)) - - @pytest.fixture(autouse=True) def isolate_model_catalog() -> Iterator[None]: model_cost: Final = litellm.model_cost model_cost_entries: Final = dict(model_cost) - models_by_provider: Final = litellm.models_by_provider - models_by_provider_entries: Final = dict(models_by_provider) - provider_model_sets: Final = tuple((models, frozenset(models)) for models in _provider_model_sets()) yield model_cost.clear() model_cost.update(model_cost_entries) - models_by_provider.clear() - models_by_provider.update(models_by_provider_entries) - for models, entries in provider_model_sets: - models.clear() - models.update(entries) + for name in LEGACY_PROVIDER_SETS: + vars(litellm).pop(name, None) litellm_utils_module._invalidate_model_cost_lowercase_map() litellm.get_model_info.cache_clear() diff --git a/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py b/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py index 0d15890c907..58c39c368f1 100644 --- a/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py +++ b/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py @@ -264,10 +264,8 @@ def test_get_llm_provider_deepseek_custom_api_base(): @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_vertex_ai_image_models(monkeypatch): - monkeypatch.setattr(litellm, "vertex_ai_image_models", set()) - monkeypatch.setattr(litellm, "models_by_provider", dict(litellm.models_by_provider)) - litellm.add_known_models( - model_cost_map={ + litellm.model_cost.update( + { "vertex_ai/imagegeneration@006": { "litellm_provider": "vertex_ai-image-models", "mode": "image_generation", diff --git a/tests/unit/llms/azure/chat/test_azure_gpt_reasoning_transformation.py b/tests/unit/llms/azure/chat/test_azure_gpt_reasoning_transformation.py index 43dbb7e2134..da535f43745 100644 --- a/tests/unit/llms/azure/chat/test_azure_gpt_reasoning_transformation.py +++ b/tests/unit/llms/azure/chat/test_azure_gpt_reasoning_transformation.py @@ -19,7 +19,7 @@ def use_local_model_cost_map(monkeypatch: pytest.MonkeyPatch): import path fetches the published map, which lags a key added in this repo.""" monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url="")) - litellm.add_known_models(model_cost_map=litellm.model_cost) + litellm.model_cost.update(litellm.model_cost) def test_azure_gpt5_supports_reasoning_effort(config: AzureOpenAIGPT5Config): diff --git a/tests/unit/llms/azure/response/test_azure_transformation.py b/tests/unit/llms/azure/response/test_azure_transformation.py index 8b8990aa5b8..1b474bd8554 100644 --- a/tests/unit/llms/azure/response/test_azure_transformation.py +++ b/tests/unit/llms/azure/response/test_azure_transformation.py @@ -788,7 +788,7 @@ def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: """Pin the bundled cost map: the published map lags a key added in this repo.""" monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url="")) - litellm.add_known_models(model_cost_map=litellm.model_cost) + litellm.model_cost.update(litellm.model_cost) def test_azure_responses_reasoning_effort_none_unlocks_temperature_off_the_azure_row( diff --git a/tests/unit/llms/baseten/chat/test_baseten_completions.py b/tests/unit/llms/baseten/chat/test_baseten_completions.py index 4f135a1a106..470873d46d9 100644 --- a/tests/unit/llms/baseten/chat/test_baseten_completions.py +++ b/tests/unit/llms/baseten/chat/test_baseten_completions.py @@ -7,6 +7,7 @@ import pytest import respx import litellm +from litellm._catalog.provider_sets import OPAQUE_KEYS from litellm.llms.baseten.chat import BasetenConfig from litellm.utils import get_valid_models @@ -66,7 +67,7 @@ class TestBasetenModelDiscovery: key for key, value in litellm.model_cost.items() if value.get("litellm_provider") == "baseten" and not litellm._is_past_deprecation_date(value) - } + } - OPAQUE_KEYS assert servable_registry_models assert servable_registry_models <= set(baseten_models) @@ -82,15 +83,14 @@ class TestBasetenModelDiscovery: """A refreshed cost map that marks a listed model retired removes it without a restart.""" model: Final = "baseten/acme/reload-probe" try: - litellm.add_known_models({model: {"litellm_provider": "baseten"}}) + litellm.model_cost.update({model: {"litellm_provider": "baseten"}}) assert model in litellm.models_by_provider["baseten"] - litellm.add_known_models({model: {"litellm_provider": "baseten", "deprecation_date": "2000-01-01"}}) + litellm.model_cost.update({model: {"litellm_provider": "baseten", "deprecation_date": "2000-01-01"}}) assert model not in litellm.models_by_provider["baseten"] finally: - litellm.baseten_models.discard(model) - litellm.add_known_models({}) + litellm.model_cost.update({}) @pytest.mark.parametrize( "value, expected", diff --git a/tests/unit/llms/bedrock/test_cross_region_inference_profile_mapping.py b/tests/unit/llms/bedrock/test_cross_region_inference_profile_mapping.py index bcd1e9d6578..6d235cb8299 100644 --- a/tests/unit/llms/bedrock/test_cross_region_inference_profile_mapping.py +++ b/tests/unit/llms/bedrock/test_cross_region_inference_profile_mapping.py @@ -21,21 +21,11 @@ from litellm.types.utils import ( def local_model_cost_map(monkeypatch): """Resolve models against this checkout's cost map instead of the network-fetched ``main`` copy, which lags this branch until merge.""" - original_converse_models = set(litellm.bedrock_converse_models) monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) litellm.get_model_info.cache_clear() - try: - litellm.bedrock_converse_models.update( - key - for key, value in litellm.model_cost.items() - if isinstance(value, dict) and value.get("litellm_provider") == "bedrock_converse" - ) - yield - finally: - litellm.bedrock_converse_models.clear() - litellm.bedrock_converse_models.update(original_converse_models) - litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() class GptProfile(NamedTuple): diff --git a/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_transformation.py b/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_transformation.py index f252d9886d3..af03ef2fd17 100644 --- a/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_transformation.py +++ b/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_transformation.py @@ -29,17 +29,13 @@ from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe @pytest.fixture def local_cost_map(monkeypatch): original_model_cost = litellm.model_cost - original_bedrock_mantle_models = set(litellm.bedrock_mantle_models) try: monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "true") litellm.model_cost = litellm.get_model_cost_map(url="") litellm.get_model_info.cache_clear() - litellm.add_known_models() yield finally: litellm.model_cost = original_model_cost - litellm.bedrock_mantle_models.clear() - litellm.bedrock_mantle_models.update(original_bedrock_mantle_models) litellm.get_model_info.cache_clear() diff --git a/tests/unit/llms/edenai/chat/test_edenai_chat_transformation.py b/tests/unit/llms/edenai/chat/test_edenai_chat_transformation.py index 739ad144277..d4b886b452b 100644 --- a/tests/unit/llms/edenai/chat/test_edenai_chat_transformation.py +++ b/tests/unit/llms/edenai/chat/test_edenai_chat_transformation.py @@ -164,9 +164,8 @@ class TestRegistration: monkeypatch.setenv("EDENAI_API_KEY", "eden-test-key") present = litellm.validate_environment(model=alias) finally: - litellm.edenai_models.discard(alias) litellm.model_cost.pop(alias, None) - litellm.add_known_models(model_cost_map={}) + litellm.model_cost.update({}) assert (missing["keys_in_environment"], missing["missing_keys"]) == (False, ["EDENAI_API_KEY"]) assert (present["keys_in_environment"], present["missing_keys"]) == (True, []) @@ -178,12 +177,11 @@ class TestRegistration: wildcard = LiteLLM_Params(model="edenai/*", api_key="wildcard-key") assert alias not in (get_provider_models("edenai", wildcard) or []) - litellm.add_known_models(model_cost_map={alias: {"litellm_provider": "edenai", "mode": "chat"}}) + litellm.model_cost.update({alias: {"litellm_provider": "edenai", "mode": "chat"}}) try: expanded = get_provider_models("edenai", wildcard) finally: - litellm.edenai_models.discard(alias) - litellm.add_known_models(model_cost_map={}) + litellm.model_cost.pop(alias, None) assert expanded is not None assert alias in expanded diff --git a/tests/unit/llms/github_copilot/responses/test_github_copilot_responses_transformation.py b/tests/unit/llms/github_copilot/responses/test_github_copilot_responses_transformation.py index e37b9f68f82..212c0adcfb9 100644 --- a/tests/unit/llms/github_copilot/responses/test_github_copilot_responses_transformation.py +++ b/tests/unit/llms/github_copilot/responses/test_github_copilot_responses_transformation.py @@ -27,7 +27,7 @@ def use_local_model_cost_map(monkeypatch: pytest.MonkeyPatch): on remote catalog fetches (and don't change behavior across remote refreshes).""" monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url="")) - litellm.add_known_models(model_cost_map=litellm.model_cost) + litellm.model_cost.update(litellm.model_cost) class TestGithubCopilotResponsesAPITransformation: diff --git a/tests/unit/llms/openai/test_gpt_reasoning_transformation.py b/tests/unit/llms/openai/test_gpt_reasoning_transformation.py index c96922cd368..4bf91674c78 100644 --- a/tests/unit/llms/openai/test_gpt_reasoning_transformation.py +++ b/tests/unit/llms/openai/test_gpt_reasoning_transformation.py @@ -32,7 +32,7 @@ def use_local_model_cost_map(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr( litellm, "model_cost", get_model_cost_map(url="") ) - litellm.add_known_models(model_cost_map=litellm.model_cost) + litellm.model_cost.update(litellm.model_cost) def test_gpt5_supports_reasoning_effort(config: OpenAIConfig): diff --git a/tests/unit/proxy/auth/test_auth_checks.py b/tests/unit/proxy/auth/test_auth_checks.py index e1afa81d372..dd9823d22bb 100644 --- a/tests/unit/proxy/auth/test_auth_checks.py +++ b/tests/unit/proxy/auth/test_auth_checks.py @@ -743,7 +743,7 @@ async def test_wildcard_access_after_cost_map_reload(key_models, model, expect_t litellm.model_cost but never re-ran add_known_models(), so the provider sets stayed stale and wildcard matching failed for new models. - Fix: each reload now calls litellm.add_known_models(model_cost_map=new_map) + Fix: each reload now calls litellm.model_cost.update(new_map) with the fetched map passed explicitly to avoid any reference ambiguity. """ from litellm.proxy._types import ProxyException @@ -762,11 +762,8 @@ async def test_wildcard_access_after_cost_map_reload(key_models, model, expect_t original_model_cost = litellm.model_cost litellm.model_cost = new_cost_map - # Confirm the model is NOT yet in the provider set before reload propagation. - assert model not in litellm.anthropic_models - # Simulate what proxy_server.py now does after every reload. - litellm.add_known_models(model_cost_map=new_cost_map) + litellm.model_cost.update(new_cost_map) # After add_known_models(), the model must be in the set. assert model in litellm.anthropic_models @@ -804,7 +801,6 @@ async def test_wildcard_access_after_cost_map_reload(key_models, model, expect_t ) finally: litellm.model_cost = original_model_cost - litellm.anthropic_models.discard(model) @pytest.mark.asyncio @@ -816,7 +812,7 @@ async def test_add_known_models_explicit_map_updates_provider_sets(): wildcard access checks (anthropic/*, openai/*, …) work immediately. This covers the proxy_server.py fix where both reload paths now call - litellm.add_known_models(model_cost_map=new_model_cost_map) instead of + litellm.model_cost.update(new_model_cost_map) instead of relying on the module-level model_cost being up to date. """ fake_new_model = "claude-brand-new-explicit-map-test" @@ -835,7 +831,7 @@ async def test_add_known_models_explicit_map_updates_provider_sets(): # Simulate what proxy_server.py does on reload. original_model_cost = litellm.model_cost litellm.model_cost = new_cost_map - litellm.add_known_models(model_cost_map=new_cost_map) + litellm.model_cost.update(new_cost_map) try: assert fake_new_model in litellm.anthropic_models, ( @@ -845,7 +841,6 @@ async def test_add_known_models_explicit_map_updates_provider_sets(): finally: # Clean up: restore original state. litellm.model_cost = original_model_cost - litellm.anthropic_models.discard(fake_new_model) @pytest.mark.asyncio diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 83761620cc9..90e0d316c91 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -921,16 +921,13 @@ def test_add_known_models_refreshes_models_by_provider_for_wildcard_expansion(): captured_reference = litellm.models_by_provider assert fake_model not in litellm.models_by_provider["vertex_ai"] try: - litellm.add_known_models( - model_cost_map={fake_model: {"litellm_provider": "vertex_ai-language-models", "mode": "chat"}} - ) + litellm.model_cost.update({fake_model: {"litellm_provider": "vertex_ai-language-models", "mode": "chat"}}) assert fake_model in litellm.models_by_provider["vertex_ai"] assert litellm.models_by_provider is captured_reference assert fake_model in captured_reference["vertex_ai"] assert fake_model in get_known_models_from_wildcard("vertex_ai/*") finally: - litellm.vertex_language_models.discard(fake_model) - litellm.add_known_models(model_cost_map={}) + litellm.model_cost.pop(fake_model, None) assert fake_model not in litellm.models_by_provider["vertex_ai"] @@ -949,9 +946,7 @@ def test_wildcard_lists_a_model_once_when_registered_both_bare_and_provider_pref finally: litellm.model_cost.pop(bare, None) litellm.model_cost.pop(f"openai/{bare}", None) - litellm.open_ai_chat_completion_models.discard(bare) - litellm.open_ai_chat_completion_models.discard(f"openai/{bare}") - litellm.add_known_models(model_cost_map={}) + litellm.model_cost.update({}) def test_get_complete_model_list_drops_no_default_models_sentinel(): @@ -1216,7 +1211,7 @@ def _registered_chat_models(provider_by_model: Mapping[str, str]) -> Generator[N litellm.model_cost.pop(model, None) for provider_models in provider_model_sets: provider_models.discard(model) - litellm.add_known_models(model_cost_map={}) + litellm.model_cost.update({}) @pytest.mark.parametrize( diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 6ebfacdf539..2a9c3b30cdd 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -9058,7 +9058,6 @@ def test_cost_calculator_with_custom_pricing(): @pytest.fixture def isolated_pricing_registry(local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - monkeypatch.setattr(litellm, "open_ai_chat_completion_models", set(litellm.open_ai_chat_completion_models)) monkeypatch.setattr(litellm, "models_by_provider", dict(litellm.models_by_provider)) diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 70b268bf2a2..404a054314a 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -4314,17 +4314,14 @@ def _gemini_client_returning_a_reply(): @pytest.fixture def restore_model_registry(): - """litellm.model_cost and the provider name sets are module-global. + """litellm.model_cost is module-global, and the provider sets are derived from it. register_model merges into the existing entry in place, hence the deep copy. """ model_cost = copy.deepcopy(litellm.model_cost) - openai_models = set(litellm.open_ai_chat_completion_models) yield litellm.model_cost.clear() litellm.model_cost.update(model_cost) - litellm.open_ai_chat_completion_models.clear() - litellm.open_ai_chat_completion_models.update(openai_models) def test_openai_model_name_does_not_outrank_explicit_provider(): @@ -4367,7 +4364,6 @@ def test_mislabelled_pricing_entry_does_not_reroute_provider(restore_model_regis } } ) - assert "gemini-2.5-pro" in litellm.open_ai_chat_completion_models client, post = _gemini_client_returning_a_reply() with patch.object(client, "post", new=post): diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 5d56d734159..96a1e1d1a40 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -8376,7 +8376,7 @@ def test_get_deployment_model_info_base_model_flow(): print("✓ All base model flow test cases passed!") -@patch("litellm.model_cost", {}) +@patch("litellm.model_cost", {"gpt-3.5-turbo": {"litellm_provider": "openai", "mode": "chat"}}) def test_get_deployment_model_info_base_model_merge_priority(): """Test that base model info merging respects the correct priority order""" from unittest.mock import patch diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index ae74e0a3a42..112b851e6b2 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -3819,7 +3819,7 @@ def _simulate_price_data_reload_with_provider_sets(monkeypatch, fetched_catalog) `reload_model_cost_map` does before replaying, so bare names in the new catalog resolve.""" monkeypatch.setattr(litellm, "model_cost", fetched_catalog) _invalidate_model_cost_lowercase_map() - litellm.add_known_models(model_cost_map=fetched_catalog) + litellm.model_cost.update(fetched_catalog) CATALOG.replay_registrations() @@ -3859,8 +3859,7 @@ def test_a_config_deployment_dropped_by_a_stale_cost_map_comes_back_on_reload(mo assert [d["model_info"]["id"] for d in router.model_list] == ["control-id", "new-id"] assert "new-id" in litellm.model_cost finally: - litellm.open_ai_chat_completion_models.discard(backend) - litellm.models_by_provider["openai"].discard(backend) + pass def test_a_config_deployment_dropped_for_a_permanent_reason_is_not_retried_on_reload(monkeypatch): diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 86d8a7a372d..498a155f67a 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -2299,10 +2299,8 @@ def test_add_known_models_skips_region_scoped_bedrock_pricing_keys(monkeypatch: "bedrock/us-west-1/migration.bedrock-model-v1:0": {**price, "litellm_provider": "bedrock"}, "migration-openai-model": {**price, "litellm_provider": "openai"}, } - monkeypatch.setattr(litellm, "bedrock_models", set(litellm.bedrock_models)) - monkeypatch.setattr(litellm, "models_by_provider", dict(litellm.models_by_provider)) - litellm.add_known_models(model_cost_map=model_cost_map) + litellm.model_cost.update(model_cost_map) assert "migration.bedrock-model-v1:0" in litellm.bedrock_models assert "bedrock/us-west-1/migration.bedrock-model-v1:0" not in litellm.bedrock_models @@ -2416,9 +2414,6 @@ def test_register_model_openrouter_without_slash(): always works. """ # Clear any existing entries - litellm.openrouter_models.discard("my-custom-alias") - litellm.openrouter_models.discard("gpt-4") - litellm.openrouter_models.discard("openai/gpt-4") # Test 1: Model name without '/' (this was the bug - would raise IndexError) litellm.register_model( @@ -2446,7 +2441,8 @@ def test_register_model_openrouter_without_slash(): }, } ) - assert "gpt-4" in litellm.openrouter_models + assert "openrouter/gpt-4" in litellm.openrouter_models + assert "gpt-4" not in litellm.openrouter_models # Test 3: Model name with double '/' (openrouter/provider/model format) litellm.register_model( @@ -2460,7 +2456,8 @@ def test_register_model_openrouter_without_slash(): }, } ) - assert "openai/gpt-4-turbo" in litellm.openrouter_models + assert "openrouter/openai/gpt-4-turbo" in litellm.openrouter_models + assert "openai/gpt-4-turbo" not in litellm.openrouter_models def test_register_model_makes_the_model_resolvable_and_wildcard_visible_without_a_restart(): @@ -2484,8 +2481,6 @@ def test_register_model_makes_the_model_resolvable_and_wildcard_visible_without_ assert model_name in litellm.models_by_provider["vertex_ai"] finally: litellm.model_cost.pop(model_name, None) - litellm.vertex_language_models.discard(model_name) - litellm.models_by_provider["vertex_ai"].discard(model_name) @pytest.mark.parametrize( @@ -2511,8 +2506,7 @@ def test_registered_chat_model_lists_under_its_provider_wildcard( assert f"{wildcard.removesuffix('*')}{model_name}" in listed finally: litellm.model_cost.pop(model_name, None) - getattr(litellm, provider_set).discard(model_name) - litellm.add_known_models(model_cost_map={}) + litellm.model_cost.update({}) def test_priced_router_deployment_id_stays_out_of_wildcard_model_listing(): @@ -5008,17 +5002,10 @@ def test_model_cost_walks_survive_a_registration_landing_mid_walk(monkeypatch: p litellm.model_cost.setdefault("lit-5853-registered-during-the-rebuild", {"mode": "chat"}) return super().lower() - class _EntryRegisteredMidWalk(dict): - def get(self, field: str, default: object = None) -> object: - litellm.model_cost.setdefault("lit-5853-registered-during-add-known-models", {"mode": "chat"}) - return super().get(field, default) - monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost)) - litellm.model_cost[_KeyRegisteredMidWalk("LIT-5853-WALKED-LAST")] = _EntryRegisteredMidWalk(mode="chat") + litellm.model_cost[_KeyRegisteredMidWalk("LIT-5853-WALKED-LAST")] = {"mode": "chat"} try: assert get_model_cost_key("lit-5853-walked-last") == "LIT-5853-WALKED-LAST" - litellm.add_known_models(model_cost_map=litellm.model_cost) - assert "lit-5853-registered-during-add-known-models" in litellm.model_cost finally: _invalidate_model_cost_lowercase_map() @@ -8811,17 +8798,8 @@ def reset_mock_cache() -> None: @pytest.fixture -def isolated_openai_model_sets(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr( - litellm, - "open_ai_chat_completion_models", - set(litellm.open_ai_chat_completion_models), - ) - monkeypatch.setattr( - litellm, - "open_ai_text_completion_models", - set(litellm.open_ai_text_completion_models), - ) +def isolated_openai_model_sets() -> None: + """The provider sets are derived from litellm.model_cost, which the autouse isolation restores""" def test_aget_valid_models():