From 18242aec9ab88909b3447f5dfd62e243d02293c0 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:04:08 -0700 Subject: [PATCH] fix(router): isolate deployment model info (#37687) Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router.py | 11 +- .../test_router_model_cost_isolation.py | 167 ++++++++++++++++++ 2 files changed, 172 insertions(+), 6 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index b5bba45f1e6..1521c798677 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -9198,7 +9198,7 @@ class Router: # get_model_info() hands back an lru_cache'd dict, so merge into a copy; unset # values are skipped or Deployment's None pricing defaults would erase the map's - merged_model_info: Final = copy.copy(model_info) + merged_model_info: Final = copy.deepcopy(model_info) if user_model_info: for key, value in user_model_info.items(): if value is not None: @@ -9249,7 +9249,7 @@ class Router: litellm_model_name_model_info: ModelInfo | None = None try: - custom_model_info = litellm.model_cost.get(model_id) + custom_model_info = copy.deepcopy(litellm.model_cost.get(model_id)) except Exception: pass @@ -9264,9 +9264,8 @@ class Router: base_model: Final = custom_model_info.get("base_model", None) if base_model is not None: ## update litellm model info with base model info - base_model_info: Final = litellm.get_model_info(model=base_model) + base_model_info: Final = copy.deepcopy(litellm.get_model_info(model=base_model)) if base_model_info is not None: - custom_model_info = custom_model_info or {} # Base model provides defaults, custom model info overrides custom_model_info = _update_dictionary( cast(dict, base_model_info), @@ -9282,13 +9281,13 @@ class Router: model_info = cast( ModelInfo, _update_dictionary( - cast(dict, litellm_model_name_model_info).copy(), + copy.deepcopy(cast(dict, litellm_model_name_model_info)), custom_model_info, ), ) elif litellm_model_name_model_info is not None: # (2) Built-in only — no custom pricing to merge - model_info = litellm_model_name_model_info + model_info = copy.deepcopy(litellm_model_name_model_info) elif custom_model_info is not None: # (3) Custom only — model not in built-in cost map yet # custom_model_info already includes base_model defaults at this point, if applicable diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index e3c2cc988c7..5bb854c12e0 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -43,6 +43,16 @@ def _simulate_price_data_reload(fetched_catalog): reapply_runtime_model_cost_registrations() +def _nested_container_ids(value: object) -> frozenset[int]: + """Identities of every dict/list reachable from `value`, so two structures can be + checked for shared mutable state without writing into either one.""" + if isinstance(value, dict): + return frozenset({id(value)} | {i for v in value.values() for i in _nested_container_ids(v)}) + if isinstance(value, list): + return frozenset({id(value)} | {i for v in value for i in _nested_container_ids(v)}) + return frozenset() + + def _restore_model_cost_entries(original_entries): for key, value in original_entries.items(): if value is None: @@ -1764,3 +1774,160 @@ def test_a_complete_reservation_still_registers(): assert entry["model_name"] == "gpt-4o-ptu" assert entry["litellm_params"]["input_cost_per_token"] == 0.0 + + +def test_nested_custom_model_info_does_not_pollute_shared_backend(): + backend_model = "gpt-4o-search-preview" + custom_id = "lit5471-search-custom" + sibling_id = "lit5471-search-sibling" + builtin_info = copy.deepcopy(litellm.get_model_info(model=backend_model)) + expected_nested = copy.deepcopy(builtin_info["search_context_cost_per_query"]) + model_keys = { + backend_model: copy.deepcopy(litellm.model_cost.get(backend_model)), + custom_id: copy.deepcopy(litellm.model_cost.get(custom_id)), + sibling_id: copy.deepcopy(litellm.model_cost.get(sibling_id)), + } + try: + router = Router( + model_list=[ + { + "model_name": "search-custom", + "litellm_params": {"model": backend_model, "api_key": "fake-key"}, + "model_info": { + "id": custom_id, + "search_context_cost_per_query": { + "search_context_size_low": 0.123, + }, + }, + }, + { + "model_name": "search-sibling", + "litellm_params": {"model": backend_model, "api_key": "fake-key"}, + "model_info": {"id": sibling_id}, + }, + ], + ) + + custom_info = router.get_deployment_model_info(model_id=custom_id, model_name=backend_model) + sibling_info = router.get_deployment_model_info(model_id=sibling_id, model_name=backend_model) + + assert custom_info is not None + assert custom_info["search_context_cost_per_query"]["search_context_size_low"] == 0.123 + assert litellm.model_cost[backend_model]["search_context_cost_per_query"] == expected_nested + assert sibling_info is not None + assert sibling_info["search_context_cost_per_query"] == expected_nested + finally: + _restore_model_cost_entries(model_keys) + litellm.get_model_info.cache_clear() + + +def test_base_model_custom_info_does_not_pollute_cached_base_model(): + base_model = "azure/gpt-4o" + deployment_id = "lit5471-base-model" + base_model_info = copy.deepcopy(litellm.get_model_info(model=base_model)) + model_keys = { + "azure/gpt-4o": copy.deepcopy(litellm.model_cost.get("azure/gpt-4o")), + deployment_id: copy.deepcopy(litellm.model_cost.get(deployment_id)), + } + try: + router = Router( + model_list=[ + { + "model_name": "azure-custom", + "litellm_params": { + "model": "gpt-4o", + "custom_llm_provider": "azure", + "api_key": "fake-key", + }, + "model_info": { + "id": deployment_id, + "base_model": base_model, + "input_cost_per_token": 0.777, + }, + } + ], + ) + + info = router.get_deployment_model_info(model_id=deployment_id, model_name=base_model) + + assert info is not None + assert info["input_cost_per_token"] == 0.777 + assert litellm.get_model_info(model=base_model) == base_model_info + finally: + _restore_model_cost_entries(model_keys) + litellm.get_model_info.cache_clear() + + +def test_builtin_only_deployment_info_is_not_the_cached_object(): + backend_model = "gpt-4o-search-preview" + deployment_id = "lit5471-builtin-only" + litellm.get_model_info.cache_clear() + model_keys = {deployment_id: copy.deepcopy(litellm.model_cost.get(deployment_id))} + try: + cached_info = litellm.get_model_info(model=backend_model) + assert cached_info["search_context_cost_per_query"] + + info = Router(model_list=[]).get_deployment_model_info(model_id=deployment_id, model_name=backend_model) + + assert info is not None + assert info["search_context_cost_per_query"] == cached_info["search_context_cost_per_query"] + assert _nested_container_ids(info).isdisjoint(_nested_container_ids(cached_info)) + finally: + _restore_model_cost_entries(model_keys) + litellm.get_model_info.cache_clear() + + +def test_custom_only_deployment_info_is_not_the_registry_entry(): + unknown_backend = "openai/lit5471-unknown-backend" + deployment_id = "lit5471-custom-only" + nested_pricing = {"search_context_size_low": 0.123} + model_keys = { + unknown_backend: copy.deepcopy(litellm.model_cost.get(unknown_backend)), + deployment_id: copy.deepcopy(litellm.model_cost.get(deployment_id)), + } + try: + router = Router( + model_list=[ + { + "model_name": "custom-only", + "litellm_params": {"model": unknown_backend, "api_key": "fake-key"}, + "model_info": {"id": deployment_id, "search_context_cost_per_query": dict(nested_pricing)}, + } + ], + ) + registry_entry = litellm.model_cost[deployment_id] + + info = router.get_deployment_model_info(model_id=deployment_id, model_name=unknown_backend) + + assert info is not None + assert info["search_context_cost_per_query"] == nested_pricing + assert _nested_container_ids(info).isdisjoint(_nested_container_ids(registry_entry)) + finally: + _restore_model_cost_entries(model_keys) + litellm.get_model_info.cache_clear() + + +def test_router_model_info_deep_copies_nested_cached_metadata(): + model = "openai/gpt-4o-search-preview" + litellm.get_model_info.cache_clear() + try: + cached_info = litellm.get_model_info(model=model) + assert cached_info is not None + expected_nested = copy.deepcopy(cached_info["search_context_cost_per_query"]) + assert expected_nested + + router = Router(model_list=[]) + merged_info = router.get_router_model_info( + deployment={ + "model_name": "search", + "litellm_params": {"model": "gpt-4o-search-preview"}, + "model_info": {"id": "lit5471-router-model-info"}, + }, + received_model_name="search", + ) + + assert merged_info["search_context_cost_per_query"] == expected_nested + assert _nested_container_ids(merged_info).isdisjoint(_nested_container_ids(cached_info)) + assert litellm.get_model_info(model=model)["search_context_cost_per_query"] == expected_nested + finally: + litellm.get_model_info.cache_clear()