From 8f5e2d2cb9d129fca5cbe7e8389053095556664c Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 30 Jun 2026 20:56:57 +0000 Subject: [PATCH] fix: populate model_id in spend_logs on failed calls and clean up pattern_router on deployment delete Two fixes: 1. model_id missing from spend_logs database on failed API calls: the standard_logging_object (which contains model_id) was only available on the litellm_logging_obj, which gets popped before spend callbacks run. Now lifted onto request_data before the pop, and get_logging_payload falls back to standard_logging_payload for model_id, model_group, api_base, call_type, and custom_llm_provider. 2. Stale wildcard deployments after delete: delete_deployment and upsert_deployment did not remove entries from pattern_router, so wildcard deployments (e.g. anthropic/*) would keep routing to stale API keys after deletion. Added PatternMatchRouter.remove_deployment and call it from both delete and upsert paths. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/anthropic_endpoints/endpoints.py | 1 - .../spend_tracking/spend_tracking_utils.py | 18 +++- litellm/proxy/utils.py | 7 ++ litellm/router.py | 6 ++ .../router_utils/pattern_match_deployments.py | 15 ++++ .../test_router_helper_utils.py | 21 +++++ .../test_spend_tracking_utils.py | 85 +++++++++++++++++++ .../test_post_call_failure_hook.py | 27 ++++++ 8 files changed, 176 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 71acc1f3106..8d48a677c3e 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -186,7 +186,6 @@ async def anthropic_response( "litellm.proxy.proxy_server.anthropic_response(): Exception occured - {}".format(str(e)) ) - # Extract model_id from request metadata (same as success path) litellm_metadata = data.get("litellm_metadata", {}) or {} model_info = litellm_metadata.get("model_info", {}) or {} model_id = model_info.get("id", "") or "" diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 22f97feecd9..d30ae8ebb44 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -266,8 +266,11 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs if not usage and isinstance(_combined_usage, litellm.Usage): usage = _combined_usage.model_dump() - id = get_spend_logs_id(call_type or "acompletion", response_obj_dict, kwargs) standard_logging_payload = cast(Optional[StandardLoggingPayload], kwargs.get("standard_logging_object", None)) + if not call_type and standard_logging_payload is not None: + call_type = standard_logging_payload.get("call_type", "") + + id = get_spend_logs_id(call_type or "acompletion", response_obj_dict, kwargs) end_user_id = get_end_user_id_for_cost_tracking(litellm_params) @@ -300,6 +303,11 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs _model_id = metadata.get("model_info", {}).get("id", "") _model_group = metadata.get("model_group", "") + if standard_logging_payload is not None: + if not _model_id: + _model_id = standard_logging_payload.get("model_id", "") or "" + if not _model_group: + _model_group = standard_logging_payload.get("model_group", "") or "" # Extract overhead from hidden_params if available litellm_overhead_time_ms = None @@ -384,6 +392,8 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs # Extract agent_id for A2A requests (set directly on model_call_details) agent_id: Optional[str] = kwargs.get("agent_id") or metadata.get("agent_id") custom_llm_provider = kwargs.get("custom_llm_provider") + if not custom_llm_provider and standard_logging_payload is not None: + custom_llm_provider = standard_logging_payload.get("custom_llm_provider", "") or "" raw_model = cast(str, kwargs.get("model") or "") model_name = reconstruct_model_name(raw_model, custom_llm_provider, metadata or {}) @@ -408,13 +418,15 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs completion_tokens=usage.get("completion_tokens", standard_logging_completion_tokens), request_tags=request_tags, end_user=end_user_id or "", - api_base=litellm_params.get("api_base", ""), + api_base=litellm_params.get("api_base", "") or ( + standard_logging_payload.get("api_base", "") if standard_logging_payload is not None else "" + ), model_group=_model_group, model_id=_model_id, mcp_namespaced_tool_name=mcp_namespaced_tool_name, agent_id=agent_id, requester_ip_address=clean_metadata.get("requester_ip_address", None), - custom_llm_provider=kwargs.get("custom_llm_provider", ""), + custom_llm_provider=custom_llm_provider or "", messages=_get_messages_for_spend_logs_payload( standard_logging_payload=standard_logging_payload, metadata=metadata ), diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 4433a35f5d0..407eb5e8484 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -2058,6 +2058,13 @@ class ProxyLogging: request_data["combined_usage_object"] = _recovered_usage request_data["response_cost"] = _model_call_details.get("response_cost") + # Lift standard_logging_object so failure-path spend tracking can + # read model_id, model_group, and other fields that are only + # available on the logging object's model_call_details. + _slo = _model_call_details.get("standard_logging_object") + if _slo is not None and not request_data.get("standard_logging_object"): + request_data["standard_logging_object"] = _slo + # Remove before callbacks iterate — not serialisable request_data.pop("litellm_logging_obj", None) diff --git a/litellm/router.py b/litellm/router.py index 2d66bc3158d..db2cffdbb37 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8143,6 +8143,9 @@ class Router: self._invalidate_model_group_info_cache() self._invalidate_access_groups_cache() self._update_deployment_indices_after_removal(model_id=deployment_id, removal_idx=removal_idx) + self.pattern_router.remove_deployment(model_id=deployment_id) + for _team_router in self.team_pattern_routers.values(): + _team_router.remove_deployment(model_id=deployment_id) # if the model_id is not in router self.add_deployment(deployment=deployment) @@ -8179,6 +8182,9 @@ class Router: _budget_limiter = self._get_router_deployment_budget_limiter() if _budget_limiter is not None: _budget_limiter.unregister_deployment_budget(model_id=id) + self.pattern_router.remove_deployment(model_id=id) + for _team_router in self.team_pattern_routers.values(): + _team_router.remove_deployment(model_id=id) return item else: return None diff --git a/litellm/router_utils/pattern_match_deployments.py b/litellm/router_utils/pattern_match_deployments.py index c08f8e95cf4..356f9bbece5 100644 --- a/litellm/router_utils/pattern_match_deployments.py +++ b/litellm/router_utils/pattern_match_deployments.py @@ -73,6 +73,21 @@ class PatternMatchRouter: self.patterns[regex] = [] self.patterns[regex].append(llm_deployment) + def remove_deployment(self, model_id: str) -> None: + """ + Remove all deployments matching the given model_id from every pattern. + + Args: + model_id: the deployment's model_info.id to remove + """ + for regex in list(self.patterns.keys()): + self.patterns[regex] = [ + d for d in self.patterns[regex] + if d.get("model_info", {}).get("id") != model_id + ] + if not self.patterns[regex]: + del self.patterns[regex] + def _pattern_to_regex(self, pattern: str) -> str: """ Convert a wildcard pattern to a regex pattern diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py index 83d6d56df4f..702ce473f28 100644 --- a/tests/router_unit_tests/test_router_helper_utils.py +++ b/tests/router_unit_tests/test_router_helper_utils.py @@ -838,6 +838,27 @@ def test_delete_deployment(model_list): assert len(router.model_list) == len(model_list) - 1 +def test_delete_deployment_cleans_up_pattern_router(): + """Wildcard deployments must be removed from pattern_router when deleted, + otherwise the stale deployment (with old API key) keeps being returned + for wildcard-matched requests.""" + router = Router( + model_list=[ + { + "model_name": "anthropic/*", + "litellm_params": {"model": "anthropic/*", "api_key": "sk-old-key"}, + "model_info": {"id": "deployment-wildcard-123"}, + } + ] + ) + assert router.pattern_router.route("anthropic/claude-opus-4-7") is not None + + router.delete_deployment(id="deployment-wildcard-123") + + assert len(router.model_list) == 0 + assert router.pattern_router.route("anthropic/claude-opus-4-7") is None + + def test_get_model_info(model_list): """Test if the 'get_model_info' function is working correctly""" router = Router(model_list=model_list) diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 874e0654a1f..6e46b290e15 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -2229,3 +2229,88 @@ def test_get_logging_payload_cache_hit_keeps_raw_litellm_call_id(): assert json.loads(payload["metadata"])["litellm_call_id"] == trace_call_id assert "_cache_hit" in payload["request_id"] assert json.loads(payload["metadata"])["litellm_call_id"] != payload["request_id"] + + +def test_get_logging_payload_failure_uses_standard_logging_object_for_model_id(): + """When a request fails, the proxy's failure hook passes request_data to + get_logging_payload without model_info in metadata (since litellm_params + is freshly built). The standard_logging_object lifted from the logging obj + must be used as a fallback for model_id, model_group, api_base, call_type, + and custom_llm_provider. + """ + slo: StandardLoggingPayload = StandardLoggingPayload( + id="test-id", + model="anthropic/claude-opus-4-7", + model_id="deployment-abc-123", + model_group="anthropic/claude-opus-4-7", + model_map_key="anthropic/claude-opus-4-7", + model_map_value=None, + api_base="https://api.anthropic.com/v1/messages", + custom_llm_provider="anthropic", + call_type="anthropic_messages", + cache_hit=None, + stream=False, + status="failure", + error_str="credit balance too low", + error_information=None, + start_time=0.0, + end_time=0.0, + completionStartTime=0.0, + response_time=0.0, + spend=0.0, + total_tokens=0, + prompt_tokens=0, + completion_tokens=0, + request_tags=[], + metadata=StandardLoggingMetadata( + user_api_key_hash="hashed-key", + user_api_key_alias="test-alias", + user_api_key_org_id=None, + user_api_key_user_id=None, + user_api_key_team_id=None, + user_api_key_team_alias=None, + user_api_key_end_user_id=None, + user_api_key_project_id=None, + user_api_key_project_alias=None, + spend_logs_metadata=None, + requester_ip_address=None, + requester_metadata=None, + ), + messages=[{"role": "user", "content": "hello"}], + response={}, + model_parameters={}, + hidden_params=StandardLoggingHiddenParams( + model_id="deployment-abc-123", + cache_key=None, + api_base="https://api.anthropic.com/v1/messages", + response_cost="0", + additional_headers=None, + litellm_overhead_time_ms=None, + batch_models=None, + ), + model_map_information=StandardLoggingModelInformation( + model_map_key="anthropic/claude-opus-4-7", model_map_value=None + ), + litellm_call_id="test-call-id", + guardrail_information=None, + response_cost_failure_debug_info=None, + standard_built_in_tools_params=None, + ) + + kwargs = { + "model": "anthropic/claude-opus-4-7", + "litellm_params": {"metadata": {"user_api_key": "sk-test", "status": "failure"}}, + "standard_logging_object": slo, + } + response_obj = Exception("credit balance too low") + now = datetime.datetime.now(timezone.utc) + + payload = get_logging_payload( + kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now + ) + + assert payload["model_id"] == "deployment-abc-123" + assert payload["model_group"] == "anthropic/claude-opus-4-7" + assert payload["api_base"] == "https://api.anthropic.com/v1/messages" + assert payload["custom_llm_provider"] == "anthropic" + assert payload["call_type"] == "anthropic_messages" diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_post_call_failure_hook.py b/tests/test_litellm/proxy/utils/proxy_logging/test_post_call_failure_hook.py index a2a57931d26..eefa12541c7 100644 --- a/tests/test_litellm/proxy/utils/proxy_logging/test_post_call_failure_hook.py +++ b/tests/test_litellm/proxy/utils/proxy_logging/test_post_call_failure_hook.py @@ -267,3 +267,30 @@ async def test_handle_logging_proxy_only_path_propagates_async_failure_raises( route="/chat/completions", original_exception=Exception("x"), ) + + +@pytest.mark.asyncio +async def test_post_call_failure_hook_lifts_standard_logging_object( + proxy_logging, make_user_api_key_auth, mock_callbacks_disabled +): + """The standard_logging_object stored on model_call_details must be lifted + onto request_data before litellm_logging_obj is popped, so that downstream + spend-tracking callbacks can read model_id, model_group, etc.""" + slo = {"model_id": "deployment-xyz", "model_group": "anthropic/claude-opus-4-7"} + logging_obj = MagicMock() + logging_obj.model_call_details = {"standard_logging_object": slo} + + proxy_logging.alert_types = [] + request_data = { + "litellm_call_id": "abc", + "model": "m", + "messages": [], + "litellm_logging_obj": logging_obj, + } + await proxy_logging.post_call_failure_hook( + request_data=request_data, + original_exception=ValueError("oops"), + user_api_key_dict=make_user_api_key_auth(), + ) + assert request_data.get("standard_logging_object") is slo + assert "litellm_logging_obj" not in request_data