diff --git a/litellm/litellm_core_utils/api_route_to_call_types.py b/litellm/litellm_core_utils/api_route_to_call_types.py index 2ae9986ce94..a029a7d8ce6 100644 --- a/litellm/litellm_core_utils/api_route_to_call_types.py +++ b/litellm/litellm_core_utils/api_route_to_call_types.py @@ -52,6 +52,22 @@ def get_call_types_for_route(route: str) -> Optional[List[CallTypes]]: return None +def get_primary_call_type_for_route(route: str | None) -> CallTypes | None: + """ + Get the primary (async) CallType for a given API route, or None if unknown. + + Every route in the mapping lists its async call type first, so callers that need a + single call type for a route (e.g. attributing a failed request that never reached + the SDK) get the async variant. + """ + if route is None: + return None + call_types = get_call_types_for_route(route) + if not call_types: + return None + return call_types[0] + + def get_routes_for_call_type(call_type: CallTypes) -> list: """ Get all routes that use a specific CallType. diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index b839426fcda..97c1aada36a 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -9,6 +9,7 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import ( _get_parent_otel_span_from_kwargs, get_litellm_metadata_from_kwargs, + get_or_create_metadata_bucket, ) from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup from litellm.proxy._types import UserAPIKeyAuth @@ -43,6 +44,22 @@ _PASS_THROUGH_CALL_TYPES: frozenset[str] = frozenset( ) +def _known_call_type(call_type: object) -> str | None: + """ + Return ``call_type`` when it is a real ``CallTypes`` value. + + Proxy-only failures (auth, rate limits) build their Logging object from the raw HTTP + route, so the attribute can hold something like ``/v1/responses``; that must not end + up in the spend log's call_type column. + """ + if not isinstance(call_type, str): + return None + try: + return CallTypes(call_type).value + except ValueError: + return None + + class _ProxyDBLogger(CustomLogger): async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): await self._PROXY_track_cost_callback(kwargs, response_obj, start_time, end_time) @@ -106,23 +123,35 @@ class _ProxyDBLogger(CustomLogger): metadata=_metadata, ) - existing_metadata: dict = request_data.get("metadata", None) or {} - existing_metadata.update(_metadata) - if "litellm_params" not in request_data: request_data["litellm_params"] = {} existing_litellm_params = request_data.get("litellm_params", {}) - existing_litellm_metadata = existing_litellm_params.get("metadata", {}) or {} - # Preserve tags from existing metadata - if existing_litellm_metadata.get("tags"): - existing_metadata["tags"] = existing_litellm_metadata.get("tags") + # Routes like /v1/responses, /v1/messages and the batch/file endpoints keep + # proxy-internal metadata (model_group, model_info, tags, retries) in + # ``litellm_metadata`` so the provider-facing ``metadata`` field stays clean. + # Collapse every bucket into one dict, then write it back to all of them so the + # bucket ``get_litellm_metadata_from_kwargs`` picks for the spend log carries the + # router attribution *and* this failure's status/error information. + metadata_key, internal_metadata = get_or_create_metadata_bucket(request_data) + merged_metadata = { + **(existing_litellm_params.get("metadata") or {}), + **(existing_litellm_params.get("litellm_metadata") or {}), + **internal_metadata, + } + # Identity comes from the authenticated key alone; a caller-supplied + # user_api_key* field in the request body must never attribute spend. + spend_metadata = {key: value for key, value in merged_metadata.items() if not key.startswith("user_api_key")} + spend_metadata.update(_metadata) request_data["litellm_params"]["proxy_server_request"] = ( request_data.get("proxy_server_request") or existing_litellm_params.get("proxy_server_request") or {} ) - request_data["litellm_params"]["metadata"] = existing_metadata + request_data[metadata_key] = spend_metadata + request_data["litellm_params"]["metadata"] = spend_metadata + if "litellm_metadata" in existing_litellm_params: + request_data["litellm_params"]["litellm_metadata"] = spend_metadata # Preserve model name and custom_llm_provider if "model" not in request_data: @@ -145,6 +174,12 @@ class _ProxyDBLogger(CustomLogger): ) if request_data.get("litellm_trace_id") is None: request_data["litellm_trace_id"] = getattr(_litellm_logging_obj, "litellm_trace_id", None) + # Without this the spend log for a failure has a blank call_type, so the + # row can't be attributed to the route the caller actually used. + if not request_data.get("call_type"): + logged_call_type = _known_call_type(getattr(_litellm_logging_obj, "call_type", None)) + if logged_call_type is not None: + request_data["call_type"] = logged_call_type # Use the actual request start time from the logging object so that # failed requests record the real duration instead of 0. diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 924189fed4b..36676f366ba 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -105,6 +105,9 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.prometheus import PrometheusLogger from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.integrations.SlackAlerting.utils import _add_langfuse_trace_id_to_alert +from litellm.litellm_core_utils.api_route_to_call_types import ( + get_primary_call_type_for_route, +) from litellm.litellm_core_utils.core_helpers import coerce_token_limit from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.safe_json_dumps import safe_dumps @@ -2251,22 +2254,26 @@ class ProxyLogging: ) input: Union[list, str, dict] = "" - normalized_call_type: Optional[str] = None + # The Logging object above was built from the raw HTTP route, so its call_type + # is not a CallTypes value yet. The route is authoritative; the request body + # shape is only a fallback for routes outside the mapping (guessing from the + # body mislabels e.g. a /v1/responses request with a list input as embeddings). + route_call_type = get_primary_call_type_for_route(route) + normalized_call_type: str | None = route_call_type.value if route_call_type is not None else None if "messages" in request_data and isinstance(request_data["messages"], list): input = request_data["messages"] litellm_logging_obj.model_call_details["messages"] = input - if litellm_logging_obj.call_type != CallTypes.pass_through.value: - normalized_call_type = CallTypes.acompletion.value + normalized_call_type = normalized_call_type or CallTypes.acompletion.value elif "prompt" in request_data and isinstance(request_data["prompt"], str): input = request_data["prompt"] litellm_logging_obj.model_call_details["prompt"] = input - if litellm_logging_obj.call_type != CallTypes.pass_through.value: - normalized_call_type = CallTypes.atext_completion.value + normalized_call_type = normalized_call_type or CallTypes.atext_completion.value elif "input" in request_data and isinstance(request_data["input"], list): input = request_data["input"] litellm_logging_obj.model_call_details["input"] = input - if litellm_logging_obj.call_type != CallTypes.pass_through.value: - normalized_call_type = CallTypes.aembedding.value + normalized_call_type = normalized_call_type or CallTypes.aembedding.value + if litellm_logging_obj.call_type == CallTypes.pass_through.value: + normalized_call_type = None if normalized_call_type is not None: litellm_logging_obj.call_type = normalized_call_type litellm_logging_obj.model_call_details["call_type"] = normalized_call_type diff --git a/tests/proxy_unit_tests/test_proxy_utils.py b/tests/proxy_unit_tests/test_proxy_utils.py index ad852c16905..47ba6b8dcf1 100644 --- a/tests/proxy_unit_tests/test_proxy_utils.py +++ b/tests/proxy_unit_tests/test_proxy_utils.py @@ -2482,6 +2482,18 @@ async def test_post_call_failure_hook_auth_error_llm_api_route(): "/v1/embeddings", "aembedding", ), + # #35068: a Responses request also carries "input"; the route decides the + # call type, so it must not be attributed to embeddings + ( + {"model": "bad-model", "input": ["hello"]}, + "/v1/responses", + "aresponses", + ), + ( + {"model": "bad-model", "messages": [{"role": "user", "content": "hello"}]}, + "/v1/messages", + "anthropic_messages", + ), ], ) async def test_handle_logging_proxy_only_error_syncs_normalized_call_type( diff --git a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py index f289148101a..b2268dfda48 100644 --- a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py @@ -1263,3 +1263,165 @@ async def test_track_cost_callback_logs_unauthenticated_pass_through_request( assert mock_proxy_logging.db_spend_update_writer.update_database.await_count == ( 1 if expect_spend_log else 0 ) + + +class _StubLoggingObj: + """Minimal stand-in for the request's Logging object.""" + + def __init__(self, call_type: str): + self.call_type = call_type + self.model_call_details: dict = {} + self.litellm_trace_id = "trace-123" + self.start_time = datetime.now() + + +async def _failure_spend_log_payload(request_data: dict, user_api_key_dict: UserAPIKeyAuth): + """Run the failure hook and build the spend log row it would write.""" + import json + + from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload + + with patch( + "litellm.proxy.db.db_spend_update_writer.DBSpendUpdateWriter.update_database", + new_callable=AsyncMock, + ) as mock_update_database: + await _ProxyDBLogger().async_post_call_failure_hook( + request_data=request_data, + original_exception=Exception("upstream exploded"), + user_api_key_dict=user_api_key_dict, + ) + + mock_update_database.assert_called_once() + payload = get_logging_payload( + kwargs=mock_update_database.call_args[1]["kwargs"], + response_obj=None, + start_time=datetime.now(), + end_time=datetime.now(), + ) + return payload, json.loads(payload["metadata"]) + + +@pytest.mark.asyncio +async def test_failure_spend_log_preserves_router_metadata_from_litellm_metadata_bucket(): + """Regression for #35068: /v1/responses, /v1/messages, batches and files keep + proxy-internal metadata in ``litellm_metadata``. A failure on those routes was + logged with no model_group/model_id/tags, so the row couldn't be attributed to a + model group or deployment.""" + payload, metadata = await _failure_spend_log_payload( + request_data={ + "model": "test-model", + "litellm_metadata": { + "user_api_key_team_id": "real-team", + "model_group": "test-model-group", + "model_info": {"id": "test-deployment-id"}, + "attempted_retries": 2, + "max_retries": 2, + "tags": ["failure-test"], + "spend_logs_metadata": {"ticket": "abc"}, + }, + "proxy_server_request": {"url": "/v1/responses"}, + }, + user_api_key_dict=UserAPIKeyAuth( + api_key="sk-test", + user_id="real-user", + team_id="real-team", + request_route="/v1/responses", + ), + ) + + assert payload["model_group"] == "test-model-group" + assert payload["model_id"] == "test-deployment-id" + assert payload["request_tags"] == '["failure-test"]' + assert metadata["attempted_retries"] == 2 + assert metadata["max_retries"] == 2 + assert metadata["spend_logs_metadata"] == {"ticket": "abc"} + assert metadata["status"] == "failure" + assert metadata["error_information"]["error_class"] == "Exception" + + +@pytest.mark.asyncio +async def test_failure_spend_log_keeps_status_when_both_metadata_buckets_are_present(): + """Regression for #35068: with ``metadata`` and ``litellm_metadata`` both set on + litellm_params, spend tracking reads only the latter, so the failure status and + error information written by this hook never reached the row.""" + payload, metadata = await _failure_spend_log_payload( + request_data={ + "model": "test-model", + "metadata": {}, + "litellm_metadata": { + "model_group": "test-model-group", + "model_info": {"id": "test-deployment-id"}, + }, + "litellm_params": { + "metadata": {"tags": ["failure-test"]}, + "litellm_metadata": {"api_base": "https://example.test/v1"}, + }, + "call_type": "aresponses", + }, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", user_id="real-user", team_id="real-team"), + ) + + assert metadata["status"] == "failure" + assert metadata["error_information"]["error_message"] == "upstream exploded" + assert payload["model_group"] == "test-model-group" + assert payload["request_tags"] == '["failure-test"]' + + +@pytest.mark.asyncio +async def test_failure_spend_log_drops_caller_supplied_key_identity(): + """A caller can put ``user_api_key_*`` fields in the request body. Every identity + field on the failure row must come from the authenticated key, including ones the + sanitized key information doesn't happen to overwrite.""" + request_data = { + "model": "test-model", + "messages": [{"role": "user", "content": "hi"}], + "metadata": { + "user_api_key_team_id": "spoofed-team", + "user_api_key_user_id": "spoofed-user", + "user_api_key_budget_ignored_by_sanitizer": "spoofed", + }, + } + user_api_key_dict = UserAPIKeyAuth(api_key="sk-test", user_id="real-user", team_id="real-team") + + with patch( + "litellm.proxy.db.db_spend_update_writer.DBSpendUpdateWriter.update_database", + new_callable=AsyncMock, + ) as mock_update_database: + await _ProxyDBLogger().async_post_call_failure_hook( + request_data=request_data, + original_exception=Exception("upstream exploded"), + user_api_key_dict=user_api_key_dict, + ) + + spend_metadata = mock_update_database.call_args[1]["kwargs"]["litellm_params"]["metadata"] + assert spend_metadata["user_api_key_team_id"] == "real-team" + assert spend_metadata["user_api_key_user_id"] == "real-user" + assert "user_api_key_budget_ignored_by_sanitizer" not in spend_metadata + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "logged_call_type, expected_call_type", + [ + ("aresponses", "aresponses"), + ("anthropic_messages", "anthropic_messages"), + ("/v1/responses", ""), + ], +) +async def test_failure_spend_log_call_type_comes_from_logging_object( + logged_call_type, expected_call_type +): + """Regression for #35068: failure rows were written with a blank call_type, so a + failed Responses request was indistinguishable from a failed chat completion. Route + strings left on the Logging object by proxy-only errors must not leak into the + column.""" + payload, _ = await _failure_spend_log_payload( + request_data={ + "model": "test-model", + "litellm_metadata": {}, + "litellm_logging_obj": _StubLoggingObj(call_type=logged_call_type), + }, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", user_id="real-user", team_id="real-team"), + ) + + assert payload["call_type"] == expected_call_type