diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 18714256a8f..fc11e734857 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3629,6 +3629,7 @@ class SpendLogsMetadata(TypedDict): autorouter_savings: ReadOnly[float | None] # stamped by the logging payload; None = not auto-routed litellm_gateway_injected_cache: ReadOnly[str | None] router_metadata: ReadOnly[SpendLogsRouterMetadata | None] # None = deployment not flagged internal_router_model + response_id: ReadOnly[str | None] # provider response id; request_id holds the proxy's per-call id class SpendLogsPayload(TypedDict): diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 7442d71bd96..4483eec72dc 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -128,6 +128,7 @@ def _get_spend_logs_metadata( litellm_call_id: str | None = None, autorouter_savings: float | None = None, router_metadata: SpendLogsRouterMetadata | None = None, + response_id: str | None = None, ) -> SpendLogsMetadata: if metadata is None: return SpendLogsMetadata( @@ -168,6 +169,7 @@ def _get_spend_logs_metadata( litellm_gateway_injected_cache=None, litellm_call_id=litellm_call_id, router_metadata=router_metadata, + response_id=response_id, ) verbose_proxy_logger.debug( "getting payload for SpendLogs, available keys in metadata: " + str(list(metadata.keys())) @@ -175,8 +177,13 @@ def _get_spend_logs_metadata( # Filter the metadata dictionary to include only the specified keys clean_metadata: Final = SpendLogsMetadata( - **{key: metadata.get(key) for key in SpendLogsMetadata.__annotations__ if key != "router_metadata"}, + **{ + key: metadata.get(key) + for key in SpendLogsMetadata.__annotations__ + if key not in ("router_metadata", "response_id") + }, router_metadata=router_metadata, + response_id=response_id, ) _raw_key: Final = clean_metadata.get("user_api_key") _trusted_hash: Final = metadata.get("user_api_key_hash") @@ -206,13 +213,38 @@ def _get_spend_logs_metadata( BATCH_COST_REQUEST_ID_SUFFIX: Final = "_batch_cost" +_RESPONSE_ID_KEYED_CALL_TYPES: Final = frozenset( + { + CallTypes.acreate_batch.value, + CallTypes.aretrieve_batch.value, + CallTypes.acreate_file.value, + } +) +"""Batch and file rows key off the object's own id so repeated polls of the same +object collapse into one row instead of billing it once per poll. Every other call +type keys off the proxy-generated per-call id: request_id is the LiteLLM_SpendLogs +primary key and the flush inserts with skip_duplicates, so keying off the provider's +response id silently drops every row after the first whenever a provider (commonly a +self-hosted OpenAI-compatible server) reuses completion ids.""" + def get_spend_logs_id(call_type: str, response_obj: dict, kwargs: dict) -> str | None: standard_logging_payload = kwargs.get("standard_logging_object") + standard_logging_id: Final = ( + standard_logging_payload.get("id") if isinstance(standard_logging_payload, dict) else None + ) candidate_ids: Final = ( - response_obj.get("id"), - standard_logging_payload.get("id") if isinstance(standard_logging_payload, dict) else None, - kwargs.get("litellm_call_id"), + ( + response_obj.get("id"), + standard_logging_id, + kwargs.get("litellm_call_id"), + ) + if call_type in _RESPONSE_ID_KEYED_CALL_TYPES + else ( + kwargs.get("litellm_call_id"), + standard_logging_id, + response_obj.get("id"), + ) ) resolved_id: Final = next( (candidate for candidate in candidate_ids if isinstance(candidate, str) and candidate), None @@ -490,6 +522,7 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs standard_logging_payload.get("autorouter_savings", None) if standard_logging_payload is not None else None ), litellm_call_id=litellm_call_id, + response_id=cast(str | None, response_obj_dict.get("id")), router_metadata=_get_router_metadata_for_spend_log( metadata=metadata, requested_model=_model_group, diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index a0dcbf802ef..05733e9ba0e 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -454,6 +454,7 @@ def test_ui_view_request_response_forbids_non_admin_without_db(client, monkeypat ignored_keys = [ "request_id", "metadata.litellm_call_id", + "metadata.response_id", "session_id", "startTime", "endTime", diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 9e5917637a8..d0fce3f839e 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -2834,9 +2834,9 @@ def test_get_logging_payload_failure_without_recovered_usage_is_zero(): def test_get_logging_payload_sets_litellm_call_id_for_correlation(): """LIT-3868: a successful spend log must carry the x-litellm-call-id (the - trace id) in its metadata, distinct from request_id, which stays the - provider response id. Without this there is no way to correlate a DB row - with its trace for a successful call. + trace id). request_id now keys off that same per-call id so duplicate + provider response ids cannot collapse rows, and the provider response id + moves to metadata.response_id so lookups by it still work. """ provider_response_id = "chatcmpl-e6e6f3e9-c392-404e-9a71-5361c79d8470" trace_call_id = "c6a77556-19ce-4406-b287-53f5fb4b2b55" @@ -2858,9 +2858,9 @@ def test_get_logging_payload_sets_litellm_call_id_for_correlation(): ) metadata = json.loads(payload["metadata"]) - assert payload["request_id"] == provider_response_id + assert payload["request_id"] == trace_call_id assert metadata["litellm_call_id"] == trace_call_id - assert metadata["litellm_call_id"] != payload["request_id"] + assert metadata["response_id"] == provider_response_id def test_get_logging_payload_litellm_call_id_falls_back_to_litellm_params(): @@ -3449,19 +3449,38 @@ def test_get_spend_logs_id_separates_distinct_batches_whose_bodies_were_both_red assert ids == ["batch_first_batch_cost", "batch_second_batch_cost"] -def test_get_spend_logs_id_prefers_the_response_id_over_the_standard_logging_id(): - """An unredacted response keeps deciding its own row key, so cache-hit ids and every - other call type behave exactly as they did before.""" +def test_get_spend_logs_id_prefers_the_per_call_id_for_completion_calls(): + """request_id is the primary key and the flush inserts with skip_duplicates, so a + completion row must key off the proxy-generated per-call id: a provider that reuses + completion ids (common on self-hosted OpenAI-compatible servers) would otherwise + silently drop every row after the first while budgets still get charged.""" assert ( get_spend_logs_id( "acompletion", {"id": "chatcmpl-from-response"}, {"litellm_call_id": "call-id-1", "standard_logging_object": {"id": "id-from-standard-payload"}}, ) - == "chatcmpl-from-response" + == "call-id-1" ) +def test_get_spend_logs_id_stays_unique_when_a_provider_reuses_completion_ids(): + """Two requests answered with the same provider response id must produce two + insertable rows. Observed against a live proxy with a self-hosted server returning a + fixed completion id: the second row was silently skipped by the duplicate-tolerant + flush while key and daily spend still incremented.""" + ids = [ + get_spend_logs_id("acompletion", {"id": "chatcmpl-reused"}, {"litellm_call_id": f"call-id-{index}"}) + for index in range(2) + ] + + assert ids == ["call-id-0", "call-id-1"] + + +def test_get_spend_logs_id_falls_back_to_the_response_id_without_a_per_call_id(): + assert get_spend_logs_id("acompletion", {"id": "chatcmpl-from-response"}, {}) == "chatcmpl-from-response" + + def test_batch_cost_row_does_not_collide_with_the_batch_creation_row(): """Creating a batch writes a row keyed by the batch's own id, so keying the cost row the same way makes the insert a duplicate of it. request_id is the primary key and the