diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index df5365017ac..7dc97a1a5d3 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -3119,6 +3119,23 @@ class BaseLLMHTTPHandler: logging_obj=logging_obj, ) + @staticmethod + def _finalize_retrieved_response_logging( + logging_obj: LiteLLMLoggingObj, + response: ResponsesAPIResponse, + ) -> None: + """Attribute a retrieved (polled) response's spend to its model and zero + the cost of ``queued`` / ``in_progress`` polls that carry no usage.""" + model = response.model + if model: + if not logging_obj.model: + logging_obj.model = model + if not logging_obj.model_call_details.get("model"): + logging_obj.model_call_details["model"] = model + + if response.status in ("queued", "in_progress"): + response._hidden_params["response_cost"] = 0.0 + def get_responses( self, response_id: str, @@ -3195,10 +3212,15 @@ class BaseLLMHTTPHandler: provider_config=responses_api_provider_config, ) - return responses_api_provider_config.transform_get_response_api_response( + transformed_response = responses_api_provider_config.transform_get_response_api_response( raw_response=response, logging_obj=logging_obj, ) + self._finalize_retrieved_response_logging( + logging_obj=logging_obj, + response=transformed_response, + ) + return transformed_response async def async_get_responses( self, @@ -3269,10 +3291,15 @@ class BaseLLMHTTPHandler: provider_config=responses_api_provider_config, ) - return responses_api_provider_config.transform_get_response_api_response( + transformed_response = responses_api_provider_config.transform_get_response_api_response( raw_response=response, logging_obj=logging_obj, ) + self._finalize_retrieved_response_logging( + logging_obj=logging_obj, + response=transformed_response, + ) + return transformed_response ##################################################################### ################ LIST RESPONSES INPUT ITEMS HANDLER ########################### diff --git a/tests/test_litellm/integrations/test_responses_background_cost.py b/tests/test_litellm/integrations/test_responses_background_cost.py index 0d4218f2137..a5648c50a65 100644 --- a/tests/test_litellm/integrations/test_responses_background_cost.py +++ b/tests/test_litellm/integrations/test_responses_background_cost.py @@ -5,11 +5,16 @@ Integration tests for responses API background cost tracking import asyncio import os from datetime import datetime +from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, Mock, patch +import httpx import pytest +import litellm from litellm.constants import MAX_OBJECTS_PER_POLL_CYCLE +from litellm.integrations.custom_logger import CustomLogger +from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse @@ -561,3 +566,154 @@ class TestCheckResponsesCost: if c.kwargs.get("where", {}).get("id") is not None ] assert len(completion_calls) == 0 + + +def _make_responses_api_response(status, with_usage): + return ResponsesAPIResponse( + id="resp_finalize_test", + object="response", + created_at=int(datetime.now().timestamp()), + status=status, + model="gpt-5", + output=[], + usage=( + ResponseAPIUsage( + input_tokens=1000, + output_tokens=2000, + total_tokens=3000, + ) + if with_usage + else None + ), + ) + + +class TestFinalizeRetrievedResponseLogging: + """Unit tests for BaseLLMHTTPHandler._finalize_retrieved_response_logging""" + + def test_completed_response_stamps_model_when_missing(self): + logging_obj = SimpleNamespace(model=None, model_call_details={"model": None}) + response = _make_responses_api_response("completed", with_usage=True) + response._hidden_params = {} + + BaseLLMHTTPHandler._finalize_retrieved_response_logging( + logging_obj=logging_obj, response=response + ) + + assert logging_obj.model == "gpt-5" + assert logging_obj.model_call_details["model"] == "gpt-5" + assert "response_cost" not in response._hidden_params + + def test_existing_model_is_not_overwritten(self): + logging_obj = SimpleNamespace( + model="already-set", model_call_details={"model": "already-set"} + ) + response = _make_responses_api_response("completed", with_usage=True) + response._hidden_params = {} + + BaseLLMHTTPHandler._finalize_retrieved_response_logging( + logging_obj=logging_obj, response=response + ) + + assert logging_obj.model == "already-set" + assert logging_obj.model_call_details["model"] == "already-set" + + @pytest.mark.parametrize("status", ["queued", "in_progress"]) + def test_non_terminal_poll_is_zero_cost(self, status): + logging_obj = SimpleNamespace(model=None, model_call_details={"model": None}) + response = _make_responses_api_response(status, with_usage=True) + response._hidden_params = {} + + BaseLLMHTTPHandler._finalize_retrieved_response_logging( + logging_obj=logging_obj, response=response + ) + + assert response._hidden_params["response_cost"] == 0.0 + assert logging_obj.model == "gpt-5" + + +class _CaptureLogger(CustomLogger): + def __init__(self): + self.events = [] + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + slp = kwargs.get("standard_logging_object") or {} + self.events.append( + { + "model": slp.get("model"), + "response_cost": slp.get("response_cost"), + "status": getattr(response_obj, "status", None), + } + ) + + +def _fake_get_response(status, with_usage): + payload = { + "id": "resp_finalize_test", + "object": "response", + "created_at": 1, + "status": status, + "model": "gpt-5", + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "output": [], + "usage": ( + { + "input_tokens": 1000, + "output_tokens": 2000, + "total_tokens": 3000, + "output_tokens_details": {"reasoning_tokens": 1500}, + } + if with_usage + else None + ), + } + + request = httpx.Request("GET", "https://api.openai.com/v1/responses/resp_finalize_test") + return httpx.Response(status_code=200, json=payload, request=request) + + +async def _capture_aget_responses(status, with_usage): + capture = _CaptureLogger() + original_callbacks = litellm.callbacks + litellm.callbacks = [capture] + try: + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.get", + return_value=_fake_get_response(status, with_usage), + ): + await litellm.aget_responses( + response_id="resp_finalize_test", + custom_llm_provider="openai", + api_key="sk-fake", + ) + for _ in range(30): + if capture.events: + break + await asyncio.sleep(0.1) + finally: + litellm.callbacks = original_callbacks + + assert capture.events, "success logging callback never fired for the retrieved response" + return capture.events + + +class TestRetrievedResponseCostAttribution: + """End-to-end regression tests for spend logging on the responses retrieve path""" + + @pytest.mark.asyncio + async def test_completed_retrieve_attributes_cost_to_model(self): + events = await _capture_aget_responses("completed", with_usage=True) + + assert len(events) == 1 + assert events[0]["model"] == "gpt-5" + assert events[0]["response_cost"] is not None + assert events[0]["response_cost"] > 0 + + @pytest.mark.asyncio + async def test_in_progress_poll_is_not_billed(self): + events = await _capture_aget_responses("in_progress", with_usage=True) + + assert len(events) == 1 + assert events[0]["response_cost"] == 0.0