From 7121e64db4903dc7b832b25f79f1f5f048b73a36 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 16 Sep 2026 03:13:53 +0000 Subject: [PATCH] fix(responses): type the dict terminal response so the estimated usage is billed Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/responses/streaming_iterator.py | 42 +++++++++++-------- .../responses/test_streaming_iterator.py | 15 +++++-- 2 files changed, 35 insertions(+), 22 deletions(-) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 10ddf07ac7e..3004337f9d1 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -429,25 +429,31 @@ class BaseResponsesAPIStreamingIterator: ): self.completed_response = openai_responses_api_chunk _response_obj: Final[object] = getattr(openai_responses_api_chunk, "response", None) - if _chunk_type in ( - openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED, - openai_types.ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, + _typed_response: Final[ResponsesAPIResponse | None] = ( + ResponsesAPIResponse.model_construct(**_response_obj) # pyright: ignore[reportUnknownArgumentType] # the model_constructed terminal event leaves response as an untyped dict + if isinstance(_response_obj, dict) + else _response_obj + if isinstance(_response_obj, ResponsesAPIResponse) + else None + ) + if ( + _typed_response is not None + and _chunk_type + in ( + openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + openai_types.ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, + ) + and _typed_response.usage is None ): - if isinstance(_response_obj, ResponsesAPIResponse) and _response_obj.usage is None: - _response_obj.usage = _estimate_usage_safely( - self.model or "", - self.request_data.get("input"), - self.request_data, - self._generated_content + self._generated_tool_arguments, - ) - elif isinstance(_response_obj, dict) and _response_obj.get("usage") is None: # pyright: ignore[reportUnknownMemberType] # the model_constructed terminal event leaves response as an untyped dict - _response_obj["usage"] = _estimate_usage_safely( - self.model or "", - self.request_data.get("input"), - self.request_data, - self._generated_content + self._generated_tool_arguments, - ) - _stamp_responses_usage_cost(getattr(openai_responses_api_chunk, "response", None), self.logging_obj) + _typed_response.usage = _estimate_usage_safely( + self.model or "", + self.request_data.get("input"), + self.request_data, + self._generated_content + self._generated_tool_arguments, + ) + if _typed_response is not None and _typed_response is not _response_obj: + openai_responses_api_chunk.response = _typed_response # pyright: ignore[reportAttributeAccessIssue] # reached only on the dict path, which only response-carrying terminal events produce + _stamp_responses_usage_cost(_typed_response, self.logging_obj) if _chunk_type == openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED: self._handle_logging_failed_response() diff --git a/tests/test_litellm/responses/test_streaming_iterator.py b/tests/test_litellm/responses/test_streaming_iterator.py index 8cf9556761e..30a7c4faaed 100644 --- a/tests/test_litellm/responses/test_streaming_iterator.py +++ b/tests/test_litellm/responses/test_streaming_iterator.py @@ -852,9 +852,10 @@ async def test_completed_event_without_usage_counts_tool_input_deltas(tool_delta @pytest.mark.asyncio -async def test_completed_event_with_a_dict_response_still_gets_the_usage_estimate(): +async def test_completed_event_with_a_dict_response_is_typed_and_billed(): """transform_streaming_response can model_construct a terminal event whose - response stays a plain dict; the estimate must fill it without raising.""" + response stays a plain dict; the iterator must type it so the estimated + usage reaches the cost stamping path.""" dict_response: Final = { "id": "resp_dict", "model": "gpt-4o-mini", @@ -874,12 +875,14 @@ async def test_completed_event_with_a_dict_response_still_gets_the_usage_estimat config: Final = Mock(spec=BaseResponsesAPIConfig) config.transform_streaming_response.side_effect = _transform + logging_obj: Final = _logging_obj_stub() + logging_obj._response_cost_calculator.return_value = 0.000704 iterator: Final = _make_iterator( sse_events=[ _sse_event({"type": "response.output_text.delta", "delta": "hello world"}), _sse_event({"type": "response.completed", "response": {}}), ], - logging_obj=_logging_obj_stub(), + logging_obj=logging_obj, config=config, request_data={"input": "count these input tokens please"}, ) @@ -887,7 +890,11 @@ async def test_completed_event_with_a_dict_response_still_gets_the_usage_estimat async for _ in iterator: pass - usage: Final = iterator.completed_response.response["usage"] + completed_response: Final = iterator.completed_response.response + assert isinstance(completed_response, ResponsesAPIResponse) + usage: Final = completed_response.usage assert usage is not None assert usage.input_tokens > 0 assert usage.output_tokens > 0 + assert usage.cost == pytest.approx(0.000704) + logging_obj._response_cost_calculator.assert_any_call(result=completed_response)