scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-16 23:41:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 102

fix(responses): type the dict terminal response so the estimated usage is billed

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
kerry 2026-09-16 03:13:53 +00:00
parent 7680d3de86
commit 7121e64db4
2 changed files with 35 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
litellm/responses/streaming_iterator.py
View file

@ -429,25 +429,31 @@ class BaseResponsesAPIStreamingIterator:
):
self.completed_response = openai_responses_api_chunk
_response_obj: Final[object] = getattr(openai_responses_api_chunk, "response", None)
if _chunk_type in (
openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED,
openai_types.ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE,
_typed_response: Final[ResponsesAPIResponse | None] = (
ResponsesAPIResponse.model_construct(**_response_obj) # pyright: ignore[reportUnknownArgumentType] # the model_constructed terminal event leaves response as an untyped dict
if isinstance(_response_obj, dict)
else _response_obj
if isinstance(_response_obj, ResponsesAPIResponse)
else None
)
if (
_typed_response is not None
and _chunk_type
in (
openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED,
openai_types.ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE,
)
and _typed_response.usage is None
):
if isinstance(_response_obj, ResponsesAPIResponse) and _response_obj.usage is None:
_response_obj.usage = _estimate_usage_safely(
self.model or "",
self.request_data.get("input"),
self.request_data,
self._generated_content + self._generated_tool_arguments,
)
elif isinstance(_response_obj, dict) and _response_obj.get("usage") is None: # pyright: ignore[reportUnknownMemberType] # the model_constructed terminal event leaves response as an untyped dict
_response_obj["usage"] = _estimate_usage_safely(
self.model or "",
self.request_data.get("input"),
self.request_data,
self._generated_content + self._generated_tool_arguments,
)
_stamp_responses_usage_cost(getattr(openai_responses_api_chunk, "response", None), self.logging_obj)
_typed_response.usage = _estimate_usage_safely(
self.model or "",
self.request_data.get("input"),
self.request_data,
self._generated_content + self._generated_tool_arguments,
)
if _typed_response is not None and _typed_response is not _response_obj:
openai_responses_api_chunk.response = _typed_response # pyright: ignore[reportAttributeAccessIssue] # reached only on the dict path, which only response-carrying terminal events produce
_stamp_responses_usage_cost(_typed_response, self.logging_obj)
if _chunk_type == openai_types.ResponsesAPIStreamEvents.RESPONSE_FAILED:
self._handle_logging_failed_response()

15
tests/test_litellm/responses/test_streaming_iterator.py
View file

@ -852,9 +852,10 @@ async def test_completed_event_without_usage_counts_tool_input_deltas(tool_delta
@pytest.mark.asyncio
async def test_completed_event_with_a_dict_response_still_gets_the_usage_estimate():
async def test_completed_event_with_a_dict_response_is_typed_and_billed():
"""transform_streaming_response can model_construct a terminal event whose
response stays a plain dict; the estimate must fill it without raising."""
response stays a plain dict; the iterator must type it so the estimated
usage reaches the cost stamping path."""
dict_response: Final = {
"id": "resp_dict",
"model": "gpt-4o-mini",
@ -874,12 +875,14 @@ async def test_completed_event_with_a_dict_response_still_gets_the_usage_estimat
config: Final = Mock(spec=BaseResponsesAPIConfig)
config.transform_streaming_response.side_effect = _transform
logging_obj: Final = _logging_obj_stub()
logging_obj._response_cost_calculator.return_value = 0.000704
iterator: Final = _make_iterator(
sse_events=[
_sse_event({"type": "response.output_text.delta", "delta": "hello world"}),
_sse_event({"type": "response.completed", "response": {}}),
],
logging_obj=_logging_obj_stub(),
logging_obj=logging_obj,
config=config,
request_data={"input": "count these input tokens please"},
)
@ -887,7 +890,11 @@ async def test_completed_event_with_a_dict_response_still_gets_the_usage_estimat
async for _ in iterator:
pass
usage: Final = iterator.completed_response.response["usage"]
completed_response: Final = iterator.completed_response.response
assert isinstance(completed_response, ResponsesAPIResponse)
usage: Final = completed_response.usage
assert usage is not None
assert usage.input_tokens > 0
assert usage.output_tokens > 0
assert usage.cost == pytest.approx(0.000704)
logging_obj._response_cost_calculator.assert_any_call(result=completed_response)