scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-04 02:31:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 70

fix chatgpt responses non-stream output

Browse source
This commit is contained in:
qazz92 2026-06-01 02:35:37 +09:00
parent 28c0d8579b
commit 4c78b1718e
5 changed files with 140 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
litellm/litellm_core_utils/litellm_logging.py
View file

@ -3482,24 +3482,26 @@ class Logging(LiteLLMLoggingBaseClass):
result,
(ResponseCompletedEvent, ResponseIncompleteEvent, ResponseFailedEvent),
):
## return unified Usage object
if isinstance(result.response.usage, ResponseAPIUsage):
response = result.response
usage = (
response.get("usage") if isinstance(response, dict) else response.usage
)
if isinstance(usage, ResponseAPIUsage):
transformed_usage = (
ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
result.response.usage
usage
)
)
# Set as dict instead of Usage object so model_dump() serializes it correctly
setattr(
result.response,
"usage",
(
transformed_usage.model_dump()
if hasattr(transformed_usage, "model_dump")
else dict(transformed_usage)
),
usage_dict = (
transformed_usage.model_dump()
if hasattr(transformed_usage, "model_dump")
else dict(transformed_usage)
)
return result.response
if isinstance(response, dict):
response["usage"] = usage_dict
else:
setattr(response, "usage", usage_dict)
return response
else:
return None

9
litellm/llms/chatgpt/responses/transformation.py
View file

@ -11,6 +11,7 @@ from litellm.responses.sse_output_recovery import (
parse_sse_json_chunk,
record_output_item_chunk,
record_output_text_chunk,
record_output_text_delta_chunk,
)
from litellm.types.llms.openai import (
ResponsesAPIResponse,
@ -174,6 +175,14 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig):
)
continue
if event_type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA:
record_output_text_delta_chunk(
parsed_chunk=parsed_chunk,
output_items=streamed_output_items,
text_only_items=text_only_output_items,
)
continue
if event_type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE:
record_output_text_chunk(
parsed_chunk=parsed_chunk,

58
litellm/responses/sse_output_recovery.py
View file

@ -64,19 +64,54 @@ def record_output_item_chunk(
output_items[output_index] = item
def record_output_text_delta_chunk(
parsed_chunk: Dict[str, Any],
output_items: Dict[int, Dict[str, Any]],
text_only_items: Dict[int, Dict[str, Any]],
) -> None:
text_delta = parsed_chunk.get("delta")
if not isinstance(text_delta, str):
return
content_item = _get_or_create_text_content_item(
parsed_chunk=parsed_chunk,
output_items=output_items,
text_only_items=text_only_items,
)
if content_item is None:
return
current_text = content_item.get("text")
if not isinstance(current_text, str):
current_text = ""
content_item["text"] = f"{current_text}{text_delta}"
def record_output_text_chunk(
parsed_chunk: Dict[str, Any],
output_items: Dict[int, Dict[str, Any]],
text_only_items: Dict[int, Dict[str, Any]],
) -> None:
"""Record an OUTPUT_TEXT_DONE chunk as a synthetic message item in
``text_only_items``. Real OUTPUT_ITEM_DONE events already captured in
``output_items`` take precedence at the same ``output_index``.
"""
text = parsed_chunk.get("text")
if not isinstance(text, str):
return
content_item = _get_or_create_text_content_item(
parsed_chunk=parsed_chunk,
output_items=output_items,
text_only_items=text_only_items,
)
if content_item is None:
return
content_item["text"] = text
if parsed_chunk.get("annotations") is not None:
content_item["annotations"] = parsed_chunk["annotations"]
else:
content_item.setdefault("annotations", [])
def _get_or_create_text_content_item(
parsed_chunk: Dict[str, Any],
output_items: Dict[int, Dict[str, Any]],
text_only_items: Dict[int, Dict[str, Any]],
) -> Optional[Dict[str, Any]]:
try:
output_index_raw = parsed_chunk.get("output_index")
if output_index_raw is None:
@ -86,7 +121,7 @@ def record_output_text_chunk(
output_index = len(text_only_items)
if output_index in output_items:
return
return None
item = text_only_items.get(output_index)
if item is None:
@ -101,7 +136,7 @@ def record_output_text_chunk(
content = item.setdefault("content", [])
if not isinstance(content, list):
return
return None
try:
content_index_raw = parsed_chunk.get("content_index")
@ -112,7 +147,7 @@ def record_output_text_chunk(
content_index = len(content)
if content_index < 0 or content_index > _MAX_CONTENT_INDEX:
return
return None
while len(content) <= content_index:
content.append(
@ -129,8 +164,7 @@ def record_output_text_chunk(
content[content_index] = content_item
content_item["type"] = "output_text"
content_item["text"] = text
if parsed_chunk.get("annotations") is not None:
content_item["annotations"] = parsed_chunk["annotations"]
else:
content_item.setdefault("annotations", [])
if not isinstance(content_item.get("text"), str):
content_item["text"] = ""
content_item.setdefault("annotations", [])
return content_item

39
tests/test_litellm/litellm_core_utils/test_litellm_logging.py
View file

@ -14,6 +14,11 @@ import time
from litellm.constants import SENTRY_DENYLIST, SENTRY_PII_DENYLIST
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
from litellm.litellm_core_utils.litellm_logging import set_callbacks
from litellm.types.llms.openai import (
ResponseAPIUsage,
ResponseCompletedEvent,
ResponsesAPIStreamEvents,
)
from litellm.types.utils import ModelResponse, TextCompletionResponse
@ -2165,6 +2170,40 @@ def test_get_assembled_streaming_response_returns_result_for_streaming():
assert assembled is result
def test_get_assembled_streaming_response_handles_response_event_with_dict_response():
import datetime
logging_obj = _make_logging_obj(stream=True)
response = {
"id": "resp-1",
"object": "response",
"status": "completed",
"model": "gpt-5.5",
"usage": ResponseAPIUsage(
input_tokens=1,
output_tokens=2,
total_tokens=3,
),
}
result = ResponseCompletedEvent.model_construct(
type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED,
response=response,
)
assembled = logging_obj._get_assembled_streaming_response(
result=result,
start_time=datetime.datetime.now(),
end_time=datetime.datetime.now(),
is_async=True,
streaming_chunks=[],
)
assert assembled is response
assert assembled["usage"]["prompt_tokens"] == 1
assert assembled["usage"]["completion_tokens"] == 2
assert assembled["usage"]["total_tokens"] == 3
def test_get_assembled_streaming_response_returns_none_for_non_streaming_text_completion():
"""Non-streaming TextCompletionResponse should also return None."""
import datetime

31
tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py
View file

@ -248,6 +248,37 @@ class TestChatGPTResponsesAPITransformation:
assert parsed.output_text == "Hello from stream!"
def test_chatgpt_non_stream_sse_response_recovers_output_text_deltas(self):
config = ChatGPTResponsesAPIConfig()
response_payload = {
"id": "resp_test",
"object": "response",
"created_at": 1700000000,
"status": "completed",
"model": "gpt-5.5",
}
sse_body = "\n".join(
[
f"data: {json.dumps({'type': 'response.output_text.delta', 'output_index': 0, 'content_index': 0, 'item_id': 'msg_0', 'delta': 'Hel'})}",
f"data: {json.dumps({'type': 'response.output_text.delta', 'output_index': 0, 'content_index': 0, 'item_id': 'msg_0', 'delta': 'lo'})}",
f"data: {json.dumps({'type': 'response.completed', 'response': response_payload})}",
"data: [DONE]",
"",
]
)
raw_response = httpx.Response(
200, headers={"content-type": "text/event-stream"}, text=sse_body
)
logging_obj = MagicMock()
parsed = config.transform_response_api_response(
model="chatgpt/gpt-5.5",
raw_response=raw_response,
logging_obj=logging_obj,
)
assert parsed.output_text == "Hello"
def test_chatgpt_non_stream_sse_recovers_whitespace_padded_chunks(self):
"""Chunks with leading whitespace before `data:` must still parse.