diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index c127b3873a7..7d323fbf2bf 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3482,24 +3482,26 @@ class Logging(LiteLLMLoggingBaseClass): result, (ResponseCompletedEvent, ResponseIncompleteEvent, ResponseFailedEvent), ): - ## return unified Usage object - if isinstance(result.response.usage, ResponseAPIUsage): + response = result.response + usage = ( + response.get("usage") if isinstance(response, dict) else response.usage + ) + if isinstance(usage, ResponseAPIUsage): transformed_usage = ( ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( - result.response.usage + usage ) ) - # Set as dict instead of Usage object so model_dump() serializes it correctly - setattr( - result.response, - "usage", - ( - transformed_usage.model_dump() - if hasattr(transformed_usage, "model_dump") - else dict(transformed_usage) - ), + usage_dict = ( + transformed_usage.model_dump() + if hasattr(transformed_usage, "model_dump") + else dict(transformed_usage) ) - return result.response + if isinstance(response, dict): + response["usage"] = usage_dict + else: + setattr(response, "usage", usage_dict) + return response else: return None diff --git a/litellm/llms/chatgpt/responses/transformation.py b/litellm/llms/chatgpt/responses/transformation.py index 56b61b66c84..cf977343e69 100644 --- a/litellm/llms/chatgpt/responses/transformation.py +++ b/litellm/llms/chatgpt/responses/transformation.py @@ -11,6 +11,7 @@ from litellm.responses.sse_output_recovery import ( parse_sse_json_chunk, record_output_item_chunk, record_output_text_chunk, + record_output_text_delta_chunk, ) from litellm.types.llms.openai import ( ResponsesAPIResponse, @@ -174,6 +175,14 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): ) continue + if event_type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA: + record_output_text_delta_chunk( + parsed_chunk=parsed_chunk, + output_items=streamed_output_items, + text_only_items=text_only_output_items, + ) + continue + if event_type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE: record_output_text_chunk( parsed_chunk=parsed_chunk, diff --git a/litellm/responses/sse_output_recovery.py b/litellm/responses/sse_output_recovery.py index 5c18770a611..7bd3d4ceca5 100644 --- a/litellm/responses/sse_output_recovery.py +++ b/litellm/responses/sse_output_recovery.py @@ -64,19 +64,54 @@ def record_output_item_chunk( output_items[output_index] = item +def record_output_text_delta_chunk( + parsed_chunk: Dict[str, Any], + output_items: Dict[int, Dict[str, Any]], + text_only_items: Dict[int, Dict[str, Any]], +) -> None: + text_delta = parsed_chunk.get("delta") + if not isinstance(text_delta, str): + return + content_item = _get_or_create_text_content_item( + parsed_chunk=parsed_chunk, + output_items=output_items, + text_only_items=text_only_items, + ) + if content_item is None: + return + current_text = content_item.get("text") + if not isinstance(current_text, str): + current_text = "" + content_item["text"] = f"{current_text}{text_delta}" + + def record_output_text_chunk( parsed_chunk: Dict[str, Any], output_items: Dict[int, Dict[str, Any]], text_only_items: Dict[int, Dict[str, Any]], ) -> None: - """Record an OUTPUT_TEXT_DONE chunk as a synthetic message item in - ``text_only_items``. Real OUTPUT_ITEM_DONE events already captured in - ``output_items`` take precedence at the same ``output_index``. - """ text = parsed_chunk.get("text") if not isinstance(text, str): return + content_item = _get_or_create_text_content_item( + parsed_chunk=parsed_chunk, + output_items=output_items, + text_only_items=text_only_items, + ) + if content_item is None: + return + content_item["text"] = text + if parsed_chunk.get("annotations") is not None: + content_item["annotations"] = parsed_chunk["annotations"] + else: + content_item.setdefault("annotations", []) + +def _get_or_create_text_content_item( + parsed_chunk: Dict[str, Any], + output_items: Dict[int, Dict[str, Any]], + text_only_items: Dict[int, Dict[str, Any]], +) -> Optional[Dict[str, Any]]: try: output_index_raw = parsed_chunk.get("output_index") if output_index_raw is None: @@ -86,7 +121,7 @@ def record_output_text_chunk( output_index = len(text_only_items) if output_index in output_items: - return + return None item = text_only_items.get(output_index) if item is None: @@ -101,7 +136,7 @@ def record_output_text_chunk( content = item.setdefault("content", []) if not isinstance(content, list): - return + return None try: content_index_raw = parsed_chunk.get("content_index") @@ -112,7 +147,7 @@ def record_output_text_chunk( content_index = len(content) if content_index < 0 or content_index > _MAX_CONTENT_INDEX: - return + return None while len(content) <= content_index: content.append( @@ -129,8 +164,7 @@ def record_output_text_chunk( content[content_index] = content_item content_item["type"] = "output_text" - content_item["text"] = text - if parsed_chunk.get("annotations") is not None: - content_item["annotations"] = parsed_chunk["annotations"] - else: - content_item.setdefault("annotations", []) + if not isinstance(content_item.get("text"), str): + content_item["text"] = "" + content_item.setdefault("annotations", []) + return content_item diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index b64cb7c6905..2b09f814d4c 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -14,6 +14,11 @@ import time from litellm.constants import SENTRY_DENYLIST, SENTRY_PII_DENYLIST from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging from litellm.litellm_core_utils.litellm_logging import set_callbacks +from litellm.types.llms.openai import ( + ResponseAPIUsage, + ResponseCompletedEvent, + ResponsesAPIStreamEvents, +) from litellm.types.utils import ModelResponse, TextCompletionResponse @@ -2165,6 +2170,40 @@ def test_get_assembled_streaming_response_returns_result_for_streaming(): assert assembled is result +def test_get_assembled_streaming_response_handles_response_event_with_dict_response(): + import datetime + + logging_obj = _make_logging_obj(stream=True) + response = { + "id": "resp-1", + "object": "response", + "status": "completed", + "model": "gpt-5.5", + "usage": ResponseAPIUsage( + input_tokens=1, + output_tokens=2, + total_tokens=3, + ), + } + result = ResponseCompletedEvent.model_construct( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + response=response, + ) + + assembled = logging_obj._get_assembled_streaming_response( + result=result, + start_time=datetime.datetime.now(), + end_time=datetime.datetime.now(), + is_async=True, + streaming_chunks=[], + ) + + assert assembled is response + assert assembled["usage"]["prompt_tokens"] == 1 + assert assembled["usage"]["completion_tokens"] == 2 + assert assembled["usage"]["total_tokens"] == 3 + + def test_get_assembled_streaming_response_returns_none_for_non_streaming_text_completion(): """Non-streaming TextCompletionResponse should also return None.""" import datetime diff --git a/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py b/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py index 90a1c24bada..d78fa210849 100644 --- a/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py +++ b/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py @@ -248,6 +248,37 @@ class TestChatGPTResponsesAPITransformation: assert parsed.output_text == "Hello from stream!" + def test_chatgpt_non_stream_sse_response_recovers_output_text_deltas(self): + config = ChatGPTResponsesAPIConfig() + response_payload = { + "id": "resp_test", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "gpt-5.5", + } + sse_body = "\n".join( + [ + f"data: {json.dumps({'type': 'response.output_text.delta', 'output_index': 0, 'content_index': 0, 'item_id': 'msg_0', 'delta': 'Hel'})}", + f"data: {json.dumps({'type': 'response.output_text.delta', 'output_index': 0, 'content_index': 0, 'item_id': 'msg_0', 'delta': 'lo'})}", + f"data: {json.dumps({'type': 'response.completed', 'response': response_payload})}", + "data: [DONE]", + "", + ] + ) + raw_response = httpx.Response( + 200, headers={"content-type": "text/event-stream"}, text=sse_body + ) + logging_obj = MagicMock() + + parsed = config.transform_response_api_response( + model="chatgpt/gpt-5.5", + raw_response=raw_response, + logging_obj=logging_obj, + ) + + assert parsed.output_text == "Hello" + def test_chatgpt_non_stream_sse_recovers_whitespace_padded_chunks(self): """Chunks with leading whitespace before `data:` must still parse.