diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index da3b9184edb..221ca7e2a16 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -583,6 +583,126 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): return choices + @classmethod + def _recover_output_items_from_raw_sse( + cls, raw_sse: Optional[str] + ) -> List[Dict[str, Any]]: + if not raw_sse or not isinstance(raw_sse, str): + return [] + + from litellm.utils import CustomStreamWrapper + + recovered_output_items: Dict[int, Dict[str, Any]] = {} + recovered_text_only_items: Dict[int, Dict[str, Any]] = {} + + for chunk in raw_sse.splitlines(): + stripped_chunk = ( + CustomStreamWrapper._strip_sse_data_from_chunk(chunk.strip()) or "" + ).strip() + if ( + not stripped_chunk + or stripped_chunk == "[DONE]" + or stripped_chunk.startswith("event:") + ): + continue + + try: + parsed_chunk = json.loads(stripped_chunk) + except json.JSONDecodeError: + continue + + if not isinstance(parsed_chunk, dict): + continue + + event_type = parsed_chunk.get("type") + if event_type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED: + response_payload = parsed_chunk.get("response") + if isinstance(response_payload, dict): + response_output = response_payload.get("output") + if isinstance(response_output, list) and len(response_output) > 0: + return cast(List[Dict[str, Any]], response_output) + continue + + if event_type == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE: + item = parsed_chunk.get("item") + if not isinstance(item, dict): + continue + try: + output_index = int(parsed_chunk.get("output_index")) + except (TypeError, ValueError): + output_index = len(recovered_output_items) + recovered_output_items[output_index] = item + continue + + if event_type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE: + text = parsed_chunk.get("text") + if not isinstance(text, str): + continue + + try: + output_index = int(parsed_chunk.get("output_index")) + except (TypeError, ValueError): + output_index = len(recovered_text_only_items) + + item = recovered_output_items.get( + output_index + ) or recovered_text_only_items.get(output_index) + if item is None: + item = { + "type": "message", + "id": parsed_chunk.get("item_id") or f"msg_{output_index}", + "role": "assistant", + "status": "completed", + "content": [], + } + recovered_text_only_items[output_index] = item + + content = item.setdefault("content", []) + if not isinstance(content, list): + continue + + try: + content_index = int(parsed_chunk.get("content_index")) + except (TypeError, ValueError): + content_index = len(content) + + while len(content) <= content_index: + content.append( + { + "type": "output_text", + "text": "", + "annotations": [], + } + ) + + content_item = content[content_index] + if not isinstance(content_item, dict): + content_item = {} + content[content_index] = content_item + + content_item["type"] = "output_text" + content_item["text"] = text + if parsed_chunk.get("annotations") is not None: + content_item["annotations"] = parsed_chunk["annotations"] + else: + content_item.setdefault("annotations", []) + + if recovered_output_items: + return [item for _, item in sorted(recovered_output_items.items())] + + if recovered_text_only_items: + return [item for _, item in sorted(recovered_text_only_items.items())] + + return [] + + @classmethod + def _recover_output_items_from_logging( + cls, logging_obj: "LiteLLMLoggingObj" + ) -> List[Dict[str, Any]]: + model_call_details = getattr(logging_obj, "model_call_details", {}) or {} + original_response = model_call_details.get("original_response") + return cls._recover_output_items_from_raw_sse(original_response) + def transform_response( # noqa: PLR0915 self, model: str, @@ -607,9 +727,21 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if raw_response.error is not None: raise ValueError(f"Error in response: {raw_response.error}") + output_items = raw_response.output + if len(output_items) == 0: + recovered_output_items = self._recover_output_items_from_logging( + logging_obj + ) + if recovered_output_items: + output_items = recovered_output_items + verbose_logger.warning( + "Recovered empty Responses API output from raw SSE for model=%s", + model, + ) + # Convert response output to choices using the static helper choices = self._convert_response_output_to_choices( - output_items=raw_response.output, + output_items=output_items, handle_raw_dict_callback=self._handle_raw_dict_response_item, ) @@ -623,7 +755,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ) else: raise ValueError( - f"Unknown items in responses API response: {raw_response.output}" + f"Unknown items in responses API response: {output_items}" ) setattr(model_response, "choices", choices) diff --git a/litellm/llms/chatgpt/responses/transformation.py b/litellm/llms/chatgpt/responses/transformation.py index 66acd933416..4874f08d05e 100644 --- a/litellm/llms/chatgpt/responses/transformation.py +++ b/litellm/llms/chatgpt/responses/transformation.py @@ -1,5 +1,5 @@ import json -from typing import Any, Optional +from typing import Any, Dict, Optional from litellm.constants import STREAM_SSE_DONE_STRING from litellm.exceptions import AuthenticationError @@ -134,6 +134,7 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): completed_response = None error_message = None + streamed_output_items: Dict[int, dict] = {} for chunk in body_text.splitlines(): stripped_chunk = CustomStreamWrapper._strip_sse_data_from_chunk(chunk) if not stripped_chunk: @@ -150,10 +151,24 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): if not isinstance(parsed_chunk, dict): continue event_type = parsed_chunk.get("type") + if event_type == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE: + item = parsed_chunk.get("item") + output_index = parsed_chunk.get("output_index") + if isinstance(item, dict): + try: + index = int(output_index) + except (TypeError, ValueError): + index = len(streamed_output_items) + streamed_output_items[index] = item + continue if event_type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED: response_payload = parsed_chunk.get("response") if isinstance(response_payload, dict): response_payload = dict(response_payload) + if not response_payload.get("output") and streamed_output_items: + response_payload["output"] = [ + item for _, item in sorted(streamed_output_items.items()) + ] if "created_at" in response_payload: response_payload["created_at"] = _safe_convert_created_field( response_payload["created_at"] diff --git a/litellm/router.py b/litellm/router.py index 7512ee387dc..749f5fc762c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -7076,6 +7076,18 @@ class Router: _shared_model_info = { k: v for k, v in _model_info.items() if k not in _custom_pricing_fields } + _existing_shared_mode = ( + cast(Optional[dict], litellm.model_cost.get(_model_name, {})) or {} + ).get("mode") + if ( + _existing_shared_mode is not None + and _shared_model_info.get("mode") != _existing_shared_mode + ): + # Keep the built-in bridge mode stable for shared backend keys. + # Multiple aliases can point at the same provider/model backend, + # but their deployment-level overrides should not downgrade the + # backend from responses -> chat via last-write-wins registration. + _shared_model_info.pop("mode", None) litellm.register_model( model_cost={ _model_name: _shared_model_info, diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index e40543e01a0..2c223a43a16 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -508,6 +508,136 @@ and I learn to carry this small calm home.""" print("✓ transform_response correctly handled reasoning items and output messages") +def _make_empty_responses_api_response(model: str = "gpt-5.4"): + from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse + + return ResponsesAPIResponse( + id="resp_from_stream", + created_at=1760144904, + error=None, + incomplete_details=None, + instructions=None, + metadata={}, + model=model, + object="response", + output=[], + parallel_tool_calls=True, + temperature=1.0, + tool_choice="auto", + tools=[], + top_p=1.0, + max_output_tokens=None, + previous_response_id=None, + reasoning={"effort": "low", "summary": "detailed"}, + status="completed", + text={"format": {"type": "text"}, "verbosity": "medium"}, + truncation="disabled", + usage=ResponseAPIUsage( + input_tokens=1, + input_tokens_details=None, + output_tokens=1, + output_tokens_details=None, + total_tokens=2, + cost=None, + ), + user=None, + store=True, + background=False, + billing={"payer": "developer"}, + max_tool_calls=None, + prompt_cache_key=None, + safety_identifier=None, + service_tier="default", + top_logprobs=0, + ) + + +def _make_empty_model_response(): + from litellm.types.utils import ModelResponse, Usage + + return ModelResponse( + id="chatcmpl-test-recovered", + created=1760144904, + model=None, + object="chat.completion", + system_fingerprint=None, + choices=[], + usage=Usage(completion_tokens=0, prompt_tokens=0, total_tokens=0), + ) + + +def test_transform_response_recovers_empty_output_from_raw_sse(): + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + + handler = LiteLLMResponsesTransformationHandler() + raw_sse = "\n".join( + [ + 'data: {"type":"response.output_text.done","output_index":0,"content_index":0,"item_id":"msg_from_stream","text":"Recovered from SSE"}', + 'data: {"type":"response.completed","response":{"id":"resp_from_stream","object":"response","created_at":1760144904,"status":"completed","model":"gpt-5.4","output":[]}}', + "data: [DONE]", + "", + ] + ) + + raw_response = _make_empty_responses_api_response() + model_response = _make_empty_model_response() + logging_obj = Mock() + logging_obj.model_call_details = {"original_response": raw_sse} + + result = handler.transform_response( + model="gpt-5.4", + raw_response=raw_response, + model_response=model_response, + logging_obj=logging_obj, + request_data={"model": "gpt-5.4"}, + messages=[{"role": "user", "content": "Reply with exactly: ok"}], + optional_params={}, + litellm_params={}, + encoding=Mock(), + ) + + assert len(result.choices) == 1 + assert result.choices[0].message.content == "Recovered from SSE" + + +def test_transform_response_recovers_output_item_done_from_raw_sse(): + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + + handler = LiteLLMResponsesTransformationHandler() + raw_sse = "\n".join( + [ + 'data: {"type":"response.output_item.done","output_index":0,"item":{"type":"message","id":"msg_from_item","role":"assistant","status":"completed","content":[{"type":"output_text","text":"Recovered from output item","annotations":[]}]}}', + 'data: {"type":"response.completed","response":{"id":"resp_from_stream","object":"response","created_at":1760144904,"status":"completed","model":"gpt-5.4","output":[]}}', + "data: [DONE]", + "", + ] + ) + + raw_response = _make_empty_responses_api_response() + model_response = _make_empty_model_response() + logging_obj = Mock() + logging_obj.model_call_details = {"original_response": raw_sse} + + result = handler.transform_response( + model="gpt-5.4", + raw_response=raw_response, + model_response=model_response, + logging_obj=logging_obj, + request_data={"model": "gpt-5.4"}, + messages=[{"role": "user", "content": "Reply with exactly: ok"}], + optional_params={}, + litellm_params={}, + encoding=Mock(), + ) + + assert len(result.choices) == 1 + assert result.choices[0].message.content == "Recovered from output item" + + def test_convert_tools_to_responses_format(): from litellm.completion_extras.litellm_responses_transformation.transformation import ( LiteLLMResponsesTransformationHandler, diff --git a/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py b/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py index 2498946bb5c..692a7e257dc 100644 --- a/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py +++ b/tests/test_litellm/llms/chatgpt/responses/test_chatgpt_responses_transformation.py @@ -14,6 +14,7 @@ import pytest sys.path.insert(0, os.path.abspath("../../../../..")) +from litellm.llms.openai.common_utils import OpenAIError from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager @@ -201,3 +202,74 @@ class TestChatGPTResponsesAPITransformation: ) assert parsed.output_text == "Hello!" + + @pytest.mark.parametrize( + ("model_name", "response_model"), + [ + ("chatgpt/gpt-5.2-codex", "gpt-5.2-codex"), + ("chatgpt/gpt-5.3-codex", "gpt-5.3-codex"), + ], + ) + def test_chatgpt_non_stream_sse_response_recovers_output_items( + self, model_name: str, response_model: str + ): + config = ChatGPTResponsesAPIConfig() + output_item = { + "type": "message", + "id": "msg_test", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "Hello from output item"}], + } + sse_body = "\n".join( + [ + f"data: {json.dumps({'type': 'response.output_item.done', 'output_index': 0, 'item': output_item})}", + f"data: {json.dumps({'type': 'response.completed', 'response': {'id': 'resp_test', 'object': 'response', 'created_at': 1700000000, 'status': 'completed', 'model': response_model, 'output': []}})}", + "data: [DONE]", + "", + ] + ) + raw_response = httpx.Response( + 200, headers={"content-type": "text/event-stream"}, text=sse_body + ) + logging_obj = MagicMock() + + parsed = config.transform_response_api_response( + model=model_name, + raw_response=raw_response, + logging_obj=logging_obj, + ) + + assert parsed.output == [output_item] + assert parsed.output_text == "Hello from output item" + + @pytest.mark.parametrize( + ("model_name", "response_model"), + [ + ("chatgpt/gpt-5.2-codex", "gpt-5.2-codex"), + ("chatgpt/gpt-5.3-codex", "gpt-5.3-codex"), + ], + ) + def test_chatgpt_non_stream_sse_response_raises_without_completed_response( + self, model_name: str, response_model: str + ): + config = ChatGPTResponsesAPIConfig() + sse_body = "\n".join( + [ + f"data: {json.dumps({'type': 'response.output_item.done', 'output_index': 0, 'item': {'type': 'message', 'role': 'assistant', 'content': [{'type': 'output_text', 'text': 'Hello'}]}})}", + f"data: {json.dumps({'type': 'response.failed', 'response': {'error': {'message': f'{response_model} failed'}}})}", + "data: [DONE]", + "", + ] + ) + raw_response = httpx.Response( + 500, headers={"content-type": "text/event-stream"}, text=sse_body + ) + logging_obj = MagicMock() + + with pytest.raises(OpenAIError, match=f"{response_model} failed"): + config.transform_response_api_response( + model=model_name, + raw_response=raw_response, + logging_obj=logging_obj, + ) diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index 7a9d5acaa27..70582fee88f 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -266,3 +266,41 @@ def test_should_preserve_builtin_pricing_regardless_of_deployment_order(): f"Order should not matter. Expected {builtin_output_cost}, " f"got {info_std_2['output_cost_per_token']}" ) + + +def test_should_preserve_shared_chatgpt_backend_mode_for_aliases(): + backend_model = "chatgpt/gpt-5.4" + builtin_info = litellm.get_model_info(model=backend_model) + builtin_mode = builtin_info.get("mode") + + assert builtin_mode == "responses" + + Router( + model_list=[ + { + "model_name": "chatgpt-alias-a", + "litellm_params": { + "model": backend_model, + "api_key": "fake-key-chatgpt-a", + }, + "model_info": { + "id": "chatgpt-alias-a-id", + }, + }, + { + "model_name": "chatgpt-alias-b", + "litellm_params": { + "model": backend_model, + "api_key": "fake-key-chatgpt-b", + }, + "model_info": { + "id": "chatgpt-alias-b-id", + "mode": "chat", + }, + }, + ], + ) + + shared_info = litellm.model_cost.get(backend_model) + assert shared_info is not None + assert shared_info.get("mode") == "responses"