fix(proxy): send the Responses fallback event without an event line so older OpenAI SDKs read it

litellm/proxy/proxy_server.py

@@ -9915,11 +9915,11 @@ def _format_fallback_metadata_sse_event(
     *,
     fallback_model: str | None,
     fallback_errors: list[dict[str, object]],
-    error_state: ResponsesStreamErrorState | None,
+    responses_stream: bool,
     return_raw_model_name: bool,
-) -> str | bytes:
+) -> str:
     litellm_fallback: Final = {"fallback_model": fallback_model, "errors": fallback_errors}
-    if error_state is not None:
+    if responses_stream:
         stamped_model: Final = None if return_raw_model_name else fallback_model
         event: Final = {
             "type": _RESPONSES_FALLBACK_METADATA_EVENT_TYPE,
@@ -9927,8 +9927,7 @@ def _format_fallback_metadata_sse_event(
             **({"model": stamped_model} if stamped_model else {}),
             "litellm_fallback": litellm_fallback,
         }
-        error_state.observe_chunk(event)
-        return error_state.mark_emitted(f"data: {json.dumps(event)}\n\n")
+        return f"data: {json.dumps(event)}\n\n"
     payload: Final = {
         "id": _FALLBACK_METADATA_ID,
         "object": "chat.completion.chunk",
@@ -10588,7 +10587,7 @@ async def async_data_generator(
                     yield _format_fallback_metadata_sse_event(
                         fallback_model=fallback_model_from_metadata,
                         fallback_errors=fallback_errors,
-                        error_state=error_state,
+                        responses_stream=error_state is not None,
                         return_raw_model_name=return_raw_model_name,
                     )
                     fallback_metadata_event_sent = True
@@ -10643,7 +10642,7 @@ async def async_data_generator(
                 yield _format_fallback_metadata_sse_event(
                     fallback_model=fallback_model_from_metadata,
                     fallback_errors=fallback_errors,
-                    error_state=error_state,
+                    responses_stream=error_state is not None,
                     return_raw_model_name=return_raw_model_name,
                 )
                 fallback_metadata_event_sent = True

tests/unit/proxy/proxy_server/test_streaming_helpers.py

@@ -504,7 +504,7 @@ def test_format_fallback_metadata_sse_event():
     event = _format_fallback_metadata_sse_event(
         fallback_model="fallback-model",
         fallback_errors=fallback_errors,
-        error_state=None,
+        responses_stream=False,
         return_raw_model_name=False,
     )
 
@@ -899,7 +899,7 @@ async def test_chat_stream_sends_fallback_metadata_in_place_of_a_stripped_usage_
 @pytest.mark.asyncio
 @pytest.mark.parametrize("fallback_model", ["fallback-model", None])
 @pytest.mark.parametrize("return_raw_model_name", [False, True])
-async def test_responses_stream_sends_fallback_metadata_as_a_typed_event_framed_like_its_neighbours(
+async def test_responses_stream_sends_fallback_metadata_as_a_typed_event_without_an_event_line(
     monkeypatch: pytest.MonkeyPatch,
     fallback_model: str | None,
     return_raw_model_name: bool,
@@ -967,51 +967,7 @@ async def test_responses_stream_sends_fallback_metadata_as_a_typed_event_framed_
         "litellm_fallback": {"fallback_model": fallback_model, "errors": fallback_errors},
     }
     assert {payload.get("model") for payload in payloads} == {stamped_model}
-    assert len({frame.startswith("event: ") for frame in event_frames}) == 1
-
-
-@pytest.mark.asyncio
-async def test_responses_stream_names_the_fallback_event_after_a_raw_upstream_frame(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    _patch_logging_flags(monkeypatch)
-    monkeypatch.setitem(ps.general_settings, "expose_fallback_errors_to_caller", True)
-    response: Final = ResponsesAPIResponse(id="resp_fallback", created_at=1, model="fallback-model", output=[])
-    created_event: Final = {"type": "response.created", "sequence_number": 0, "response": response.model_dump(mode="json")}
-    upstream: Final = _FakeStream(
-        [
-            f"event: response.created\ndata: {json.dumps(created_event)}\n\n".encode(),
-            ResponseCompletedEvent.model_validate(
-                {"type": "response.completed", "sequence_number": 1, "response": response}
-            ),
-        ],
-        hidden_params={
-            "additional_headers": {
-                "x-litellm-attempted-fallbacks": 1,
-                "x-litellm-model-group": "fallback-model",
-                "x-litellm-fallback-errors": json.dumps([{"message": "upstream failed", "code": "500"}]),
-            }
-        },
-    )
-
-    frames: Final = tuple(
-        [
-            frame.decode() if isinstance(frame, bytes) else frame
-            async for frame in async_data_generator(
-                response=upstream,
-                user_api_key_dict=_user_auth(),
-                request_data={"model": "primary-model", "include_fallback_errors": True},
-                responses_stream_errors=True,
-            )
-        ]
-    )
-    event_names: Final = tuple(
-        tuple(line.removeprefix("event: ") for line in frame.splitlines() if line.startswith("event: "))
-        for frame in frames[1:3]
-    )
-
-    assert '"type": "litellm.fallback_metadata"' in frames[1]
-    assert event_names in (((), ()), (("litellm.fallback_metadata",), ("response.completed",)))
+    assert event_frames[1] == f"data: {json.dumps(payloads[1])}\n\n"
 
 
 @pytest.mark.asyncio