diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index bbb62fcf490..3828da43c8c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -9915,11 +9915,11 @@ def _format_fallback_metadata_sse_event( *, fallback_model: str | None, fallback_errors: list[dict[str, object]], - error_state: ResponsesStreamErrorState | None, + responses_stream: bool, return_raw_model_name: bool, -) -> str | bytes: +) -> str: litellm_fallback: Final = {"fallback_model": fallback_model, "errors": fallback_errors} - if error_state is not None: + if responses_stream: stamped_model: Final = None if return_raw_model_name else fallback_model event: Final = { "type": _RESPONSES_FALLBACK_METADATA_EVENT_TYPE, @@ -9927,8 +9927,7 @@ def _format_fallback_metadata_sse_event( **({"model": stamped_model} if stamped_model else {}), "litellm_fallback": litellm_fallback, } - error_state.observe_chunk(event) - return error_state.mark_emitted(f"data: {json.dumps(event)}\n\n") + return f"data: {json.dumps(event)}\n\n" payload: Final = { "id": _FALLBACK_METADATA_ID, "object": "chat.completion.chunk", @@ -10588,7 +10587,7 @@ async def async_data_generator( yield _format_fallback_metadata_sse_event( fallback_model=fallback_model_from_metadata, fallback_errors=fallback_errors, - error_state=error_state, + responses_stream=error_state is not None, return_raw_model_name=return_raw_model_name, ) fallback_metadata_event_sent = True @@ -10643,7 +10642,7 @@ async def async_data_generator( yield _format_fallback_metadata_sse_event( fallback_model=fallback_model_from_metadata, fallback_errors=fallback_errors, - error_state=error_state, + responses_stream=error_state is not None, return_raw_model_name=return_raw_model_name, ) fallback_metadata_event_sent = True diff --git a/tests/unit/proxy/proxy_server/test_streaming_helpers.py b/tests/unit/proxy/proxy_server/test_streaming_helpers.py index 6c35d919f56..d3df5b1231d 100644 --- a/tests/unit/proxy/proxy_server/test_streaming_helpers.py +++ b/tests/unit/proxy/proxy_server/test_streaming_helpers.py @@ -504,7 +504,7 @@ def test_format_fallback_metadata_sse_event(): event = _format_fallback_metadata_sse_event( fallback_model="fallback-model", fallback_errors=fallback_errors, - error_state=None, + responses_stream=False, return_raw_model_name=False, ) @@ -899,7 +899,7 @@ async def test_chat_stream_sends_fallback_metadata_in_place_of_a_stripped_usage_ @pytest.mark.asyncio @pytest.mark.parametrize("fallback_model", ["fallback-model", None]) @pytest.mark.parametrize("return_raw_model_name", [False, True]) -async def test_responses_stream_sends_fallback_metadata_as_a_typed_event_framed_like_its_neighbours( +async def test_responses_stream_sends_fallback_metadata_as_a_typed_event_without_an_event_line( monkeypatch: pytest.MonkeyPatch, fallback_model: str | None, return_raw_model_name: bool, @@ -967,51 +967,7 @@ async def test_responses_stream_sends_fallback_metadata_as_a_typed_event_framed_ "litellm_fallback": {"fallback_model": fallback_model, "errors": fallback_errors}, } assert {payload.get("model") for payload in payloads} == {stamped_model} - assert len({frame.startswith("event: ") for frame in event_frames}) == 1 - - -@pytest.mark.asyncio -async def test_responses_stream_names_the_fallback_event_after_a_raw_upstream_frame( - monkeypatch: pytest.MonkeyPatch, -) -> None: - _patch_logging_flags(monkeypatch) - monkeypatch.setitem(ps.general_settings, "expose_fallback_errors_to_caller", True) - response: Final = ResponsesAPIResponse(id="resp_fallback", created_at=1, model="fallback-model", output=[]) - created_event: Final = {"type": "response.created", "sequence_number": 0, "response": response.model_dump(mode="json")} - upstream: Final = _FakeStream( - [ - f"event: response.created\ndata: {json.dumps(created_event)}\n\n".encode(), - ResponseCompletedEvent.model_validate( - {"type": "response.completed", "sequence_number": 1, "response": response} - ), - ], - hidden_params={ - "additional_headers": { - "x-litellm-attempted-fallbacks": 1, - "x-litellm-model-group": "fallback-model", - "x-litellm-fallback-errors": json.dumps([{"message": "upstream failed", "code": "500"}]), - } - }, - ) - - frames: Final = tuple( - [ - frame.decode() if isinstance(frame, bytes) else frame - async for frame in async_data_generator( - response=upstream, - user_api_key_dict=_user_auth(), - request_data={"model": "primary-model", "include_fallback_errors": True}, - responses_stream_errors=True, - ) - ] - ) - event_names: Final = tuple( - tuple(line.removeprefix("event: ") for line in frame.splitlines() if line.startswith("event: ")) - for frame in frames[1:3] - ) - - assert '"type": "litellm.fallback_metadata"' in frames[1] - assert event_names in (((), ()), (("litellm.fallback_metadata",), ("response.completed",))) + assert event_frames[1] == f"data: {json.dumps(payloads[1])}\n\n" @pytest.mark.asyncio