From c17a753adc84635cc49f0b59443c341e60108432 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 00:13:52 -0700 Subject: [PATCH] fix(logging): log a dropped attempt's failure once when several stream layers wrap it A chat completions stream on a deployment bridged to the Responses API has the Responses stream inside and the chat stream wrapper outside, both on one logging object, and both hand a dropped attempt to dispatch_attempt_failure_handlers. The helper now claims each attempt once by its attempt marker, so the inner stream's row with the provider's own error is the only one, while a fallback hop with its own marker still logs. With no marker it dispatches as before. --- litellm/litellm_core_utils/litellm_logging.py | 15 +++ .../test_litellm_logging.py | 114 ++++++++++++++++++ 2 files changed, 129 insertions(+) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 8b66c27b930..a8e49e6a46d 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -617,6 +617,10 @@ def same_attempt_marker(first: object, second: object) -> bool: return first is second or first == second +async def _attempt_failure_already_dispatched() -> None: + return None + + class Logging(LiteLLMLoggingBaseClass): global \ supabaseClient, \ @@ -641,6 +645,7 @@ class Logging(LiteLLMLoggingBaseClass): baseline_cache_context: "BaselineCacheContext | None" = None baseline_observation: "CapturedBaselineObservation | None" = None _on_failure_payload_built: "Callable[[], None] | None" = None + _failure_dispatched_attempt: object = None @property def _defer_async_logging(self) -> bool: @@ -2499,6 +2504,8 @@ class Logging(LiteLLMLoggingBaseClass): ) -> Coroutine[None, None, None]: shared_payload: Final = self.model_call_details.get("standard_logging_object") attempt_marker: Final = self.model_call_details.get("current_attempt_api_call_start_time") + if not self._claim_attempt_failure_dispatch(attempt_marker): + return _attempt_failure_already_dispatched() attempt: Final = self.attempt_scoped_copy() def publish() -> None: @@ -2518,6 +2525,14 @@ class Logging(LiteLLMLoggingBaseClass): return dispatch() + def _claim_attempt_failure_dispatch(self, attempt_marker: object) -> bool: + if attempt_marker is None: + return True + if same_attempt_marker(self._failure_dispatched_attempt, attempt_marker): + return False + self._failure_dispatched_attempt = attempt_marker + return True + def carry_failed_attempt_usage(self) -> None: """Move what a failed earlier attempt stashed off the retry sharing this logging object, so the retry logs its own usage while the request's failure log still bills every attempt.""" diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 299088a6c4f..6923517bf01 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -20,6 +20,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest import pytest_asyncio +import respx from mcp.types import AudioContent, CallToolResult, ImageContent, TextContent from openai import AsyncOpenAI from openai._legacy_response import HttpxBinaryResponseContent @@ -1945,6 +1946,119 @@ async def test_streaming_sync_and_async_failure_dedupe_independently( assert failure_logger.sync_failure_events.qsize() == 1 +class _FailureRowsByDeployment(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.rows: tuple[tuple[object, object], ...] = () + + async def async_log_failure_event( + self, + kwargs: dict[str, object], + response_obj: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + payload: Final = kwargs["standard_logging_object"] + assert isinstance(payload, dict) + self.rows = (*self.rows, (payload["model_id"], payload["error_str"])) + + +_PEER_DROP_MESSAGE: Final = "peer closed connection without sending complete message body" + + +class _SseStreamDroppedByPeer(httpx.AsyncByteStream): + def __init__(self, events: tuple[Mapping[str, object], ...]) -> None: + self._events: Final = events + + async def __aiter__(self) -> AsyncIterator[bytes]: + for event in self._events: + yield f"data: {json.dumps(event)}\n\n".encode() + raise httpx.RemoteProtocolError(_PEER_DROP_MESSAGE) + + +def _responses_stream_dropped_by_peer() -> httpx.Response: + opened: Final = { + "id": "resp_dropped", + "object": "response", + "created_at": 1790000000, + "status": "in_progress", + "model": "gpt-5.6", + "output": [], + } + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=_SseStreamDroppedByPeer( + ( + {"type": "response.created", "sequence_number": 0, "response": opened}, + {"type": "response.in_progress", "sequence_number": 1, "response": opened}, + ) + ), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("backup_drops_mid_stream", [False, True]) +async def test_chat_stream_bridged_to_responses_logs_each_failed_attempt_once( + monkeypatch: pytest.MonkeyPatch, + backup_drops_mid_stream: bool, +) -> None: + recorder: Final = _FailureRowsByDeployment() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + messages: Final = [{"role": "user", "content": "Write a 300 word story about the sea."}] + router: Final = litellm.Router( + model_list=[ + { + "model_name": "primary", + "litellm_params": { + "model": "openai/gpt-5.6", + "api_key": "primary-key", + "api_base": "https://primary.example/v1", + }, + "model_info": {"id": "primary-deployment", "mode": "responses"}, + }, + { + "model_name": "backup", + "litellm_params": { + "model": "openai/gpt-5.6-luna", + "api_key": "backup-key", + "api_base": "https://backup.example/v1", + }, + "model_info": {"id": "backup-deployment", "mode": "responses"}, + }, + ], + fallbacks=[{"primary": ["backup"]}], + num_retries=0, + ) + shared_logging_obj, _ = litellm.utils.function_setup( + "acompletion", + litellm.utils.Rules(), + datetime_unit_test.now(), + model="primary", + messages=messages, + stream=True, + litellm_call_id="bridged-chat-dropped-attempt", + ) + + with respx.mock() as upstream: + upstream.post("https://primary.example/v1/responses").mock(return_value=_responses_stream_dropped_by_peer()) + upstream.post("https://backup.example/v1/responses").mock( + return_value=( + _responses_stream_dropped_by_peer() + if backup_drops_mid_stream + else httpx.Response(503, json={"error": {"message": "backup is down"}}) + ) + ) + with pytest.raises(httpx.RemoteProtocolError): + await router.acompletion( + model="primary", messages=messages, stream=True, litellm_logging_obj=shared_logging_obj + ) + + assert [model_id for model_id, _ in recorder.rows] == ["primary-deployment", "backup-deployment"] + assert recorder.rows[0][1] == _PEER_DROP_MESSAGE + + def test_streaming_success_is_not_marked_complete(logging_obj: LitellmLogging) -> None: logging_obj.stream = True