mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(logging): log a dropped attempt's failure once when several stream layers wrap it
A chat completions stream on a deployment bridged to the Responses API has the Responses stream inside and the chat stream wrapper outside, both on one logging object, and both hand a dropped attempt to dispatch_attempt_failure_handlers. The helper now claims each attempt once by its attempt marker, so the inner stream's row with the provider's own error is the only one, while a fallback hop with its own marker still logs. With no marker it dispatches as before.
This commit is contained in:
parent
81a2602270
commit
c17a753adc
2 changed files with 129 additions and 0 deletions
|
|
@ -617,6 +617,10 @@ def same_attempt_marker(first: object, second: object) -> bool:
|
|||
return first is second or first == second
|
||||
|
||||
|
||||
async def _attempt_failure_already_dispatched() -> None:
|
||||
return None
|
||||
|
||||
|
||||
class Logging(LiteLLMLoggingBaseClass):
|
||||
global \
|
||||
supabaseClient, \
|
||||
|
|
@ -641,6 +645,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
baseline_cache_context: "BaselineCacheContext | None" = None
|
||||
baseline_observation: "CapturedBaselineObservation | None" = None
|
||||
_on_failure_payload_built: "Callable[[], None] | None" = None
|
||||
_failure_dispatched_attempt: object = None
|
||||
|
||||
@property
|
||||
def _defer_async_logging(self) -> bool:
|
||||
|
|
@ -2499,6 +2504,8 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
) -> Coroutine[None, None, None]:
|
||||
shared_payload: Final = self.model_call_details.get("standard_logging_object")
|
||||
attempt_marker: Final = self.model_call_details.get("current_attempt_api_call_start_time")
|
||||
if not self._claim_attempt_failure_dispatch(attempt_marker):
|
||||
return _attempt_failure_already_dispatched()
|
||||
attempt: Final = self.attempt_scoped_copy()
|
||||
|
||||
def publish() -> None:
|
||||
|
|
@ -2518,6 +2525,14 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
return dispatch()
|
||||
|
||||
def _claim_attempt_failure_dispatch(self, attempt_marker: object) -> bool:
|
||||
if attempt_marker is None:
|
||||
return True
|
||||
if same_attempt_marker(self._failure_dispatched_attempt, attempt_marker):
|
||||
return False
|
||||
self._failure_dispatched_attempt = attempt_marker
|
||||
return True
|
||||
|
||||
def carry_failed_attempt_usage(self) -> None:
|
||||
"""Move what a failed earlier attempt stashed off the retry sharing this logging object, so the retry logs its
|
||||
own usage while the request's failure log still bills every attempt."""
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
|
|||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import respx
|
||||
from mcp.types import AudioContent, CallToolResult, ImageContent, TextContent
|
||||
from openai import AsyncOpenAI
|
||||
from openai._legacy_response import HttpxBinaryResponseContent
|
||||
|
|
@ -1945,6 +1946,119 @@ async def test_streaming_sync_and_async_failure_dedupe_independently(
|
|||
assert failure_logger.sync_failure_events.qsize() == 1
|
||||
|
||||
|
||||
class _FailureRowsByDeployment(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.rows: tuple[tuple[object, object], ...] = ()
|
||||
|
||||
async def async_log_failure_event(
|
||||
self,
|
||||
kwargs: dict[str, object],
|
||||
response_obj: object,
|
||||
start_time: datetime.datetime,
|
||||
end_time: datetime.datetime,
|
||||
) -> None:
|
||||
payload: Final = kwargs["standard_logging_object"]
|
||||
assert isinstance(payload, dict)
|
||||
self.rows = (*self.rows, (payload["model_id"], payload["error_str"]))
|
||||
|
||||
|
||||
_PEER_DROP_MESSAGE: Final = "peer closed connection without sending complete message body"
|
||||
|
||||
|
||||
class _SseStreamDroppedByPeer(httpx.AsyncByteStream):
|
||||
def __init__(self, events: tuple[Mapping[str, object], ...]) -> None:
|
||||
self._events: Final = events
|
||||
|
||||
async def __aiter__(self) -> AsyncIterator[bytes]:
|
||||
for event in self._events:
|
||||
yield f"data: {json.dumps(event)}\n\n".encode()
|
||||
raise httpx.RemoteProtocolError(_PEER_DROP_MESSAGE)
|
||||
|
||||
|
||||
def _responses_stream_dropped_by_peer() -> httpx.Response:
|
||||
opened: Final = {
|
||||
"id": "resp_dropped",
|
||||
"object": "response",
|
||||
"created_at": 1790000000,
|
||||
"status": "in_progress",
|
||||
"model": "gpt-5.6",
|
||||
"output": [],
|
||||
}
|
||||
return httpx.Response(
|
||||
200,
|
||||
headers={"content-type": "text/event-stream"},
|
||||
stream=_SseStreamDroppedByPeer(
|
||||
(
|
||||
{"type": "response.created", "sequence_number": 0, "response": opened},
|
||||
{"type": "response.in_progress", "sequence_number": 1, "response": opened},
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("backup_drops_mid_stream", [False, True])
|
||||
async def test_chat_stream_bridged_to_responses_logs_each_failed_attempt_once(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
backup_drops_mid_stream: bool,
|
||||
) -> None:
|
||||
recorder: Final = _FailureRowsByDeployment()
|
||||
monkeypatch.setattr(litellm, "callbacks", [recorder])
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
messages: Final = [{"role": "user", "content": "Write a 300 word story about the sea."}]
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "primary",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-5.6",
|
||||
"api_key": "primary-key",
|
||||
"api_base": "https://primary.example/v1",
|
||||
},
|
||||
"model_info": {"id": "primary-deployment", "mode": "responses"},
|
||||
},
|
||||
{
|
||||
"model_name": "backup",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-5.6-luna",
|
||||
"api_key": "backup-key",
|
||||
"api_base": "https://backup.example/v1",
|
||||
},
|
||||
"model_info": {"id": "backup-deployment", "mode": "responses"},
|
||||
},
|
||||
],
|
||||
fallbacks=[{"primary": ["backup"]}],
|
||||
num_retries=0,
|
||||
)
|
||||
shared_logging_obj, _ = litellm.utils.function_setup(
|
||||
"acompletion",
|
||||
litellm.utils.Rules(),
|
||||
datetime_unit_test.now(),
|
||||
model="primary",
|
||||
messages=messages,
|
||||
stream=True,
|
||||
litellm_call_id="bridged-chat-dropped-attempt",
|
||||
)
|
||||
|
||||
with respx.mock() as upstream:
|
||||
upstream.post("https://primary.example/v1/responses").mock(return_value=_responses_stream_dropped_by_peer())
|
||||
upstream.post("https://backup.example/v1/responses").mock(
|
||||
return_value=(
|
||||
_responses_stream_dropped_by_peer()
|
||||
if backup_drops_mid_stream
|
||||
else httpx.Response(503, json={"error": {"message": "backup is down"}})
|
||||
)
|
||||
)
|
||||
with pytest.raises(httpx.RemoteProtocolError):
|
||||
await router.acompletion(
|
||||
model="primary", messages=messages, stream=True, litellm_logging_obj=shared_logging_obj
|
||||
)
|
||||
|
||||
assert [model_id for model_id, _ in recorder.rows] == ["primary-deployment", "backup-deployment"]
|
||||
assert recorder.rows[0][1] == _PEER_DROP_MESSAGE
|
||||
|
||||
|
||||
def test_streaming_success_is_not_marked_complete(logging_obj: LitellmLogging) -> None:
|
||||
logging_obj.stream = True
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue