From 8cfb3c32ca244dee1e26171840ca2c9a3f9aeb35 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 03:06:46 -0700 Subject: [PATCH 01/10] test(responses): fail a Responses stream mid-way and expect partial usage and the proxy failure hook --- ...t_responses_adapters_streaming_iterator.py | 158 ++++++++++++++++++ .../test_responses_streaming_iterator.py | 139 ++++++++++++++- .../unit/responses/test_streaming_iterator.py | 4 +- 3 files changed, 298 insertions(+), 3 deletions(-) diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py index e1dded214bf..68412bfbfa2 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py +++ b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py @@ -4,9 +4,11 @@ Tests for AnthropicResponsesStreamWrapper """ import asyncio +import datetime import json import os import sys +from collections.abc import AsyncIterator, Awaitable, Callable from types import SimpleNamespace import pytest @@ -15,6 +17,7 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../. import litellm from litellm.exceptions import MidStreamFallbackError +from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging from litellm.litellm_core_utils.prompt_templates.common_utils import ( encrypted_reasoning_signature, ) @@ -622,3 +625,158 @@ class TestUpstreamFailureEndsStreamWithErrorEvent: chunks = _collect(iter(())) assert [chunk["type"] for chunk in chunks] == ["message_start", "error"] assert chunks[1]["error"] == {"type": "api_error", "message": INCOMPLETE_STREAM_ERROR_MESSAGE} + + +class _FailureHookRecorder: + def __init__(self) -> None: + self.received: tuple[Exception, ...] = () + + async def __call__(self, failure: Exception) -> None: + self.received = (*self.received, failure) + + +def _logging_obj_with_failure_hook(on_failure: Callable[[Exception], Awaitable[None]]) -> LitellmLogging: + logging_obj = LitellmLogging( + model="m", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="anthropic_messages", + start_time=datetime.datetime(2026, 1, 1), + litellm_call_id="lit-6871-test", + function_id="lit-6871-test", + ) + logging_obj._on_detached_stream_failure = on_failure + return logging_obj + + +def _collect_with_logging( + stream: AsyncIterator[dict[str, object]], logging_obj: LitellmLogging +) -> list[dict[str, object]]: + async def _run() -> list[dict[str, object]]: + wrapper = AnthropicResponsesStreamWrapper(responses_stream=stream, model="m", litellm_logging_obj=logging_obj) + return [chunk async for chunk in wrapper] + + return asyncio.run(_run()) + + +class _DeliveryCountingConsumer: + def __init__(self) -> None: + self.delivered: tuple[dict[str, object], ...] = () + self.delivered_when_hook_fired: tuple[int, ...] = () + + async def hook(self, failure: Exception) -> None: + self.delivered_when_hook_fired = (*self.delivered_when_hook_fired, len(self.delivered)) + + async def drain(self, wrapper: AnthropicResponsesStreamWrapper) -> None: + async for chunk in wrapper: + self.delivered = (*self.delivered, chunk) + + +def _failing_after_one_delta() -> AsyncIterator[dict[str, object]]: + async def _gen() -> AsyncIterator[dict[str, object]]: + yield {"type": "response.created"} + yield {"type": "response.output_item.added", "item": {"type": "message", "id": "msg_1"}} + yield {"type": "response.output_text.delta", "item_id": "msg_1", "delta": "Hi"} + raise litellm.RateLimitError(message="You have no credits remaining.", llm_provider="openai", model="m") + + return _gen() + + +class TestUpstreamFailureReachesTheProxyFailureHook: + def test_raised_upstream_failure_reaches_the_hook_unwrapped_once_the_error_event_was_consumed(self): + rate_limit = litellm.RateLimitError(message="You have no credits remaining.", llm_provider="openai", model="m") + wrapped = MidStreamFallbackError( + message=str(rate_limit), + model="m", + llm_provider="openai", + original_exception=rate_limit, + is_pre_first_chunk=False, + ) + recorder = _FailureHookRecorder() + + async def _gen() -> AsyncIterator[dict[str, object]]: + yield {"type": "response.created"} + yield {"type": "response.output_item.added", "item": {"type": "message", "id": "msg_1"}} + yield {"type": "response.output_text.delta", "item_id": "msg_1", "delta": "Hi"} + raise wrapped + + chunks = _collect_with_logging(_gen(), _logging_obj_with_failure_hook(recorder)) + assert [chunk["type"] for chunk in chunks] == [ + "message_start", + "content_block_start", + "content_block_delta", + "error", + ] + assert recorder.received == (rate_limit,) + + def test_response_failed_event_reaches_the_hook_as_an_exception_carrying_the_upstream_status(self): + recorder = _FailureHookRecorder() + failed = SimpleNamespace( + status="failed", + output=[], + usage=None, + error={"code": "rate_limit_exceeded", "message": "Rate limit reached for gpt-5.5, try again in 20s."}, + ) + + async def _gen() -> AsyncIterator[dict[str, object]]: + yield {"type": "response.created"} + yield {"type": "response.failed", "response": failed} + + chunks = _collect_with_logging(_gen(), _logging_obj_with_failure_hook(recorder)) + assert chunks[-1]["type"] == "error" + (failure,) = recorder.received + assert failure.status_code == 429 + assert "Rate limit reached for gpt-5.5, try again in 20s." in failure.message + + def test_a_raising_hook_never_hides_the_error_event_from_the_client(self): + async def _exploding_hook(failure: Exception) -> None: + raise RuntimeError("spend writer is down") + + logging_obj = _logging_obj_with_failure_hook(_exploding_hook) + + async def _gen() -> AsyncIterator[dict[str, object]]: + yield {"type": "response.created"} + raise ConnectionResetError("Response payload is not completed") + + chunks = _collect_with_logging(_gen(), logging_obj) + assert [chunk["type"] for chunk in chunks] == ["message_start", "error"] + assert chunks[-1]["error"] == {"type": "api_error", "message": "Response payload is not completed"} + + def test_the_hook_fires_only_after_the_consumer_took_the_error_event(self): + consumer = _DeliveryCountingConsumer() + + async def _run() -> None: + wrapper = AnthropicResponsesStreamWrapper( + responses_stream=_failing_after_one_delta(), + model="m", + litellm_logging_obj=_logging_obj_with_failure_hook(consumer.hook), + ) + await consumer.drain(wrapper) + with pytest.raises(StopAsyncIteration): + await wrapper.__anext__() + + asyncio.run(_run()) + assert [chunk["type"] for chunk in consumer.delivered] == [ + "message_start", + "content_block_start", + "content_block_delta", + "error", + ] + assert consumer.delivered_when_hook_fired == (len(consumer.delivered),) + + def test_a_consumer_that_closes_the_stream_on_the_error_event_recovers_it_without_the_hook(self): + recorder = _FailureHookRecorder() + + async def _close_on_the_error_event() -> None: + wrapper = AnthropicResponsesStreamWrapper( + responses_stream=_failing_after_one_delta(), model="m", litellm_logging_obj=_logging_obj_with_failure_hook(recorder) + ) + sse = wrapper.async_anthropic_sse_wrapper() + async for frame in sse: + if frame.startswith(b"event: error\n"): + await sse.aclose() + return + pytest.fail("the stream ended without an error event") + + asyncio.run(_close_on_the_error_event()) + assert recorder.received == () diff --git a/tests/unit/responses/test_responses_streaming_iterator.py b/tests/unit/responses/test_responses_streaming_iterator.py index 0419fce8ec2..fd710a479a4 100644 --- a/tests/unit/responses/test_responses_streaming_iterator.py +++ b/tests/unit/responses/test_responses_streaming_iterator.py @@ -7,7 +7,11 @@ crashed with exit 139 whenever any CustomLogger was registered). """ import asyncio +import json import time +from collections.abc import AsyncIterator +from datetime import datetime +from typing import cast import httpx import pytest @@ -15,10 +19,12 @@ import pytest import litellm from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils import thread_pool_executor as thread_pool_executor_module -from litellm.responses import streaming_iterator as responses_streaming_iterator_module from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.responses import streaming_iterator as responses_streaming_iterator_module from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator -from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.llms.openai import ResponsesAPIResponse, ResponsesAPIStreamingResponse +from litellm.types.utils import StandardLoggingPayload class RecordingCustomLogger(CustomLogger): @@ -163,3 +169,132 @@ async def test_sync_callbacks_run_only_after_async_handler_completes(recording_e submit_times = recording_executor.submit_times_for(logging_obj) assert len(submit_times) == 1 assert submit_times[0] >= recorder.async_hook_finished + + +class FailureRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.failure_payloads: tuple[StandardLoggingPayload, ...] = () + self.success_payloads: tuple[StandardLoggingPayload, ...] = () + + async def async_log_failure_event( + self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + self.failure_payloads = (*self.failure_payloads, cast(StandardLoggingPayload, kwargs["standard_logging_object"])) + + async def async_log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + self.success_payloads = (*self.success_payloads, cast(StandardLoggingPayload, kwargs["standard_logging_object"])) + + +class _ChunkRecorder: + def __init__(self) -> None: + self.chunks: tuple[ResponsesAPIStreamingResponse, ...] = () + + async def drain(self, iterator: ResponsesAPIStreamingIterator) -> None: + async for chunk in iterator: + self.chunks = (*self.chunks, chunk) + + +_SEA_DELTAS = ("The sea ", "is wide ", "and deep.") +_SEA_PROMPT = "Write a 300 word story about the sea." +_UNREACHABLE_IMAGE_INPUT: list[dict[str, object]] = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "Describe this picture."}, + {"type": "input_image", "image_url": "http://images.invalid/photo.png", "detail": "high"}, + ], + } +] + + +class _UpstreamThatTimesOutAfterThreeDeltas: + headers: dict[str, str] = {} + + async def aiter_bytes(self) -> AsyncIterator[bytes]: + for sequence_number, delta in enumerate(_SEA_DELTAS): + event = { + "type": "response.output_text.delta", + "item_id": "msg_1", + "output_index": 0, + "content_index": 0, + "delta": delta, + "sequence_number": sequence_number, + } + yield f"data: {json.dumps(event)}\n\n".encode() + raise httpx.ReadTimeout("Timeout on reading data from socket") + + +async def _failure_payload_after_mid_stream_timeout( + monkeypatch: pytest.MonkeyPatch, request_input: str | list[dict[str, object]] +) -> tuple[FailureRecorder, StandardLoggingPayload]: + recorder = FailureRecorder() + monkeypatch.setattr(litellm, "failure_callback", [recorder]) + monkeypatch.setattr(litellm, "_async_failure_callback", [recorder]) + monkeypatch.setattr(litellm, "success_callback", [recorder]) + monkeypatch.setattr(litellm, "_async_success_callback", [recorder]) + logging_obj = _make_logging_obj() + logging_obj.update_environment_variables( + model="gpt-5.4-nano", + optional_params={"stream": True}, + litellm_params={"custom_llm_provider": "openai", "aresponses": True}, + ) + iterator = ResponsesAPIStreamingIterator( + response=cast(httpx.Response, _UpstreamThatTimesOutAfterThreeDeltas()), + model="gpt-5.4-nano", + responses_api_provider_config=OpenAIResponsesAPIConfig(), + logging_obj=logging_obj, + custom_llm_provider="openai", + request_data={"model": "gpt-5.4-nano", "input": request_input, "stream": True}, + ) + received = _ChunkRecorder() + + with pytest.raises(litellm.Timeout, match="Timeout on reading data from socket"): + await received.drain(iterator) + assert [getattr(chunk, "delta", None) for chunk in received.chunks] == list(_SEA_DELTAS) + + await iterator._await_pending_logging() + (payload,) = recorder.failure_payloads + return recorder, payload + + +@pytest.mark.asyncio +async def test_mid_stream_read_timeout_logs_failure_with_partial_usage_and_no_success( + monkeypatch: pytest.MonkeyPatch, +) -> None: + recorder, payload = await _failure_payload_after_mid_stream_timeout(monkeypatch, _SEA_PROMPT) + + assert payload["status"] == "failure" + assert payload["prompt_tokens"] == litellm.token_counter( + model="gpt-5.4-nano", messages=[{"role": "user", "content": _SEA_PROMPT}] + ) + assert payload["completion_tokens"] == litellm.token_counter( + model="gpt-5.4-nano", text="".join(_SEA_DELTAS), count_response_tokens=True + ) + assert payload["response_cost"] > 0 + assert recorder.success_payloads == () + + +@pytest.mark.asyncio +async def test_mid_stream_failure_usage_estimate_never_fetches_image_dimensions( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _, payload = await _failure_payload_after_mid_stream_timeout(monkeypatch, _UNREACHABLE_IMAGE_INPUT) + + assert payload["status"] == "failure" + assert payload["prompt_tokens"] == litellm.token_counter( + model="gpt-5.4-nano", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this picture."}, + {"type": "image_url", "image_url": {"url": "http://images.invalid/photo.png", "detail": "high"}}, + ], + } + ], + use_default_image_token_count=True, + ) + assert payload["completion_tokens"] > 0 diff --git a/tests/unit/responses/test_streaming_iterator.py b/tests/unit/responses/test_streaming_iterator.py index d0e07e1cc23..d08ff52be91 100644 --- a/tests/unit/responses/test_streaming_iterator.py +++ b/tests/unit/responses/test_streaming_iterator.py @@ -1205,7 +1205,9 @@ async def test_completed_event_survives_a_failing_usage_estimate(): falls back to usage None.""" malformed_input: Final = [{"type": "message", "role": "user", "content": 42}] with pytest.raises(ValueError, match="Invalid content type"): - _estimate_usage_from_text("gpt-4o-mini", malformed_input, {"input": malformed_input}, "hello world") + _estimate_usage_from_text( + "gpt-4o-mini", malformed_input, {"input": malformed_input}, "hello world", use_default_image_token_count=False + ) response = _responses_api_response_without_usage() iterator = _make_iterator( From 862e500df623cb445311a8022906ddf14bf24307 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 03:25:37 -0700 Subject: [PATCH 02/10] fix(responses): bill partial usage on mid-stream failures and report bridged /v1/messages failures to the proxy --- .../responses_adapters/streaming_iterator.py | 36 ++++++++++++++++--- litellm/responses/streaming_iterator.py | 29 +++++++++++++-- 2 files changed, 59 insertions(+), 6 deletions(-) diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/pass_through/responses_adapters/streaming_iterator.py index 85287f5433a..097109666f7 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/pass_through/responses_adapters/streaming_iterator.py @@ -2,7 +2,7 @@ ## Translates OpenAI call to Anthropic `/v1/messages` format import asyncio from collections import deque -from collections.abc import AsyncIterator, Iterator, Mapping +from collections.abc import AsyncIterator, Awaitable, Callable, Iterator, Mapping from typing import TYPE_CHECKING, Any, Final from pydantic import ConfigDict, field_validator @@ -10,7 +10,7 @@ from pydantic import ConfigDict, field_validator from litellm import verbose_logger from litellm._logging import redact_internal_details_from_client_message from litellm._uuid import uuid -from litellm.exceptions import MidStreamFallbackError +from litellm.exceptions import APIError, MidStreamFallbackError from litellm.litellm_core_utils.prompt_templates.common_utils import ( encrypted_reasoning_signature, ) @@ -126,10 +126,12 @@ class AnthropicResponsesStreamWrapper: ) -> None: self.responses_stream = responses_stream self.model = model + self.litellm_logging_obj = litellm_logging_obj self.safeguards_evaluator: Final = safeguards_evaluator self._message_id: str = f"msg_{uuid.uuid4()}" if litellm_logging_obj is not None: litellm_logging_obj.record_streamed_anthropic_message_id(self._message_id) + self._unreported_failure: Exception | None = None self._current_block_index: int = -1 # Map item_id -> content_block_index so we can stop the right block later self._item_id_to_block_index: dict[str, int] = {} @@ -435,10 +437,35 @@ class AnthropicResponsesStreamWrapper: self._sent_message_stop = True return - def _fail_stream(self, status_code: int, message: str) -> None: + def _fail_stream(self, status_code: int, message: str, failure: Exception | None = None) -> None: self._stream_failed = True + self._unreported_failure = failure or APIError( + status_code=status_code, + message=message, + llm_provider=getattr(self.responses_stream, "custom_llm_provider", None), + model=self.model, + ) self._chunk_queue.append(_anthropic_error_chunk(status_code, message)) + async def _report_failure_once_the_error_event_was_consumed(self) -> None: + failure: Final = self._unreported_failure + if failure is None: + return + self._unreported_failure = None + on_failure: Final[Callable[[Exception], Awaitable[None]] | None] = getattr( + self.litellm_logging_obj, "_on_detached_stream_failure", None + ) + if on_failure is None: + return + try: + await on_failure(failure) + except Exception as hook_failure: # noqa: BLE001 # a failing proxy hook must not hide the error event from the client + verbose_logger.warning( + "AnthropicResponsesStreamWrapper: stream failure hook raised: %s(%s)", + type(hook_failure).__name__, + hook_failure, + ) + def __aiter__(self) -> "AnthropicResponsesStreamWrapper": return self @@ -446,6 +473,7 @@ class AnthropicResponsesStreamWrapper: if self._chunk_queue: return self._chunk_queue.popleft() if self._stream_failed: + await self._report_failure_once_the_error_event_was_consumed() raise StopAsyncIteration if not self._sent_message_start: @@ -474,7 +502,7 @@ class AnthropicResponsesStreamWrapper: verbose_logger.exception( "AnthropicResponsesStreamWrapper: upstream Responses stream for %s failed", self.model ) - self._fail_stream(*_failure_status_and_message(e)) + self._fail_stream(*_failure_status_and_message(e), failure=_original_failure(e)) if not self._chunk_queue and not self._sent_message_stop and not self._stream_failed: verbose_logger.error( diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index bcd917c69d5..6aa427f16e7 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -62,7 +62,7 @@ from litellm.types.llms.openai import ( ResponsesAPIStreamEvents, ResponsesAPIStreamingResponse, ) -from litellm.types.utils import CallTypes +from litellm.types.utils import CallTypes, ModelResponse from litellm.utils import async_post_call_success_deployment_hook if TYPE_CHECKING: @@ -563,6 +563,7 @@ class BaseResponsesAPIStreamingIterator: self.request_data.get("input"), self.request_data, self._generated_content + self._generated_tool_arguments, + use_default_image_token_count=False, ) if _estimate_wanted else None @@ -962,6 +963,24 @@ class BaseResponsesAPIStreamingIterator: except Exception: pass + def _record_partial_usage_for_failure(self) -> None: + generated_text: Final = self._generated_content + self._generated_tool_arguments + if not generated_text or self.logging_obj.model_call_details.get("combined_usage_object") is not None: + return + estimate: Final = _estimate_usage_safely( + self.model or "", + self.request_data.get("input"), + self.request_data, + generated_text, + use_default_image_token_count=True, + ) + if estimate is None: + return + usage: Final = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( # pyright: ignore[reportPrivateUsage] # same shared transform the success path above uses + estimate + ) + self.logging_obj.record_assembled_response_for_failure(ModelResponse(model=self.model, usage=usage)) + def _handle_failure(self, exception: Exception): """ Trigger failure handlers before bubbling the exception. @@ -971,6 +990,7 @@ class BaseResponsesAPIStreamingIterator: if self._failure_handled: return self._failure_handled = True + self._record_partial_usage_for_failure() traceback_exception: Final = traceback.format_exc() end_time: Final = datetime.now() @@ -1631,13 +1651,15 @@ def _estimate_usage_from_text( request_input: object, responses_api_request: Mapping[str, object], generated_text: str, + *, + use_default_image_token_count: bool, ) -> ResponseAPIUsage: messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( # pyright: ignore[reportUnknownMemberType] # the transformer's signature is partially untyped input=request_input, # pyright: ignore[reportArgumentType] # the raw Responses API input is a str or ResponseInputParam list, matching the helper's declared union responses_api_request=dict(responses_api_request), ) input_tokens: Final = litellm.token_counter( # pyright: ignore[reportUnknownMemberType] # token_counter's public signature is untyped - model=model, messages=messages + model=model, messages=messages, use_default_image_token_count=use_default_image_token_count ) output_tokens: Final = litellm.token_counter( # pyright: ignore[reportUnknownMemberType] # token_counter's public signature is untyped model=model, text=generated_text, count_response_tokens=True @@ -1654,6 +1676,8 @@ def _estimate_usage_safely( request_input: object, responses_api_request: Mapping[str, object], generated_text: str, + *, + use_default_image_token_count: bool, ) -> ResponseAPIUsage | None: try: return _estimate_usage_from_text( @@ -1661,6 +1685,7 @@ def _estimate_usage_safely( request_input=request_input, responses_api_request=responses_api_request, generated_text=generated_text, + use_default_image_token_count=use_default_image_token_count, ) except Exception as e: verbose_logger.debug("Could not estimate usage from stream text, billing $0: %s", e) From 1f1d843319b9c228aad1e51a6837c552dc459bb8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 04:41:38 -0700 Subject: [PATCH 03/10] fix(responses): keep a fallback's usage its own and report bridged failures on close A Responses fallback now moves the failed attempt's partial usage aside before the next hop, so the fallback's success row bills its own usage. The bridge reports an unreported failure when the client closes the stream, unless a fallback takes the stream over, and runs the failure hook shielded from a client disconnect. A response.failed event now reaches the hook as its mapped exception class. --- .../responses_adapters/streaming_iterator.py | 36 +++-- litellm/responses/streaming_iterator.py | 6 +- litellm/router.py | 3 + .../test_exception_mapping_utils.py | 4 +- ...t_responses_adapters_streaming_iterator.py | 42 +++++- tests/unit/test_router/test_router.py | 127 ++++++++++++++++++ 6 files changed, 200 insertions(+), 18 deletions(-) diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/pass_through/responses_adapters/streaming_iterator.py index 097109666f7..f5f9ba14c43 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/pass_through/responses_adapters/streaming_iterator.py @@ -5,6 +5,7 @@ from collections import deque from collections.abc import AsyncIterator, Awaitable, Callable, Iterator, Mapping from typing import TYPE_CHECKING, Any, Final +import anyio from pydantic import ConfigDict, field_validator from litellm import verbose_logger @@ -21,7 +22,7 @@ from litellm.llms.anthropic.pass_through.messages.utils import ( ) from litellm.llms.anthropic.pass_through.safeguards import StreamedSafeguardResults from litellm.llms.anthropic.pass_through.utils import anthropic_sse_frame -from litellm.responses.streaming_iterator import stream_error_status_and_message +from litellm.responses.streaming_iterator import map_stream_error_to_exception, stream_error_status_and_message from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage from litellm.types.llms.base import LiteLLMBaseModel @@ -375,16 +376,24 @@ class AnthropicResponsesStreamWrapper: if event_type == "response.failed": failed: Final = _FailedResponseEvent.model_validate(event) - status_code, message = stream_error_status_and_message( - failed.response.error if failed.response is not None else None - ) + upstream_error: Final = failed.response.error if failed.response is not None else None + status_code, message = stream_error_status_and_message(upstream_error) verbose_logger.error( "AnthropicResponsesStreamWrapper: upstream Responses stream for %s failed (%s): %s", self.model, status_code, message, ) - self._fail_stream(status_code, message) + self._fail_stream( + status_code, + message, + failure=map_stream_error_to_exception( + upstream_error, + self.model, + getattr(self.responses_stream, "custom_llm_provider", None) or "", + logging_obj=self.litellm_logging_obj, + ), + ) return # ---- response completed -> message_delta + message_stop ---- @@ -447,7 +456,7 @@ class AnthropicResponsesStreamWrapper: ) self._chunk_queue.append(_anthropic_error_chunk(status_code, message)) - async def _report_failure_once_the_error_event_was_consumed(self) -> None: + async def _report_unreported_failure(self) -> None: failure: Final = self._unreported_failure if failure is None: return @@ -458,7 +467,8 @@ class AnthropicResponsesStreamWrapper: if on_failure is None: return try: - await on_failure(failure) + with anyio.CancelScope(shield=True): + await on_failure(failure) except Exception as hook_failure: # noqa: BLE001 # a failing proxy hook must not hide the error event from the client verbose_logger.warning( "AnthropicResponsesStreamWrapper: stream failure hook raised: %s(%s)", @@ -473,7 +483,7 @@ class AnthropicResponsesStreamWrapper: if self._chunk_queue: return self._chunk_queue.popleft() if self._stream_failed: - await self._report_failure_once_the_error_event_was_consumed() + await self._report_unreported_failure() raise StopAsyncIteration if not self._sent_message_start: @@ -519,5 +529,11 @@ class AnthropicResponsesStreamWrapper: async def async_anthropic_sse_wrapper(self) -> AsyncIterator[bytes]: """Yield SSE-encoded bytes for each Anthropic event chunk.""" safeguard_results: Final = StreamedSafeguardResults(self.safeguards_evaluator) - async for chunk in self: - yield anthropic_sse_frame(await safeguard_results.observe(chunk)) + try: + async for chunk in self: + yield anthropic_sse_frame(await safeguard_results.observe(chunk)) + finally: + from litellm.llms.anthropic.pass_through.messages.streaming_iterator import closing_superseded_attempt + + if not closing_superseded_attempt(): + await self._report_unreported_failure() diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 6aa427f16e7..1bd63a3ff05 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -284,7 +284,7 @@ def stream_error_status_and_message(error_obj: object) -> tuple[int, str]: return _status_code_for_error_fields(error_type, error_code), message -def _map_stream_error_to_exception( +def map_stream_error_to_exception( error_obj: object, model: str, custom_llm_provider: str, @@ -713,7 +713,7 @@ class BaseResponsesAPIStreamingIterator: ) def _map_error_event_exception(self, error_obj: object) -> Exception: - return _map_stream_error_to_exception( + return map_stream_error_to_exception( error_obj, self.model or "", self.custom_llm_provider or "", @@ -2204,7 +2204,7 @@ class _ResponsesWsTurn: ) if failed_event is None: return None - return _map_stream_error_to_exception( + return map_stream_error_to_exception( _ws_event_error(failed_event), model, custom_llm_provider, logging_obj=connection_logging_obj ) diff --git a/litellm/router.py b/litellm/router.py index aadbdaa8520..6f749562ab7 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -6153,6 +6153,9 @@ class Router: hop_kwargs: Final = mid_stream_fallback_hop_kwargs( model=model, original_generic_function=original_generic_function, controls=controls, kwargs=kwargs ) + shared_logging: Final = kwargs.get("litellm_logging_obj") + if isinstance(shared_logging, LiteLLMLogging): + shared_logging.carry_failed_attempt_usage() response: Final = await self._ageneric_api_call_with_fallbacks_helper( model=model, original_generic_function=original_generic_function, **kwargs ) diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index 5ee14ff530d..fc9043c83bc 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -1510,9 +1510,9 @@ def test_bedrock_mantle_openai_envelope_context_overflow_maps_to_context_window_ def test_bedrock_mantle_streamed_context_overflow_event_maps_to_context_window_exceeded(): - from litellm.responses.streaming_iterator import _map_stream_error_to_exception + from litellm.responses.streaming_iterator import map_stream_error_to_exception - mapped_exception = _map_stream_error_to_exception( + mapped_exception = map_stream_error_to_exception( { "code": "context_length_exceeded", "message": "Your input exceeds the context window of this model. Please adjust your input and try again.", diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py index 68412bfbfa2..851e6392e32 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py +++ b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py @@ -11,6 +11,7 @@ import sys from collections.abc import AsyncIterator, Awaitable, Callable from types import SimpleNamespace +import anyio import pytest sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../../.."))) @@ -21,6 +22,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging from litellm.litellm_core_utils.prompt_templates.common_utils import ( encrypted_reasoning_signature, ) +from litellm.llms.anthropic.pass_through.messages.streaming_iterator import aclose_superseded_attempt from litellm.llms.anthropic.pass_through.messages.utils import INCOMPLETE_STREAM_ERROR_MESSAGE from litellm.llms.anthropic.pass_through.responses_adapters.streaming_iterator import ( AnthropicResponsesStreamWrapper, @@ -725,6 +727,7 @@ class TestUpstreamFailureReachesTheProxyFailureHook: chunks = _collect_with_logging(_gen(), _logging_obj_with_failure_hook(recorder)) assert chunks[-1]["type"] == "error" (failure,) = recorder.received + assert isinstance(failure, litellm.RateLimitError) assert failure.status_code == 429 assert "Rate limit reached for gpt-5.5, try again in 20s." in failure.message @@ -764,7 +767,14 @@ class TestUpstreamFailureReachesTheProxyFailureHook: ] assert consumer.delivered_when_hook_fired == (len(consumer.delivered),) - def test_a_consumer_that_closes_the_stream_on_the_error_event_recovers_it_without_the_hook(self): + @pytest.mark.parametrize( + ("close", "expected_reports"), + [ + pytest.param(lambda sse: sse.aclose(), 1, id="client_closes"), + pytest.param(aclose_superseded_attempt, 0, id="fallback_takes_over"), + ], + ) + def test_closing_the_stream_on_the_error_event_reports_unless_a_fallback_takes_over(self, close, expected_reports): recorder = _FailureHookRecorder() async def _close_on_the_error_event() -> None: @@ -774,9 +784,35 @@ class TestUpstreamFailureReachesTheProxyFailureHook: sse = wrapper.async_anthropic_sse_wrapper() async for frame in sse: if frame.startswith(b"event: error\n"): - await sse.aclose() + await close(sse) return pytest.fail("the stream ended without an error event") asyncio.run(_close_on_the_error_event()) - assert recorder.received == () + assert len(recorder.received) == expected_reports + + def test_a_client_disconnect_during_the_failure_hook_lets_the_hook_finish(self): + hook_started = anyio.Event() + finished: list[Exception] = [] + + async def _slow_hook(failure: Exception) -> None: + hook_started.set() + await anyio.sleep(0.05) + finished.append(failure) + + async def _disconnect_mid_hook() -> None: + wrapper = AnthropicResponsesStreamWrapper( + responses_stream=_failing_after_one_delta(), model="m", litellm_logging_obj=_logging_obj_with_failure_hook(_slow_hook) + ) + + async def _consume() -> None: + async for _ in wrapper.async_anthropic_sse_wrapper(): + pass + + async with anyio.create_task_group() as task_group: + task_group.start_soon(_consume) + await hook_started.wait() + task_group.cancel_scope.cancel() + + asyncio.run(_disconnect_mid_hook()) + assert len(finished) == 1 diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 4a9dd199c08..a78c30055fc 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -3630,6 +3630,133 @@ async def test_continued_stream_sharing_the_proxy_logging_object_bills_the_cut_t ) +def _responses_sse(*events: Mapping[str, object]) -> str: + return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events) + + +def _responses_envelope(response_id: str, model: str, status: str, **fields: object) -> dict[str, object]: + return { + "id": response_id, + "object": "response", + "created_at": 1700000000, + "status": status, + "model": model, + "output": [], + **fields, + } + + +@pytest.mark.asyncio +async def test_responses_fallback_sharing_the_proxy_logging_object_logs_the_fallback_usage( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + recorder: Final = _SpendRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + respx_mock.post("https://primary.example/v1/responses").mock( + return_value=httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + text=_responses_sse( + {"type": "response.created", "response": _responses_envelope("resp_primary", "gpt-4o", "in_progress")}, + *( + { + "type": "response.output_text.delta", + "item_id": "msg_primary", + "output_index": 0, + "content_index": 0, + "delta": word, + } + for word in ("Once ", "upon ", "a ", "time ", "there") + ), + { + "type": "response.failed", + "response": _responses_envelope( + "resp_primary", + "gpt-4o", + "failed", + error={"code": "server_error", "message": "The model had an internal error"}, + ), + }, + ), + ) + ) + respx_mock.post("https://backup.example/v1/responses").mock( + return_value=httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + text=_responses_sse( + {"type": "response.created", "response": _responses_envelope("resp_backup", "gpt-4o-mini", "in_progress")}, + { + "type": "response.completed", + "response": _responses_envelope( + "resp_backup", + "gpt-4o-mini", + "completed", + output=[ + { + "type": "message", + "id": "msg_backup", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Once upon a time", "annotations": []}], + } + ], + usage={"input_tokens": 1000, "output_tokens": 700, "total_tokens": 1700}, + ), + }, + ), + ) + ) + router: Final = Router( + model_list=[ + { + "model_name": "primary", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "k1", "api_base": "https://primary.example/v1"}, + }, + { + "model_name": "backup", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "k2", + "api_base": "https://backup.example/v1", + }, + }, + ], + fallbacks=[{"primary": ["backup"]}], + num_retries=0, + ) + shared_logging, _ = litellm.utils.function_setup( + original_function="aresponses", + rules_obj=litellm.utils.Rules(), + start_time=datetime.now(), + model="primary", + input="Tell me a story", + stream=True, + litellm_call_id="shared-call", + ) + + response: Final = await router.aresponses( + model="primary", + input="Tell me a story", + stream=True, + litellm_call_id="shared-call", + litellm_logging_obj=shared_logging, + ) + _ = [event async for event in response] + await asyncio.wait_for(recorder.succeeded.wait(), timeout=10) + await asyncio.sleep(0.5) + + success: Final = recorder.successes[-1] + assert (success["prompt_tokens"], success["completion_tokens"]) == (1000, 700) + spend_row: Final = recorder.spend_rows[-1] + assert (spend_row["prompt_tokens"], spend_row["completion_tokens"], spend_row["spend"]) == ( + 1000, + 700, + success["response_cost"], + ) + + def test_carry_partial_spend_to_fallback_adds_the_partial_the_cut_stream_inherited(): from litellm.types.utils import Usage From 34a819afbae24cb59ad79a9218a7e731d5c954cc Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 05:48:58 -0700 Subject: [PATCH 04/10] test(responses): keep the provider's reported usage on a failed stream over the estimate --- .../test_responses_streaming_iterator.py | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tests/unit/responses/test_responses_streaming_iterator.py b/tests/unit/responses/test_responses_streaming_iterator.py index fd710a479a4..ba6687f9124 100644 --- a/tests/unit/responses/test_responses_streaming_iterator.py +++ b/tests/unit/responses/test_responses_streaming_iterator.py @@ -17,6 +17,7 @@ import httpx import pytest import litellm +from litellm.exceptions import MidStreamFallbackError from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils import thread_pool_executor as thread_pool_executor_module from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging @@ -227,6 +228,37 @@ class _UpstreamThatTimesOutAfterThreeDeltas: raise httpx.ReadTimeout("Timeout on reading data from socket") +class _UpstreamThatFailsWithUsageAfterThreeDeltas: + headers: dict[str, str] = {} + + async def aiter_bytes(self) -> AsyncIterator[bytes]: + for sequence_number, delta in enumerate(_SEA_DELTAS): + event = { + "type": "response.output_text.delta", + "item_id": "msg_1", + "output_index": 0, + "content_index": 0, + "delta": delta, + "sequence_number": sequence_number, + } + yield f"data: {json.dumps(event)}\n\n".encode() + failed = { + "type": "response.failed", + "sequence_number": len(_SEA_DELTAS), + "response": { + "id": "resp_1", + "object": "response", + "created_at": 1700000000, + "status": "failed", + "model": "gpt-5.4-nano", + "output": [], + "error": {"code": "server_error", "message": "The model had an internal error"}, + "usage": {"input_tokens": 400, "output_tokens": 9, "total_tokens": 409}, + }, + } + yield f"data: {json.dumps(failed)}\n\n".encode() + + async def _failure_payload_after_mid_stream_timeout( monkeypatch: pytest.MonkeyPatch, request_input: str | list[dict[str, object]] ) -> tuple[FailureRecorder, StandardLoggingPayload]: @@ -298,3 +330,31 @@ async def test_mid_stream_failure_usage_estimate_never_fetches_image_dimensions( use_default_image_token_count=True, ) assert payload["completion_tokens"] > 0 + + +@pytest.mark.asyncio +async def test_mid_stream_failure_keeps_the_usage_the_provider_reported(monkeypatch: pytest.MonkeyPatch) -> None: + recorder = FailureRecorder() + monkeypatch.setattr(litellm, "failure_callback", [recorder]) + monkeypatch.setattr(litellm, "_async_failure_callback", [recorder]) + logging_obj = _make_logging_obj() + logging_obj.update_environment_variables( + model="gpt-5.4-nano", + optional_params={"stream": True}, + litellm_params={"custom_llm_provider": "openai", "aresponses": True}, + ) + iterator = ResponsesAPIStreamingIterator( + response=cast(httpx.Response, _UpstreamThatFailsWithUsageAfterThreeDeltas()), + model="gpt-5.4-nano", + responses_api_provider_config=OpenAIResponsesAPIConfig(), + logging_obj=logging_obj, + custom_llm_provider="openai", + request_data={"model": "gpt-5.4-nano", "input": _SEA_PROMPT, "stream": True}, + ) + + with pytest.raises(MidStreamFallbackError, match="The model had an internal error"): + await _ChunkRecorder().drain(iterator) + await iterator._await_pending_logging() + + (payload,) = recorder.failure_payloads + assert (payload["prompt_tokens"], payload["completion_tokens"]) == (400, 9) From 012243d7b08d55d1d2a880126a22a1f1ab2a7aec Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 07:00:34 -0700 Subject: [PATCH 05/10] test(responses): bill streamed tool call arguments on a mid-stream failure --- .../test_responses_streaming_iterator.py | 40 +++++++++++++++---- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/tests/unit/responses/test_responses_streaming_iterator.py b/tests/unit/responses/test_responses_streaming_iterator.py index ba6687f9124..8f5381cce08 100644 --- a/tests/unit/responses/test_responses_streaming_iterator.py +++ b/tests/unit/responses/test_responses_streaming_iterator.py @@ -11,7 +11,7 @@ import json import time from collections.abc import AsyncIterator from datetime import datetime -from typing import cast +from typing import Final, cast import httpx import pytest @@ -211,14 +211,21 @@ _UNREACHABLE_IMAGE_INPUT: list[dict[str, object]] = [ ] +_TOOL_ARGUMENT_DELTAS = ('{"city": ', '"Lisbon", ', '"unit": "celsius"}') + + class _UpstreamThatTimesOutAfterThreeDeltas: headers: dict[str, str] = {} + def __init__(self, event_type: str = "response.output_text.delta", deltas: tuple[str, ...] = _SEA_DELTAS) -> None: + self.event_type = event_type + self.deltas = deltas + async def aiter_bytes(self) -> AsyncIterator[bytes]: - for sequence_number, delta in enumerate(_SEA_DELTAS): + for sequence_number, delta in enumerate(self.deltas): event = { - "type": "response.output_text.delta", - "item_id": "msg_1", + "type": self.event_type, + "item_id": "item_1", "output_index": 0, "content_index": 0, "delta": delta, @@ -260,8 +267,11 @@ class _UpstreamThatFailsWithUsageAfterThreeDeltas: async def _failure_payload_after_mid_stream_timeout( - monkeypatch: pytest.MonkeyPatch, request_input: str | list[dict[str, object]] + monkeypatch: pytest.MonkeyPatch, + request_input: str | list[dict[str, object]], + upstream: _UpstreamThatTimesOutAfterThreeDeltas | None = None, ) -> tuple[FailureRecorder, StandardLoggingPayload]: + upstream_response: Final = upstream or _UpstreamThatTimesOutAfterThreeDeltas() recorder = FailureRecorder() monkeypatch.setattr(litellm, "failure_callback", [recorder]) monkeypatch.setattr(litellm, "_async_failure_callback", [recorder]) @@ -274,7 +284,7 @@ async def _failure_payload_after_mid_stream_timeout( litellm_params={"custom_llm_provider": "openai", "aresponses": True}, ) iterator = ResponsesAPIStreamingIterator( - response=cast(httpx.Response, _UpstreamThatTimesOutAfterThreeDeltas()), + response=cast(httpx.Response, upstream_response), model="gpt-5.4-nano", responses_api_provider_config=OpenAIResponsesAPIConfig(), logging_obj=logging_obj, @@ -285,7 +295,7 @@ async def _failure_payload_after_mid_stream_timeout( with pytest.raises(litellm.Timeout, match="Timeout on reading data from socket"): await received.drain(iterator) - assert [getattr(chunk, "delta", None) for chunk in received.chunks] == list(_SEA_DELTAS) + assert [getattr(chunk, "delta", None) for chunk in received.chunks] == list(upstream_response.deltas) await iterator._await_pending_logging() (payload,) = recorder.failure_payloads @@ -309,6 +319,22 @@ async def test_mid_stream_read_timeout_logs_failure_with_partial_usage_and_no_su assert recorder.success_payloads == () +@pytest.mark.asyncio +async def test_mid_stream_read_timeout_bills_the_tool_call_arguments_already_streamed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + upstream: Final = _UpstreamThatTimesOutAfterThreeDeltas( + event_type="response.function_call_arguments.delta", deltas=_TOOL_ARGUMENT_DELTAS + ) + _, payload = await _failure_payload_after_mid_stream_timeout(monkeypatch, _SEA_PROMPT, upstream) + + assert payload["status"] == "failure" + assert payload["completion_tokens"] == litellm.token_counter( + model="gpt-5.4-nano", text="".join(_TOOL_ARGUMENT_DELTAS), count_response_tokens=True + ) + assert payload["response_cost"] > 0 + + @pytest.mark.asyncio async def test_mid_stream_failure_usage_estimate_never_fetches_image_dimensions( monkeypatch: pytest.MonkeyPatch, From a874b8a1e6ff911c6eb7faa63a3468c74d797fff Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:01:20 -0700 Subject: [PATCH 06/10] fix(proxy): report a bridged stream failure to the failure hook once and skip the estimate for background responses --- litellm/litellm_core_utils/litellm_logging.py | 7 ++ litellm/proxy/common_request_processing.py | 19 ++++- litellm/responses/streaming_iterator.py | 6 +- .../proxy/test_common_request_processing.py | 75 +++++++++++++++++++ .../test_responses_streaming_iterator.py | 14 +++- 5 files changed, 115 insertions(+), 6 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 8d5be388eff..c851495f904 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -796,6 +796,7 @@ class Logging(LiteLLMLoggingBaseClass): self._async_success_scheduled: bool = False self._sync_success_scheduled: bool = False self.on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None + self._stream_failure_claimed: bool = False self.shadow_eval_request_snapshot: GuardrailRequestSnapshot | None = None def copy_for_new_call(self, litellm_call_id: str, start_time: datetime.datetime) -> "Logging": @@ -2409,6 +2410,12 @@ class Logging(LiteLLMLoggingBaseClass): }, ) + def claim_stream_failure_report(self) -> bool: + if self._stream_failure_claimed: + return False + self._stream_failure_claimed = True + return True + def record_assembled_response_for_failure(self, assembled: ModelResponse) -> None: """Bill a fully streamed response on the failure log when a post-call hook rejects it.""" usage: Final = getattr(assembled, "usage", None) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 24a7b46da4b..1a45c507566 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -3437,6 +3437,8 @@ class ProxyBaseLLMRequestProcessing: request_data: Final = self.data async def _on_detached_stream_failure(exc: Exception) -> None: + if not logging_obj.claim_stream_failure_report(): + return try: await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, @@ -4384,6 +4386,7 @@ class ProxyBaseLLMRequestProcessing: client_disconnected = False delivered_chunk = False recent_tail = SSE_STREAM_START_TAIL # rebind-ok: rolling window over the yielded bytes + stream_logging_obj: Final[object] = request_data.get("litellm_logging_obj") # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType] # legacy request data is narrowed to Logging below guarded_stream: Final[AsyncGenerator[object, None]] = proxy_logging_obj.async_post_call_streaming_iterator_hook( user_api_key_dict=user_api_key_dict, response=response, @@ -4459,10 +4462,18 @@ class ProxyBaseLLMRequestProcessing: verbose_proxy_logger.exception( "litellm.proxy.proxy_server.async_data_generator(): Exception occured - %s", e ) - transformed_exception: Final = await proxy_logging_obj.post_call_failure_hook( - user_api_key_dict=user_api_key_dict, - original_exception=e, - request_data=request_data, + already_reported: Final = ( + isinstance(stream_logging_obj, LiteLLMLoggingObj) + and not stream_logging_obj.claim_stream_failure_report() + ) + transformed_exception: Final = ( + None + if already_reported + else await proxy_logging_obj.post_call_failure_hook( + user_api_key_dict=user_api_key_dict, + original_exception=e, + request_data=request_data, + ) ) if transformed_exception is not None: e = transformed_exception diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 1bd63a3ff05..fb5e989b6bc 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -965,7 +965,11 @@ class BaseResponsesAPIStreamingIterator: def _record_partial_usage_for_failure(self) -> None: generated_text: Final = self._generated_content + self._generated_tool_arguments - if not generated_text or self.logging_obj.model_call_details.get("combined_usage_object") is not None: + if ( + not generated_text + or self.request_data.get("background") is True + or self.logging_obj.model_call_details.get("combined_usage_object") is not None + ): return estimate: Final = _estimate_usage_safely( self.model or "", diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index f837dc3fb5f..b04967ac7d9 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -10669,6 +10669,81 @@ class TestDetachedStreamFailureHook: assert [call["original_exception"] for call in recorder.calls] == [failure] + @staticmethod + def _armed_stream(recorder: _FailureHookRecorder, iterator_hook_for): + from litellm.proxy._types import UserAPIKeyAuth as RealUserAPIKeyAuth + + logging_obj = LiteLLMLoggingObj( + model="gpt-6.1-sol", + messages=[{"role": "user", "content": "Write a story."}], + stream=True, + call_type="anthropic_messages", + start_time=None, + litellm_call_id="call-lit6871", + function_id="function-lit6871", + ) + user_api_key_dict = RealUserAPIKeyAuth(api_key="sk-test") + processing_obj = ProxyBaseLLMRequestProcessing(data={"litellm_logging_obj": logging_obj}) + ProxyLogging._callback_capabilities_cache.clear() + proxy_logging_obj = ProxyLogging(user_api_key_cache=MagicMock()) + proxy_logging_obj.post_call_failure_hook = recorder.post_call_failure_hook + proxy_logging_obj.async_post_call_streaming_iterator_hook = iterator_hook_for(logging_obj) + processing_obj._arm_detached_stream_failure_hook( + logging_obj=logging_obj, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + ) + stream = ProxyBaseLLMRequestProcessing.async_streaming_data_generator( + response=MagicMock(), + user_api_key_dict=user_api_key_dict, + request_data=processing_obj.data, + proxy_logging_obj=proxy_logging_obj, + serialize_chunk=lambda chunk: "data: chunk\n\n", + serialize_error=lambda error: "data: error\n\n", + ) + return logging_obj, stream + + @pytest.mark.asyncio + async def test_failure_reported_by_the_stream_reaches_the_failure_hook_once_when_an_iterator_hook_raises_after_it( + self, + ): + recorder = _FailureHookRecorder() + upstream_failure = RuntimeError("upstream Responses stream failed") + hook_failure = RuntimeError("iterator hook rejected the stream") + + def iterator_hook_for(logging_obj): + async def iterator_hook(**_kwargs): + yield b"event: error\n\n" + await logging_obj.on_detached_stream_failure(upstream_failure) + raise hook_failure + + return iterator_hook + + _, stream = self._armed_stream(recorder, iterator_hook_for) + + frames = [frame async for frame in stream] + + assert frames == ["data: chunk\n\n", "data: error\n\n"] + assert [call["original_exception"] for call in recorder.calls] == [upstream_failure] + + @pytest.mark.asyncio + async def test_failure_reported_by_an_iterator_hook_is_not_reported_again_when_the_stream_closes(self): + recorder = _FailureHookRecorder() + upstream_failure = RuntimeError("upstream Responses stream failed") + hook_failure = RuntimeError("iterator hook rejected the error frame") + + async def iterator_hook(**_kwargs): + yield b"event: message_start\n\n" + raise hook_failure + + logging_obj, stream = self._armed_stream(recorder, lambda _logging_obj: iterator_hook) + + frames = [frame async for frame in stream] + await logging_obj.on_detached_stream_failure(upstream_failure) + + assert frames == ["data: chunk\n\n", "data: error\n\n"] + assert [call["original_exception"] for call in recorder.calls] == [hook_failure] + class TestPostCallMaskedOutputReachesDeferredLogging: @pytest.mark.asyncio diff --git a/tests/unit/responses/test_responses_streaming_iterator.py b/tests/unit/responses/test_responses_streaming_iterator.py index 8f5381cce08..8c4eec4d7d6 100644 --- a/tests/unit/responses/test_responses_streaming_iterator.py +++ b/tests/unit/responses/test_responses_streaming_iterator.py @@ -270,6 +270,7 @@ async def _failure_payload_after_mid_stream_timeout( monkeypatch: pytest.MonkeyPatch, request_input: str | list[dict[str, object]], upstream: _UpstreamThatTimesOutAfterThreeDeltas | None = None, + background: bool = False, ) -> tuple[FailureRecorder, StandardLoggingPayload]: upstream_response: Final = upstream or _UpstreamThatTimesOutAfterThreeDeltas() recorder = FailureRecorder() @@ -289,7 +290,7 @@ async def _failure_payload_after_mid_stream_timeout( responses_api_provider_config=OpenAIResponsesAPIConfig(), logging_obj=logging_obj, custom_llm_provider="openai", - request_data={"model": "gpt-5.4-nano", "input": request_input, "stream": True}, + request_data={"model": "gpt-5.4-nano", "input": request_input, "stream": True, "background": background}, ) received = _ChunkRecorder() @@ -335,6 +336,17 @@ async def test_mid_stream_read_timeout_bills_the_tool_call_arguments_already_str assert payload["response_cost"] > 0 +@pytest.mark.asyncio +async def test_mid_stream_failure_of_a_background_response_bills_nothing_since_retrieving_it_bills_the_whole_run( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _, payload = await _failure_payload_after_mid_stream_timeout(monkeypatch, _SEA_PROMPT, background=True) + + assert payload["status"] == "failure" + assert payload["completion_tokens"] == 0 + assert payload["response_cost"] == 0 + + @pytest.mark.asyncio async def test_mid_stream_failure_usage_estimate_never_fetches_image_dimensions( monkeypatch: pytest.MonkeyPatch, From 88075bd45af35cc618021413f97a398fa563a7f2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:36:01 -0700 Subject: [PATCH 07/10] test: wait on events instead of fixed sleeps in the bridged failure and fallback tests --- .../test_responses_adapters_streaming_iterator.py | 8 +++++--- tests/unit/test_router/test_router.py | 1 - 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py index 851e6392e32..5264df3c9c5 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py +++ b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py @@ -793,16 +793,17 @@ class TestUpstreamFailureReachesTheProxyFailureHook: def test_a_client_disconnect_during_the_failure_hook_lets_the_hook_finish(self): hook_started = anyio.Event() + hook_released = anyio.Event() finished: list[Exception] = [] - async def _slow_hook(failure: Exception) -> None: + async def _held_hook(failure: Exception) -> None: hook_started.set() - await anyio.sleep(0.05) + await hook_released.wait() finished.append(failure) async def _disconnect_mid_hook() -> None: wrapper = AnthropicResponsesStreamWrapper( - responses_stream=_failing_after_one_delta(), model="m", litellm_logging_obj=_logging_obj_with_failure_hook(_slow_hook) + responses_stream=_failing_after_one_delta(), model="m", litellm_logging_obj=_logging_obj_with_failure_hook(_held_hook) ) async def _consume() -> None: @@ -813,6 +814,7 @@ class TestUpstreamFailureReachesTheProxyFailureHook: task_group.start_soon(_consume) await hook_started.wait() task_group.cancel_scope.cancel() + hook_released.set() asyncio.run(_disconnect_mid_hook()) assert len(finished) == 1 diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index a78c30055fc..07d3c23aabf 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -3745,7 +3745,6 @@ async def test_responses_fallback_sharing_the_proxy_logging_object_logs_the_fall ) _ = [event async for event in response] await asyncio.wait_for(recorder.succeeded.wait(), timeout=10) - await asyncio.sleep(0.5) success: Final = recorder.successes[-1] assert (success["prompt_tokens"], success["completion_tokens"]) == (1000, 700) From a7832c290ab6e817799fbb2698dcf85a87fbccb8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 14:52:02 -0700 Subject: [PATCH 08/10] fix(proxy): report a bridged stream failure once, after the client stream's own errors, and skip disconnect billing for failed or background Responses streams --- litellm/proxy/common_request_processing.py | 42 +++++++++ litellm/responses/streaming_iterator.py | 13 ++- .../proxy/test_common_request_processing.py | 92 ++++++++++++++++--- .../test_responses_streaming_iterator.py | 36 +++++++- 4 files changed, 164 insertions(+), 19 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 5ff6f5c92fd..b57ff91f650 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -4,6 +4,7 @@ import json import logging import math from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Coroutine, Mapping, Sequence +from dataclasses import dataclass from datetime import datetime from functools import lru_cache from types import MappingProxyType @@ -1962,6 +1963,43 @@ def _content_text(fields: Mapping[str, object]) -> str: return str(fields.get("content", "")) +async def _no_stream_failure_to_release() -> None: + return None + + +@dataclass(slots=True) +class _ClientStreamFailureReport: + logging_obj: LiteLLMLoggingObj + report: Callable[[Exception], Awaitable[None]] + held: Exception | None = None + + @classmethod + def hold_while_streaming(cls, logging_obj: LiteLLMLoggingObj | None) -> Callable[[], Awaitable[None]]: + if logging_obj is None or logging_obj.on_detached_stream_failure is None: + return _no_stream_failure_to_release + return cls(logging_obj, logging_obj.on_detached_stream_failure)._start_holding() + + def _start_holding(self) -> Callable[[], Awaitable[None]]: + self.logging_obj.on_detached_stream_failure = self.hold + return self.release + + async def hold(self, failure: Exception) -> None: + if self.held is None: + self.held = failure + + async def release(self) -> None: + self.logging_obj.on_detached_stream_failure = self.report + if self.held is None: + return + try: + with anyio.CancelScope(shield=True): + await self.report(self.held) + except Exception as hook_failure: # noqa: BLE001 # a failing failure hook must not skip the stream cleanup + verbose_proxy_logger.warning( + "Stream failure hook raised: %s(%s)", type(hook_failure).__name__, sanitize_for_log(hook_failure) + ) + + class ProxyBaseLLMRequestProcessing: def __init__(self, data: dict, served_model: str | None = None): self.data = data @@ -4404,6 +4442,9 @@ class ProxyBaseLLMRequestProcessing: response=response, request_data=request_data, ) + release_stream_failure: Final = _ClientStreamFailureReport.hold_while_streaming( + stream_logging_obj if isinstance(stream_logging_obj, LiteLLMLoggingObj) else None + ) try: str_so_far = "" async for chunk in guarded_stream: @@ -4508,6 +4549,7 @@ class ProxyBaseLLMRequestProcessing: seal: Final = "" if seal_open_frame is None else seal_open_frame(recent_tail) yield seal + error_frame if seal else error_frame finally: + await release_stream_failure() await ProxyBaseLLMRequestProcessing.finalize_streaming_generator_cleanup( request=request, request_data=request_data, diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 6edb1fe10a0..8ec2d2563d8 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -1067,12 +1067,14 @@ class BaseResponsesAPIStreamingIterator: the same shape the success handler assembles for a natural end of stream. Its response is the response.created payload (it carries the served service_tier) plus the text and tool arguments relayed so far, with counted - usage since the provider never sent any. None when nothing was relayed yet - or the stream already reached its terminal event and billed itself. + usage since the provider never sent any. None when nothing was relayed yet, + the stream already reached its terminal event and billed itself, its failure + log already billed the relayed output, or it runs in the background, where + retrieving it bills the whole run. """ created: Final = self._created_response generated: Final = self._generated_content + self._generated_tool_arguments - if self.completed_response is not None or created is None or not generated: + if created is None or not generated or self._billed_without_disconnect(): return None usage: Final = _estimate_usage_safely( self.model or "", @@ -1093,6 +1095,11 @@ class BaseResponsesAPIStreamingIterator: stamp_responses_usage_cost(partial, self.logging_obj) return ResponseIncompleteEvent(type=ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, response=partial) + def _billed_without_disconnect(self) -> bool: + return ( + self.completed_response is not None or self._failure_handled or self.request_data.get("background") is True + ) + def _note_yielded_event(self, event: ResponsesAPIStreamingResponse) -> None: self._yielded_first_chunk = True if event.type not in PRE_OUTPUT_LIFECYCLE_EVENT_TYPES: diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 8b56c046235..4bca44dbe3c 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -10815,27 +10815,93 @@ class TestDetachedStreamFailureHook: ) return logging_obj, stream + @staticmethod + def _hook_reporting(upstream_failure: Exception, after_report=None): + def iterator_hook_for(logging_obj): + async def iterator_hook(**_kwargs): + yield b"event: message_start\n\n" + await logging_obj.on_detached_stream_failure(upstream_failure) + yield b"event: error\n\n" + yield b"event: message_stop\n\n" + if after_report is not None: + raise after_report + + return iterator_hook + + return iterator_hook_for + @pytest.mark.asyncio - async def test_failure_reported_by_the_stream_reaches_the_failure_hook_once_when_an_iterator_hook_raises_after_it( - self, - ): + async def test_iterator_hook_failure_after_a_bridged_failure_is_the_one_the_failure_hook_gets(self): recorder = _FailureHookRecorder() upstream_failure = RuntimeError("upstream Responses stream failed") hook_failure = RuntimeError("iterator hook rejected the stream") - def iterator_hook_for(logging_obj): - async def iterator_hook(**_kwargs): - yield b"event: error\n\n" - await logging_obj.on_detached_stream_failure(upstream_failure) - raise hook_failure - - return iterator_hook - - _, stream = self._armed_stream(recorder, iterator_hook_for) + _, stream = self._armed_stream(recorder, self._hook_reporting(upstream_failure, after_report=hook_failure)) frames = [frame async for frame in stream] - assert frames == ["data: chunk\n\n", "data: error\n\n"] + assert frames == ["data: chunk\n\n"] * 3 + ["data: error\n\n"] + assert [call["original_exception"] for call in recorder.calls] == [hook_failure] + + @pytest.mark.asyncio + async def test_failure_hook_replacing_the_iterator_hook_failure_aborts_the_stream(self): + replacement = HTTPException(status_code=418, detail="rejected by a custom failure hook") + recorder = _FailureHookRecorder(raises=replacement) + upstream_failure = RuntimeError("upstream Responses stream failed") + hook_failure = RuntimeError("iterator hook rejected the stream") + + _, stream = self._armed_stream(recorder, self._hook_reporting(upstream_failure, after_report=hook_failure)) + + frames = [await stream.__anext__() for _ in range(3)] + with pytest.raises(HTTPException) as raised: + await stream.__anext__() + + assert raised.value is replacement + assert frames == ["data: chunk\n\n"] * 3 + assert [call["original_exception"] for call in recorder.calls] == [hook_failure] + + @pytest.mark.asyncio + async def test_failure_reported_by_the_stream_reaches_the_failure_hook_once_after_the_stream_ends(self): + recorder = _FailureHookRecorder() + upstream_failure = RuntimeError("upstream Responses stream failed") + + _, stream = self._armed_stream(recorder, self._hook_reporting(upstream_failure)) + + frames = [] + async for frame in stream: + frames.append(frame) + assert recorder.calls == [] + + assert frames == ["data: chunk\n\n"] * 3 + assert [call["original_exception"] for call in recorder.calls] == [upstream_failure] + + @pytest.mark.asyncio + async def test_failure_reported_by_the_stream_reaches_the_failure_hook_once_when_the_client_leaves_early(self): + recorder = _FailureHookRecorder() + upstream_failure = RuntimeError("upstream Responses stream failed") + + _, stream = self._armed_stream(recorder, self._hook_reporting(upstream_failure)) + + await stream.__anext__() + await stream.__anext__() + await stream.aclose() + + assert [call["original_exception"] for call in recorder.calls] == [upstream_failure] + + @pytest.mark.asyncio + async def test_failure_reported_after_the_stream_ends_still_reaches_the_failure_hook(self): + recorder = _FailureHookRecorder() + upstream_failure = RuntimeError("upstream Responses stream failed after the client stream closed") + + async def iterator_hook(**_kwargs): + yield b"event: message_start\n\n" + + logging_obj, stream = self._armed_stream(recorder, lambda _logging_obj: iterator_hook) + + frames = [frame async for frame in stream] + await logging_obj.on_detached_stream_failure(upstream_failure) + + assert frames == ["data: chunk\n\n"] assert [call["original_exception"] for call in recorder.calls] == [upstream_failure] @pytest.mark.asyncio diff --git a/tests/unit/responses/test_responses_streaming_iterator.py b/tests/unit/responses/test_responses_streaming_iterator.py index 4dbe795ab1b..9f64540d6ba 100644 --- a/tests/unit/responses/test_responses_streaming_iterator.py +++ b/tests/unit/responses/test_responses_streaming_iterator.py @@ -440,17 +440,28 @@ def _completed_after(text: str) -> Mapping[str, object]: } -def _live_iterator(logging_obj: LitellmLogging) -> ResponsesAPIStreamingIterator: +def _live_iterator( + logging_obj: LitellmLogging, upstream: object = None, background: bool = False +) -> ResponsesAPIStreamingIterator: return ResponsesAPIStreamingIterator( - response=httpx.Response(200), + response=cast(httpx.Response, upstream or httpx.Response(200)), model="gpt-5.4-nano", responses_api_provider_config=OpenAIResponsesAPIConfig(), logging_obj=logging_obj, - request_data={"model": "gpt-5.4-nano", "input": "tell me a story"}, + request_data={"model": "gpt-5.4-nano", "input": "tell me a story", "background": background}, custom_llm_provider="openai", ) +class _UpstreamThatTimesOutAfterRelayingText: + headers: dict[str, str] = {} + + async def aiter_bytes(self) -> AsyncIterator[bytes]: + for event in (_CREATED_WITH_PRIORITY_TIER, _text_delta("Once upon a time", 1)): + yield f"data: {json.dumps(event)}\n\n".encode() + raise httpx.ReadTimeout("Timeout on reading data from socket") + + def test_partial_response_on_disconnect_bills_the_relayed_text_at_the_served_service_tier(): iterator: Final = _live_iterator(_make_logging_obj()) for event in (_CREATED_WITH_PRIORITY_TIER, _text_delta("Once upon a time ", 1), _text_delta("a fox slept.", 2)): @@ -492,3 +503,22 @@ async def test_partial_response_on_disconnect_is_none_when_there_is_nothing_to_b assert iterator.partial_response_on_disconnect() is None await iterator._await_pending_logging() + + +@pytest.mark.asyncio +async def test_partial_response_on_disconnect_is_none_once_the_failure_log_billed_the_relayed_text(): + iterator: Final = _live_iterator(_make_logging_obj(), upstream=_UpstreamThatTimesOutAfterRelayingText()) + + with pytest.raises(litellm.Timeout): + await _ChunkRecorder().drain(iterator) + + assert iterator.partial_response_on_disconnect() is None + await iterator._await_pending_logging() + + +def test_partial_response_on_disconnect_is_none_for_a_background_response(): + iterator: Final = _live_iterator(_make_logging_obj(), background=True) + for event in (_CREATED_WITH_PRIORITY_TIER, _text_delta("Once upon a time", 1)): + iterator._process_chunk(json.dumps(event)) + + assert iterator.partial_response_on_disconnect() is None From cb581a4cd083fda69de33f0ab0d721928fa1ad26 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:50:32 -0700 Subject: [PATCH 09/10] fix(proxy): keep the bridged failure report and stream cleanup alive through a client cancel, bill failed estimates at the served tier, and count extra_body background --- litellm/proxy/common_request_processing.py | 34 +++++---- litellm/responses/streaming_iterator.py | 20 +++--- .../proxy/test_common_request_processing.py | 37 +++++++++- .../test_responses_streaming_iterator.py | 71 +++++++++++++++++-- 4 files changed, 131 insertions(+), 31 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index b57ff91f650..6f8ef830ee6 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -129,7 +129,7 @@ from litellm.proxy.utils import ( # noqa: F401 # legacy module exports _check_and_merge_model_level_guardrails, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export check_and_merge_model_level_guardrails, ) -from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect +from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect, spawn_logging_task from litellm.router import Router from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict from litellm.router_utils.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX @@ -1991,9 +1991,15 @@ class _ClientStreamFailureReport: self.logging_obj.on_detached_stream_failure = self.report if self.held is None: return + await asyncio.shield( + spawn_logging_task( + asyncio.get_running_loop(), self._report_held(self.held), task_name="stream failure report" + ) + ) + + async def _report_held(self, failure: Exception) -> None: try: - with anyio.CancelScope(shield=True): - await self.report(self.held) + await self.report(failure) except Exception as hook_failure: # noqa: BLE001 # a failing failure hook must not skip the stream cleanup verbose_proxy_logger.warning( "Stream failure hook raised: %s(%s)", type(hook_failure).__name__, sanitize_for_log(hook_failure) @@ -4549,16 +4555,18 @@ class ProxyBaseLLMRequestProcessing: seal: Final = "" if seal_open_frame is None else seal_open_frame(recent_tail) yield seal + error_frame if seal else error_frame finally: - await release_stream_failure() - await ProxyBaseLLMRequestProcessing.finalize_streaming_generator_cleanup( - request=request, - request_data=request_data, - response=response, - stream_completed=stream_completed, - client_disconnected=client_disconnected, - user_api_key_dict=user_api_key_dict, - proxy_logging_obj=proxy_logging_obj, - ) + try: + await release_stream_failure() + finally: + await ProxyBaseLLMRequestProcessing.finalize_streaming_generator_cleanup( + request=request, + request_data=request_data, + response=response, + stream_completed=stream_completed, + client_disconnected=client_disconnected, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + ) @staticmethod def async_sse_data_generator( diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 8ec2d2563d8..22a97a07d3b 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -41,7 +41,7 @@ from litellm.exceptions import MidStreamFallbackError, RateLimitError from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_api_base -from litellm.litellm_core_utils.internal_call_metadata import without_budget_reservation +from litellm.litellm_core_utils.internal_call_metadata import requests_background, without_budget_reservation from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base from litellm.litellm_core_utils.llm_response_utils.get_headers import provider_request_id_headers @@ -974,7 +974,7 @@ class BaseResponsesAPIStreamingIterator: generated_text: Final = self._generated_content + self._generated_tool_arguments if ( not generated_text - or self.request_data.get("background") is True + or requests_background(self.request_data) or self.logging_obj.model_call_details.get("combined_usage_object") is not None ): return @@ -990,7 +990,12 @@ class BaseResponsesAPIStreamingIterator: usage: Final = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( # pyright: ignore[reportPrivateUsage] # same shared transform the success path above uses estimate ) - self.logging_obj.record_assembled_response_for_failure(ModelResponse(model=self.model, usage=usage)) + served_tier: Final[object] = getattr(self._created_response, "service_tier", None) + self.logging_obj.record_assembled_response_for_failure( + ModelResponse( + model=self.model, usage=usage, service_tier=served_tier if isinstance(served_tier, str) else None + ) + ) def _handle_failure(self, exception: Exception): """ @@ -1068,9 +1073,8 @@ class BaseResponsesAPIStreamingIterator: Its response is the response.created payload (it carries the served service_tier) plus the text and tool arguments relayed so far, with counted usage since the provider never sent any. None when nothing was relayed yet, - the stream already reached its terminal event and billed itself, its failure - log already billed the relayed output, or it runs in the background, where - retrieving it bills the whole run. + the stream already reached its terminal event and billed itself, or its failure + log already billed the relayed output. """ created: Final = self._created_response generated: Final = self._generated_content + self._generated_tool_arguments @@ -1096,9 +1100,7 @@ class BaseResponsesAPIStreamingIterator: return ResponseIncompleteEvent(type=ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, response=partial) def _billed_without_disconnect(self) -> bool: - return ( - self.completed_response is not None or self._failure_handled or self.request_data.get("background") is True - ) + return self.completed_response is not None or self._failure_handled def _note_yielded_event(self, event: ResponsesAPIStreamingResponse) -> None: self._yielded_first_chunk = True diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 4bca44dbe3c..ebef1fbbef7 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -10676,12 +10676,19 @@ class TestResolveLitellmCallId: class _FailureHookRecorder: """Stands in for ProxyLogging.post_call_failure_hook, recording what the detached-failure closure hands it.""" - def __init__(self, raises: Optional[Exception] = None): + def __init__(self, raises: Optional[Exception] = None, gate: Optional[asyncio.Event] = None): self.calls = [] + self.entered = asyncio.Event() + self.reported = asyncio.Event() self._raises = raises + self._gate = gate async def post_call_failure_hook(self, **kwargs): + self.entered.set() + if self._gate is not None: + await self._gate.wait() self.calls.append(kwargs) + self.reported.set() if self._raises is not None: raise self._raises @@ -10782,7 +10789,7 @@ class TestDetachedStreamFailureHook: assert [call["original_exception"] for call in recorder.calls] == [failure] @staticmethod - def _armed_stream(recorder: _FailureHookRecorder, iterator_hook_for): + def _armed_stream(recorder: _FailureHookRecorder, iterator_hook_for, response: Optional[MagicMock] = None): from litellm.proxy._types import UserAPIKeyAuth as RealUserAPIKeyAuth logging_obj = LiteLLMLoggingObj( @@ -10806,7 +10813,7 @@ class TestDetachedStreamFailureHook: proxy_logging_obj=proxy_logging_obj, ) stream = ProxyBaseLLMRequestProcessing.async_streaming_data_generator( - response=MagicMock(), + response=MagicMock() if response is None else response, user_api_key_dict=user_api_key_dict, request_data=processing_obj.data, proxy_logging_obj=proxy_logging_obj, @@ -10875,6 +10882,30 @@ class TestDetachedStreamFailureHook: assert frames == ["data: chunk\n\n"] * 3 assert [call["original_exception"] for call in recorder.calls] == [upstream_failure] + @pytest.mark.asyncio + async def test_failure_report_and_stream_cleanup_survive_the_client_task_being_cancelled_mid_report(self): + gate = asyncio.Event() + recorder = _FailureHookRecorder(gate=gate) + upstream_failure = RuntimeError("upstream Responses stream failed") + response = MagicMock() + response.aclose = AsyncMock() + + _, stream = self._armed_stream(recorder, self._hook_reporting(upstream_failure), response=response) + + async def client_reads_the_whole_stream(): + return [frame async for frame in stream] + + client = asyncio.create_task(client_reads_the_whole_stream()) + await asyncio.wait_for(recorder.entered.wait(), timeout=5) + client.cancel() + with pytest.raises(asyncio.CancelledError): + await client + + response.aclose.assert_awaited_once() + gate.set() + await asyncio.wait_for(recorder.reported.wait(), timeout=5) + assert [call["original_exception"] for call in recorder.calls] == [upstream_failure] + @pytest.mark.asyncio async def test_failure_reported_by_the_stream_reaches_the_failure_hook_once_when_the_client_leaves_early(self): recorder = _FailureHookRecorder() diff --git a/tests/unit/responses/test_responses_streaming_iterator.py b/tests/unit/responses/test_responses_streaming_iterator.py index 9f64540d6ba..c572b4151b2 100644 --- a/tests/unit/responses/test_responses_streaming_iterator.py +++ b/tests/unit/responses/test_responses_streaming_iterator.py @@ -270,7 +270,7 @@ async def _failure_payload_after_mid_stream_timeout( monkeypatch: pytest.MonkeyPatch, request_input: str | list[dict[str, object]], upstream: _UpstreamThatTimesOutAfterThreeDeltas | None = None, - background: bool = False, + background_fields: Mapping[str, object] | None = None, ) -> tuple[FailureRecorder, StandardLoggingPayload]: upstream_response: Final = upstream or _UpstreamThatTimesOutAfterThreeDeltas() recorder = FailureRecorder() @@ -290,7 +290,7 @@ async def _failure_payload_after_mid_stream_timeout( responses_api_provider_config=OpenAIResponsesAPIConfig(), logging_obj=logging_obj, custom_llm_provider="openai", - request_data={"model": "gpt-5.4-nano", "input": request_input, "stream": True, "background": background}, + request_data={"model": "gpt-5.4-nano", "input": request_input, "stream": True, **(background_fields or {})}, ) received = _ChunkRecorder() @@ -336,11 +336,18 @@ async def test_mid_stream_read_timeout_bills_the_tool_call_arguments_already_str assert payload["response_cost"] > 0 +@pytest.mark.parametrize( + "background_fields", + [{"background": True}, {"extra_body": {"background": True}}], + ids=["top_level", "extra_body"], +) @pytest.mark.asyncio async def test_mid_stream_failure_of_a_background_response_bills_nothing_since_retrieving_it_bills_the_whole_run( - monkeypatch: pytest.MonkeyPatch, + monkeypatch: pytest.MonkeyPatch, background_fields: Mapping[str, object] ) -> None: - _, payload = await _failure_payload_after_mid_stream_timeout(monkeypatch, _SEA_PROMPT, background=True) + _, payload = await _failure_payload_after_mid_stream_timeout( + monkeypatch, _SEA_PROMPT, background_fields=background_fields + ) assert payload["status"] == "failure" assert payload["completion_tokens"] == 0 @@ -462,6 +469,55 @@ class _UpstreamThatTimesOutAfterRelayingText: raise httpx.ReadTimeout("Timeout on reading data from socket") +class _UpstreamServedAtTierThatTimesOut: + headers: dict[str, str] = {} + + def __init__(self, served_tier: str) -> None: + self.served_tier = served_tier + + async def aiter_bytes(self) -> AsyncIterator[bytes]: + created: Final = { + **_CREATED_WITH_PRIORITY_TIER, + "response": {**_CREATED_RESPONSE_WITH_PRIORITY_TIER, "service_tier": self.served_tier}, + } + for event in (created, _text_delta("Once upon a time", 1)): + yield f"data: {json.dumps(event)}\n\n".encode() + raise httpx.ReadTimeout("Timeout on reading data from socket") + + +async def _mid_stream_failure_cost( + monkeypatch: pytest.MonkeyPatch, requested_tier: str | None, served_tier: str +) -> float: + recorder: Final = FailureRecorder() + monkeypatch.setattr(litellm, "failure_callback", [recorder]) + monkeypatch.setattr(litellm, "_async_failure_callback", [recorder]) + logging_obj: Final = _make_logging_obj() + logging_obj.update_environment_variables( + model="gpt-5.4-nano", + optional_params={"stream": True} if requested_tier is None else {"stream": True, "service_tier": requested_tier}, + litellm_params={"custom_llm_provider": "openai", "aresponses": True}, + ) + iterator: Final = _live_iterator(logging_obj, upstream=_UpstreamServedAtTierThatTimesOut(served_tier)) + + with pytest.raises(litellm.Timeout): + await _ChunkRecorder().drain(iterator) + await iterator._await_pending_logging() + + (payload,) = recorder.failure_payloads + return payload["response_cost"] + + +@pytest.mark.asyncio +async def test_mid_stream_failure_bills_the_relayed_text_at_the_tier_the_provider_served( + monkeypatch: pytest.MonkeyPatch, +) -> None: + flex_requested_default_served: Final = await _mid_stream_failure_cost(monkeypatch, "flex", "default") + nothing_requested_default_served: Final = await _mid_stream_failure_cost(monkeypatch, None, "default") + + assert nothing_requested_default_served > 0 + assert flex_requested_default_served == nothing_requested_default_served + + def test_partial_response_on_disconnect_bills_the_relayed_text_at_the_served_service_tier(): iterator: Final = _live_iterator(_make_logging_obj()) for event in (_CREATED_WITH_PRIORITY_TIER, _text_delta("Once upon a time ", 1), _text_delta("a fox slept.", 2)): @@ -516,9 +572,12 @@ async def test_partial_response_on_disconnect_is_none_once_the_failure_log_bille await iterator._await_pending_logging() -def test_partial_response_on_disconnect_is_none_for_a_background_response(): +def test_partial_response_on_disconnect_reports_the_relayed_text_of_a_background_response(): iterator: Final = _live_iterator(_make_logging_obj(), background=True) for event in (_CREATED_WITH_PRIORITY_TIER, _text_delta("Once upon a time", 1)): iterator._process_chunk(json.dumps(event)) - assert iterator.partial_response_on_disconnect() is None + incomplete_event: Final = iterator.partial_response_on_disconnect() + + assert incomplete_event is not None + assert incomplete_event.response.output[0].content[0].text == "Once upon a time" From 599eae642095429084c8dc42ed23bca24cc7c172 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:55:49 -0700 Subject: [PATCH 10/10] fix(responses): bill no failure estimate when the provider says the response runs in background A stream sent with background in extra_body never carries it in the iterator's request context, so the failure estimate double-billed against the later retrieve. response.created echoes the flag, so read it there too. --- litellm/responses/streaming_iterator.py | 7 ++- .../test_responses_streaming_iterator.py | 45 ++++++++++--------- 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 22a97a07d3b..3ecfc31f632 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -41,7 +41,11 @@ from litellm.exceptions import MidStreamFallbackError, RateLimitError from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_api_base -from litellm.litellm_core_utils.internal_call_metadata import requests_background, without_budget_reservation +from litellm.litellm_core_utils.internal_call_metadata import ( + is_background_response, + requests_background, + without_budget_reservation, +) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base from litellm.litellm_core_utils.llm_response_utils.get_headers import provider_request_id_headers @@ -975,6 +979,7 @@ class BaseResponsesAPIStreamingIterator: if ( not generated_text or requests_background(self.request_data) + or is_background_response(self._created_response) or self.logging_obj.model_call_details.get("combined_usage_object") is not None ): return diff --git a/tests/unit/responses/test_responses_streaming_iterator.py b/tests/unit/responses/test_responses_streaming_iterator.py index c572b4151b2..f4d6240e593 100644 --- a/tests/unit/responses/test_responses_streaming_iterator.py +++ b/tests/unit/responses/test_responses_streaming_iterator.py @@ -336,17 +336,12 @@ async def test_mid_stream_read_timeout_bills_the_tool_call_arguments_already_str assert payload["response_cost"] > 0 -@pytest.mark.parametrize( - "background_fields", - [{"background": True}, {"extra_body": {"background": True}}], - ids=["top_level", "extra_body"], -) @pytest.mark.asyncio async def test_mid_stream_failure_of_a_background_response_bills_nothing_since_retrieving_it_bills_the_whole_run( - monkeypatch: pytest.MonkeyPatch, background_fields: Mapping[str, object] + monkeypatch: pytest.MonkeyPatch, ) -> None: _, payload = await _failure_payload_after_mid_stream_timeout( - monkeypatch, _SEA_PROMPT, background_fields=background_fields + monkeypatch, _SEA_PROMPT, background_fields={"background": True} ) assert payload["status"] == "failure" @@ -469,25 +464,25 @@ class _UpstreamThatTimesOutAfterRelayingText: raise httpx.ReadTimeout("Timeout on reading data from socket") -class _UpstreamServedAtTierThatTimesOut: +class _UpstreamThatTimesOutAfterCreated: headers: dict[str, str] = {} - def __init__(self, served_tier: str) -> None: - self.served_tier = served_tier + def __init__(self, **created_fields: object) -> None: + self.created_fields = created_fields async def aiter_bytes(self) -> AsyncIterator[bytes]: created: Final = { **_CREATED_WITH_PRIORITY_TIER, - "response": {**_CREATED_RESPONSE_WITH_PRIORITY_TIER, "service_tier": self.served_tier}, + "response": {**_CREATED_RESPONSE_WITH_PRIORITY_TIER, **self.created_fields}, } for event in (created, _text_delta("Once upon a time", 1)): yield f"data: {json.dumps(event)}\n\n".encode() raise httpx.ReadTimeout("Timeout on reading data from socket") -async def _mid_stream_failure_cost( - monkeypatch: pytest.MonkeyPatch, requested_tier: str | None, served_tier: str -) -> float: +async def _mid_stream_failure_payload( + monkeypatch: pytest.MonkeyPatch, requested_tier: str | None, **created_fields: object +) -> StandardLoggingPayload: recorder: Final = FailureRecorder() monkeypatch.setattr(litellm, "failure_callback", [recorder]) monkeypatch.setattr(litellm, "_async_failure_callback", [recorder]) @@ -497,25 +492,35 @@ async def _mid_stream_failure_cost( optional_params={"stream": True} if requested_tier is None else {"stream": True, "service_tier": requested_tier}, litellm_params={"custom_llm_provider": "openai", "aresponses": True}, ) - iterator: Final = _live_iterator(logging_obj, upstream=_UpstreamServedAtTierThatTimesOut(served_tier)) + iterator: Final = _live_iterator(logging_obj, upstream=_UpstreamThatTimesOutAfterCreated(**created_fields)) with pytest.raises(litellm.Timeout): await _ChunkRecorder().drain(iterator) await iterator._await_pending_logging() (payload,) = recorder.failure_payloads - return payload["response_cost"] + return payload @pytest.mark.asyncio async def test_mid_stream_failure_bills_the_relayed_text_at_the_tier_the_provider_served( monkeypatch: pytest.MonkeyPatch, ) -> None: - flex_requested_default_served: Final = await _mid_stream_failure_cost(monkeypatch, "flex", "default") - nothing_requested_default_served: Final = await _mid_stream_failure_cost(monkeypatch, None, "default") + flex_requested: Final = await _mid_stream_failure_payload(monkeypatch, "flex", service_tier="default") + nothing_requested: Final = await _mid_stream_failure_payload(monkeypatch, None, service_tier="default") - assert nothing_requested_default_served > 0 - assert flex_requested_default_served == nothing_requested_default_served + assert nothing_requested["response_cost"] > 0 + assert flex_requested["response_cost"] == nothing_requested["response_cost"] + + +@pytest.mark.asyncio +async def test_mid_stream_failure_bills_nothing_when_the_provider_runs_the_response_in_background( + monkeypatch: pytest.MonkeyPatch, +) -> None: + payload: Final = await _mid_stream_failure_payload(monkeypatch, None, background=True) + + assert payload["status"] == "failure" + assert (payload["completion_tokens"], payload["response_cost"]) == (0, 0) def test_partial_response_on_disconnect_bills_the_relayed_text_at_the_served_service_tier():