From 327d4e20e1275b835c7915f26229e8aa50eebbce Mon Sep 17 00:00:00 2001 From: Rodrigo-Palma Date: Fri, 3 Jul 2026 09:00:35 -0300 Subject: [PATCH 1/7] fix(logging): stop dropping streaming /v1/messages success logs Two silent-loss paths for anthropic_messages streaming success logs: - get_standard_logging_object_payload swallows exceptions and returns None; loggers gated on standard_logging_object (e.g. s3_v2) then skip the event while spend tracking still succeeds via the response_cost fallback, so requests disappear from success logs with no error. Rebuild the payload from an empty response object when construction fails for an assembled streaming response, mirroring the failure-path fallback. Fixes #32019. - _handle_streaming_logging scheduled end-of-stream logging with an unreferenced asyncio.create_task, which can be garbage-collected before it runs. Enqueue on the logging worker instead, matching the fix applied to pass-through routes in #31485. --- litellm/litellm_core_utils/litellm_logging.py | 23 +++++ .../test_litellm_logging_slo_fallback.py | 94 +++++++++++++++++++ .../test_streaming_iterator_logging.py | 37 ++++++++ 3 files changed, 154 insertions(+) create mode 100644 tests/unit/litellm_core_utils/test_litellm_logging_slo_fallback.py create mode 100644 tests/unit/llms/anthropic/pass_through/messages/test_streaming_iterator_logging.py diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 154893b6c21..37a203cac0e 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3346,6 +3346,29 @@ class Logging(LiteLLMLoggingBaseClass): "LiteLLM.LoggingError: [Non-Blocking] Exception building the standard logging payload " "for a streaming response; callbacks still run without it" ) + if self.model_call_details.get("standard_logging_object") is None: + # get_standard_logging_object_payload swallows exceptions and + # returns None, and the except branch above leaves the key + # unset. Loggers gated on standard_logging_object (e.g. s3_v2) + # would then silently drop the request while spend tracking + # still succeeds via the response_cost fallback. Rebuild from an + # empty response object instead, mirroring the failure-path + # fallback. See #32019. + verbose_logger.error( + "standard_logging_object build failed for streaming call_id=%s call_type=%s; " + "rebuilding a degraded payload so success loggers still fire", + self.litellm_call_id, + self.call_type, + ) + try: + self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload( + {}, start_time, end_time + ) + except Exception: # noqa: BLE001 # same rule as above: never block later callbacks + verbose_logger.exception( + "LiteLLM.LoggingError: [Non-Blocking] Exception rebuilding the degraded standard " + "logging payload for a streaming response" + ) # print standard logging payload if (standard_logging_payload := self.model_call_details.get("standard_logging_object")) is not None: diff --git a/tests/unit/litellm_core_utils/test_litellm_logging_slo_fallback.py b/tests/unit/litellm_core_utils/test_litellm_logging_slo_fallback.py new file mode 100644 index 00000000000..60ad485fd1e --- /dev/null +++ b/tests/unit/litellm_core_utils/test_litellm_logging_slo_fallback.py @@ -0,0 +1,94 @@ +""" +Regression tests for https://github.com/BerriAI/litellm/issues/32019. + +Streaming /v1/messages responses were silently dropped by loggers gated on +``standard_logging_object`` (e.g. ``s3_v2``) whenever StandardLoggingPayload +construction failed: ``get_standard_logging_object_payload`` swallows the +exception and returns ``None``, spend tracking still succeeds via the +``response_cost`` fallback, and no error surfaces to the caller. +""" + +from datetime import datetime + +import pytest + +import litellm +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.types.utils import ModelResponse + +DEGRADED_PAYLOAD = {"trace_id": "degraded-fallback"} + + +@pytest.mark.asyncio +async def test_streaming_slo_build_failure_falls_back_to_degraded_payload(monkeypatch): + """When the SLP build returns None for an assembled streaming response, + async_success_handler must rebuild it from an empty response object so + SLP-gated loggers still record the request.""" + logging_obj = Logging( + model="claude-3-5-sonnet-20240620", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="anthropic_messages", + start_time=datetime.now(), + litellm_call_id="test-32019", + function_id="test-32019", + ) + build_calls = [] + + def fake_build(self, init_response_obj, start_time, end_time): + build_calls.append(init_response_obj) + if init_response_obj == {}: + return DEGRADED_PAYLOAD + return None # simulates get_standard_logging_object_payload swallowing an exception + + monkeypatch.setattr(Logging, "_build_standard_logging_payload", fake_build) + monkeypatch.setattr(litellm, "_async_success_callback", []) + monkeypatch.setattr(litellm, "success_callback", []) + + result = ModelResponse(model="claude-3-5-sonnet-20240620") + + await logging_obj.async_success_handler( + result, + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + ) + + assert logging_obj.model_call_details["standard_logging_object"] == DEGRADED_PAYLOAD + assert build_calls == [result, {}] + + +@pytest.mark.asyncio +async def test_streaming_slo_build_success_does_not_rebuild(monkeypatch): + """The degraded rebuild must not run when the SLP builds normally.""" + logging_obj = Logging( + model="claude-3-5-sonnet-20240620", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="anthropic_messages", + start_time=datetime.now(), + litellm_call_id="test-32019-ok", + function_id="test-32019-ok", + ) + build_calls = [] + healthy_payload = {"trace_id": "healthy"} + + def fake_build(self, init_response_obj, start_time, end_time): + build_calls.append(init_response_obj) + return healthy_payload + + monkeypatch.setattr(Logging, "_build_standard_logging_payload", fake_build) + monkeypatch.setattr(litellm, "_async_success_callback", []) + monkeypatch.setattr(litellm, "success_callback", []) + + result = ModelResponse(model="claude-3-5-sonnet-20240620") + + await logging_obj.async_success_handler( + result, + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + ) + + assert logging_obj.model_call_details["standard_logging_object"] == healthy_payload + assert len(build_calls) == 1 diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_streaming_iterator_logging.py b/tests/unit/llms/anthropic/pass_through/messages/test_streaming_iterator_logging.py new file mode 100644 index 00000000000..287e09099ac --- /dev/null +++ b/tests/unit/llms/anthropic/pass_through/messages/test_streaming_iterator_logging.py @@ -0,0 +1,37 @@ +""" +Regression test for https://github.com/BerriAI/litellm/issues/32019. + +End-of-stream logging for /v1/messages must be enqueued on the logging +worker instead of a bare ``asyncio.create_task``: an unreferenced task can +be garbage-collected before it runs, dropping the success log entirely +(same class of bug fixed for pass-through routes in #31485). +""" + +from unittest.mock import MagicMock + +import pytest + +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import ( + BaseAnthropicMessagesStreamingIterator, +) + + +@pytest.mark.asyncio +async def test_handle_streaming_logging_enqueues_on_logging_worker(monkeypatch): + enqueued = [] + + def fake_enqueue(async_coroutine): + enqueued.append(async_coroutine) + async_coroutine.close() + + monkeypatch.setattr(GLOBAL_LOGGING_WORKER, "ensure_initialized_and_enqueue", fake_enqueue) + + iterator = BaseAnthropicMessagesStreamingIterator( + litellm_logging_obj=MagicMock(), + request_body={"model": "claude-3-5-sonnet-20240620", "stream": True}, + ) + + await iterator._handle_streaming_logging(collected_chunks=[b"data: {}\n\n"]) + + assert len(enqueued) == 1 From 4d2713e8b09d21dd0cc747ba40b35bc1c79facea Mon Sep 17 00:00:00 2001 From: Rodrigo-Palma Date: Fri, 3 Jul 2026 09:16:15 -0300 Subject: [PATCH 2/7] fix(logging): surface persistent SLP rebuild failure Per review: if the empty-response retry also returns None, log a second explicit error so the double failure is observable instead of masked by the first message, and stop promising success in the first log line. --- litellm/litellm_core_utils/litellm_logging.py | 9 ++++- .../test_litellm_logging_slo_fallback.py | 39 +++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 37a203cac0e..2e90ee3a9b9 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3356,7 +3356,7 @@ class Logging(LiteLLMLoggingBaseClass): # fallback. See #32019. verbose_logger.error( "standard_logging_object build failed for streaming call_id=%s call_type=%s; " - "rebuilding a degraded payload so success loggers still fire", + "retrying with an empty response object", self.litellm_call_id, self.call_type, ) @@ -3364,6 +3364,13 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload( {}, start_time, end_time ) + if self.model_call_details["standard_logging_object"] is None: + verbose_logger.error( + "standard_logging_object rebuild failed for streaming call_id=%s call_type=%s; " + "loggers gated on standard_logging_object will skip this request", + self.litellm_call_id, + self.call_type, + ) except Exception: # noqa: BLE001 # same rule as above: never block later callbacks verbose_logger.exception( "LiteLLM.LoggingError: [Non-Blocking] Exception rebuilding the degraded standard " diff --git a/tests/unit/litellm_core_utils/test_litellm_logging_slo_fallback.py b/tests/unit/litellm_core_utils/test_litellm_logging_slo_fallback.py index 60ad485fd1e..55ee04ca52b 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging_slo_fallback.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging_slo_fallback.py @@ -8,6 +8,7 @@ exception and returns ``None``, spend tracking still succeeds via the ``response_cost`` fallback, and no error surfaces to the caller. """ +import logging from datetime import datetime import pytest @@ -92,3 +93,41 @@ async def test_streaming_slo_build_success_does_not_rebuild(monkeypatch): assert logging_obj.model_call_details["standard_logging_object"] == healthy_payload assert len(build_calls) == 1 + + +@pytest.mark.asyncio +async def test_streaming_slo_double_build_failure_is_logged(monkeypatch, caplog): + """If the empty-response rebuild also fails, the persistent failure must be + surfaced with a second error log instead of silently leaving the payload None.""" + logging_obj = Logging( + model="claude-3-5-sonnet-20240620", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="anthropic_messages", + start_time=datetime.now(), + litellm_call_id="test-32019-double", + function_id="test-32019-double", + ) + build_calls = [] + + def fake_build(self, init_response_obj, start_time, end_time): + build_calls.append(init_response_obj) + return None # both the primary build and the empty-response retry fail + + monkeypatch.setattr(Logging, "_build_standard_logging_payload", fake_build) + monkeypatch.setattr(litellm, "_async_success_callback", []) + monkeypatch.setattr(litellm, "success_callback", []) + + result = ModelResponse(model="claude-3-5-sonnet-20240620") + + with caplog.at_level(logging.ERROR, logger="LiteLLM"): + await logging_obj.async_success_handler( + result, + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + ) + + assert logging_obj.model_call_details["standard_logging_object"] is None + assert build_calls == [result, {}] + assert "rebuild failed" in caplog.text From d2be9c33746e164c978790fee6716fe40f84a837 Mon Sep 17 00:00:00 2001 From: Rodrigo-Palma Date: Fri, 25 Sep 2026 12:19:08 -0300 Subject: [PATCH 3/7] test: drop the streaming iterator enqueue test, now covered upstream _handle_streaming_logging already enqueues the end-of-stream logging coroutine on GLOBAL_LOGGING_WORKER on main, so the assertion no longer belongs to this change. What remains here is the standard_logging_object fallback in litellm_logging.py, which main still does not handle: the build can return None (exceptions are swallowed inside get_standard_logging_object_payload) and loggers gated on that key drop the request while spend tracking still succeeds. --- .../test_streaming_iterator_logging.py | 37 ------------------- 1 file changed, 37 deletions(-) delete mode 100644 tests/unit/llms/anthropic/pass_through/messages/test_streaming_iterator_logging.py diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_streaming_iterator_logging.py b/tests/unit/llms/anthropic/pass_through/messages/test_streaming_iterator_logging.py deleted file mode 100644 index 287e09099ac..00000000000 --- a/tests/unit/llms/anthropic/pass_through/messages/test_streaming_iterator_logging.py +++ /dev/null @@ -1,37 +0,0 @@ -""" -Regression test for https://github.com/BerriAI/litellm/issues/32019. - -End-of-stream logging for /v1/messages must be enqueued on the logging -worker instead of a bare ``asyncio.create_task``: an unreferenced task can -be garbage-collected before it runs, dropping the success log entirely -(same class of bug fixed for pass-through routes in #31485). -""" - -from unittest.mock import MagicMock - -import pytest - -from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import ( - BaseAnthropicMessagesStreamingIterator, -) - - -@pytest.mark.asyncio -async def test_handle_streaming_logging_enqueues_on_logging_worker(monkeypatch): - enqueued = [] - - def fake_enqueue(async_coroutine): - enqueued.append(async_coroutine) - async_coroutine.close() - - monkeypatch.setattr(GLOBAL_LOGGING_WORKER, "ensure_initialized_and_enqueue", fake_enqueue) - - iterator = BaseAnthropicMessagesStreamingIterator( - litellm_logging_obj=MagicMock(), - request_body={"model": "claude-3-5-sonnet-20240620", "stream": True}, - ) - - await iterator._handle_streaming_logging(collected_chunks=[b"data: {}\n\n"]) - - assert len(enqueued) == 1 From fcb90388ab478f669582f52bfa80aa6930e5fca7 Mon Sep 17 00:00:00 2001 From: Rodrigo-Palma Date: Fri, 25 Sep 2026 12:54:53 -0300 Subject: [PATCH 4/7] test: drop the explicit return None that ruff RET501 now rejects The stub still returns None, which is the whole point of the test; the annotation says so instead of the statement. --- .../litellm_core_utils/test_litellm_logging_slo_fallback.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/unit/litellm_core_utils/test_litellm_logging_slo_fallback.py b/tests/unit/litellm_core_utils/test_litellm_logging_slo_fallback.py index 55ee04ca52b..16031e93b3f 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging_slo_fallback.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging_slo_fallback.py @@ -110,9 +110,10 @@ async def test_streaming_slo_double_build_failure_is_logged(monkeypatch, caplog) ) build_calls = [] - def fake_build(self, init_response_obj, start_time, end_time): + def fake_build(self, init_response_obj, start_time, end_time) -> None: + # Returning None is the point: both the primary build and the + # empty-response retry fail. build_calls.append(init_response_obj) - return None # both the primary build and the empty-response retry fail monkeypatch.setattr(Logging, "_build_standard_logging_payload", fake_build) monkeypatch.setattr(litellm, "_async_success_callback", []) From d55ab571a9dc39346aa5ca5d8d0d60cbad51bc33 Mon Sep 17 00:00:00 2001 From: Rodrigo-Palma Date: Fri, 25 Sep 2026 13:52:01 -0300 Subject: [PATCH 5/7] test: cover the rebuild that raises The rebuild sits in its own try/except so a failing payload build cannot stop the later callbacks, which is the rule main wrote around the primary build. Nothing asserted that, and the except branch was the one part of this change with no coverage: the test now makes the empty-response retry raise and checks that the success callback still fires. --- .../test_litellm_logging_slo_fallback.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/unit/litellm_core_utils/test_litellm_logging_slo_fallback.py b/tests/unit/litellm_core_utils/test_litellm_logging_slo_fallback.py index 16031e93b3f..4410af7ef87 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging_slo_fallback.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging_slo_fallback.py @@ -132,3 +132,50 @@ async def test_streaming_slo_double_build_failure_is_logged(monkeypatch, caplog) assert logging_obj.model_call_details["standard_logging_object"] is None assert build_calls == [result, {}] assert "rebuild failed" in caplog.text + + +@pytest.mark.asyncio +async def test_streaming_slo_rebuild_raising_does_not_block_callbacks(monkeypatch, caplog): + """The rebuild is wrapped for the same reason main wraps the primary build: + a payload build must never stop the later callbacks from running. If the + empty-response retry raises, the success callback still fires.""" + logging_obj = Logging( + model="claude-3-5-sonnet-20240620", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="anthropic_messages", + start_time=datetime.now(), + litellm_call_id="test-32019-raise", + function_id="test-32019-raise", + ) + build_calls = [] + + def fake_build(self, init_response_obj, start_time, end_time): + build_calls.append(init_response_obj) + if init_response_obj == {}: + raise RuntimeError("rebuild blew up") + return None + + called_back = [] + + async def async_success_callback(kwargs, completion_response, start_time, end_time): + called_back.append(completion_response) + + monkeypatch.setattr(Logging, "_build_standard_logging_payload", fake_build) + monkeypatch.setattr(litellm, "_async_success_callback", [async_success_callback]) + monkeypatch.setattr(litellm, "success_callback", []) + + result = ModelResponse(model="claude-3-5-sonnet-20240620") + + with caplog.at_level(logging.ERROR, logger="LiteLLM"): + await logging_obj.async_success_handler( + result, + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + ) + + assert build_calls == [result, {}] + assert logging_obj.model_call_details["standard_logging_object"] is None + assert "Exception rebuilding the degraded standard logging payload" in caplog.text + assert len(called_back) == 1 From 7896f5c5d45fb9e43b9f6858bc558b161f0de33c Mon Sep 17 00:00:00 2001 From: Rodrigo-Palma Date: Thu, 1 Oct 2026 09:33:34 -0300 Subject: [PATCH 6/7] fix(logging): keep the degraded SLP rebuild within the type budgets --- litellm/litellm_core_utils/litellm_logging.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 2e90ee3a9b9..f8fc1c9a078 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3355,21 +3355,21 @@ class Logging(LiteLLMLoggingBaseClass): # empty response object instead, mirroring the failure-path # fallback. See #32019. verbose_logger.error( - "standard_logging_object build failed for streaming call_id=%s call_type=%s; " + "standard_logging_object build failed for streaming call_id=%s; " "retrying with an empty response object", self.litellm_call_id, - self.call_type, ) try: self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload( - {}, start_time, end_time + {}, # mutable-ok: empty response object, mirrors the failure-path fallback + start_time, # pyright: ignore[reportUnknownArgumentType] # untyped _success_handler_helper_fn return + end_time, # pyright: ignore[reportUnknownArgumentType] # untyped _success_handler_helper_fn return ) if self.model_call_details["standard_logging_object"] is None: verbose_logger.error( - "standard_logging_object rebuild failed for streaming call_id=%s call_type=%s; " + "standard_logging_object rebuild failed for streaming call_id=%s; " "loggers gated on standard_logging_object will skip this request", self.litellm_call_id, - self.call_type, ) except Exception: # noqa: BLE001 # same rule as above: never block later callbacks verbose_logger.exception( From 48151ad4181d825c0e1817eda043ec4ec76b2618 Mon Sep 17 00:00:00 2001 From: Rodrigo-Palma Date: Thu, 1 Oct 2026 10:21:02 -0300 Subject: [PATCH 7/7] fix(logging): keep the caller-supplied call id out of the rebuild log lines --- litellm/litellm_core_utils/litellm_logging.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index f8fc1c9a078..a52625639f1 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3355,9 +3355,8 @@ class Logging(LiteLLMLoggingBaseClass): # empty response object instead, mirroring the failure-path # fallback. See #32019. verbose_logger.error( - "standard_logging_object build failed for streaming call_id=%s; " - "retrying with an empty response object", - self.litellm_call_id, + "standard_logging_object build failed for a streaming response; " + "retrying with an empty response object" ) try: self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload( @@ -3367,9 +3366,8 @@ class Logging(LiteLLMLoggingBaseClass): ) if self.model_call_details["standard_logging_object"] is None: verbose_logger.error( - "standard_logging_object rebuild failed for streaming call_id=%s; " - "loggers gated on standard_logging_object will skip this request", - self.litellm_call_id, + "standard_logging_object rebuild failed for a streaming response; " + "loggers gated on standard_logging_object will skip this request" ) except Exception: # noqa: BLE001 # same rule as above: never block later callbacks verbose_logger.exception(