Merge 48151ad418 into 527d882508

litellm/litellm_core_utils/litellm_logging.py

@@ -4443,6 +4443,34 @@ class Logging(LiteLLMLoggingBaseClass):
                     "LiteLLM.LoggingError: [Non-Blocking] Exception building the standard logging payload "
                     "for a streaming response; callbacks still run without it"
                 )
+            if self.model_call_details.get("standard_logging_object") is None:
+                # get_standard_logging_object_payload swallows exceptions and
+                # returns None, and the except branch above leaves the key
+                # unset. Loggers gated on standard_logging_object (e.g. s3_v2)
+                # would then silently drop the request while spend tracking
+                # still succeeds via the response_cost fallback. Rebuild from an
+                # empty response object instead, mirroring the failure-path
+                # fallback. See #32019.
+                verbose_logger.error(
+                    "standard_logging_object build failed for a streaming response; "
+                    "retrying with an empty response object"
+                )
+                try:
+                    self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
+                        {},  # mutable-ok: empty response object, mirrors the failure-path fallback
+                        start_time,  # pyright: ignore[reportUnknownArgumentType]  # untyped _success_handler_helper_fn return
+                        end_time,  # pyright: ignore[reportUnknownArgumentType]  # untyped _success_handler_helper_fn return
+                    )
+                    if self.model_call_details["standard_logging_object"] is None:
+                        verbose_logger.error(
+                            "standard_logging_object rebuild failed for a streaming response; "
+                            "loggers gated on standard_logging_object will skip this request"
+                        )
+                except Exception:  # noqa: BLE001  # same rule as above: never block later callbacks
+                    verbose_logger.exception(
+                        "LiteLLM.LoggingError: [Non-Blocking] Exception rebuilding the degraded standard "
+                        "logging payload for a streaming response"
+                    )
 
             # print standard logging payload
             self._emit_stored_standard_logging_payload()

tests/unit/litellm_core_utils/test_litellm_logging_slo_fallback.py

@@ -0,0 +1,181 @@
+"""
+Regression tests for https://github.com/BerriAI/litellm/issues/32019.
+
+Streaming /v1/messages responses were silently dropped by loggers gated on
+``standard_logging_object`` (e.g. ``s3_v2``) whenever StandardLoggingPayload
+construction failed: ``get_standard_logging_object_payload`` swallows the
+exception and returns ``None``, spend tracking still succeeds via the
+``response_cost`` fallback, and no error surfaces to the caller.
+"""
+
+import logging
+from datetime import datetime
+
+import pytest
+
+import litellm
+from litellm.litellm_core_utils.litellm_logging import Logging
+from litellm.types.utils import ModelResponse
+
+DEGRADED_PAYLOAD = {"trace_id": "degraded-fallback"}
+
+
+@pytest.mark.asyncio
+async def test_streaming_slo_build_failure_falls_back_to_degraded_payload(monkeypatch):
+    """When the SLP build returns None for an assembled streaming response,
+    async_success_handler must rebuild it from an empty response object so
+    SLP-gated loggers still record the request."""
+    logging_obj = Logging(
+        model="claude-3-5-sonnet-20240620",
+        messages=[{"role": "user", "content": "hi"}],
+        stream=True,
+        call_type="anthropic_messages",
+        start_time=datetime.now(),
+        litellm_call_id="test-32019",
+        function_id="test-32019",
+    )
+    build_calls = []
+
+    def fake_build(self, init_response_obj, start_time, end_time):
+        build_calls.append(init_response_obj)
+        if init_response_obj == {}:
+            return DEGRADED_PAYLOAD
+        return None  # simulates get_standard_logging_object_payload swallowing an exception
+
+    monkeypatch.setattr(Logging, "_build_standard_logging_payload", fake_build)
+    monkeypatch.setattr(litellm, "_async_success_callback", [])
+    monkeypatch.setattr(litellm, "success_callback", [])
+
+    result = ModelResponse(model="claude-3-5-sonnet-20240620")
+
+    await logging_obj.async_success_handler(
+        result,
+        start_time=datetime.now(),
+        end_time=datetime.now(),
+        cache_hit=False,
+    )
+
+    assert logging_obj.model_call_details["standard_logging_object"] == DEGRADED_PAYLOAD
+    assert build_calls == [result, {}]
+
+
+@pytest.mark.asyncio
+async def test_streaming_slo_build_success_does_not_rebuild(monkeypatch):
+    """The degraded rebuild must not run when the SLP builds normally."""
+    logging_obj = Logging(
+        model="claude-3-5-sonnet-20240620",
+        messages=[{"role": "user", "content": "hi"}],
+        stream=True,
+        call_type="anthropic_messages",
+        start_time=datetime.now(),
+        litellm_call_id="test-32019-ok",
+        function_id="test-32019-ok",
+    )
+    build_calls = []
+    healthy_payload = {"trace_id": "healthy"}
+
+    def fake_build(self, init_response_obj, start_time, end_time):
+        build_calls.append(init_response_obj)
+        return healthy_payload
+
+    monkeypatch.setattr(Logging, "_build_standard_logging_payload", fake_build)
+    monkeypatch.setattr(litellm, "_async_success_callback", [])
+    monkeypatch.setattr(litellm, "success_callback", [])
+
+    result = ModelResponse(model="claude-3-5-sonnet-20240620")
+
+    await logging_obj.async_success_handler(
+        result,
+        start_time=datetime.now(),
+        end_time=datetime.now(),
+        cache_hit=False,
+    )
+
+    assert logging_obj.model_call_details["standard_logging_object"] == healthy_payload
+    assert len(build_calls) == 1
+
+
+@pytest.mark.asyncio
+async def test_streaming_slo_double_build_failure_is_logged(monkeypatch, caplog):
+    """If the empty-response rebuild also fails, the persistent failure must be
+    surfaced with a second error log instead of silently leaving the payload None."""
+    logging_obj = Logging(
+        model="claude-3-5-sonnet-20240620",
+        messages=[{"role": "user", "content": "hi"}],
+        stream=True,
+        call_type="anthropic_messages",
+        start_time=datetime.now(),
+        litellm_call_id="test-32019-double",
+        function_id="test-32019-double",
+    )
+    build_calls = []
+
+    def fake_build(self, init_response_obj, start_time, end_time) -> None:
+        # Returning None is the point: both the primary build and the
+        # empty-response retry fail.
+        build_calls.append(init_response_obj)
+
+    monkeypatch.setattr(Logging, "_build_standard_logging_payload", fake_build)
+    monkeypatch.setattr(litellm, "_async_success_callback", [])
+    monkeypatch.setattr(litellm, "success_callback", [])
+
+    result = ModelResponse(model="claude-3-5-sonnet-20240620")
+
+    with caplog.at_level(logging.ERROR, logger="LiteLLM"):
+        await logging_obj.async_success_handler(
+            result,
+            start_time=datetime.now(),
+            end_time=datetime.now(),
+            cache_hit=False,
+        )
+
+    assert logging_obj.model_call_details["standard_logging_object"] is None
+    assert build_calls == [result, {}]
+    assert "rebuild failed" in caplog.text
+
+
+@pytest.mark.asyncio
+async def test_streaming_slo_rebuild_raising_does_not_block_callbacks(monkeypatch, caplog):
+    """The rebuild is wrapped for the same reason main wraps the primary build:
+    a payload build must never stop the later callbacks from running. If the
+    empty-response retry raises, the success callback still fires."""
+    logging_obj = Logging(
+        model="claude-3-5-sonnet-20240620",
+        messages=[{"role": "user", "content": "hi"}],
+        stream=True,
+        call_type="anthropic_messages",
+        start_time=datetime.now(),
+        litellm_call_id="test-32019-raise",
+        function_id="test-32019-raise",
+    )
+    build_calls = []
+
+    def fake_build(self, init_response_obj, start_time, end_time):
+        build_calls.append(init_response_obj)
+        if init_response_obj == {}:
+            raise RuntimeError("rebuild blew up")
+        return None
+
+    called_back = []
+
+    async def async_success_callback(kwargs, completion_response, start_time, end_time):
+        called_back.append(completion_response)
+
+    monkeypatch.setattr(Logging, "_build_standard_logging_payload", fake_build)
+    monkeypatch.setattr(litellm, "_async_success_callback", [async_success_callback])
+    monkeypatch.setattr(litellm, "success_callback", [])
+
+    result = ModelResponse(model="claude-3-5-sonnet-20240620")
+
+    with caplog.at_level(logging.ERROR, logger="LiteLLM"):
+        await logging_obj.async_success_handler(
+            result,
+            start_time=datetime.now(),
+            end_time=datetime.now(),
+            cache_hit=False,
+        )
+
+    assert build_calls == [result, {}]
+    assert logging_obj.model_call_details["standard_logging_object"] is None
+    assert "Exception rebuilding the degraded standard logging payload" in caplog.text
+    assert len(called_back) == 1