diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 0fe520103e2..0fe04f98792 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -4443,6 +4443,34 @@ class Logging(LiteLLMLoggingBaseClass): "LiteLLM.LoggingError: [Non-Blocking] Exception building the standard logging payload " "for a streaming response; callbacks still run without it" ) + if self.model_call_details.get("standard_logging_object") is None: + # get_standard_logging_object_payload swallows exceptions and + # returns None, and the except branch above leaves the key + # unset. Loggers gated on standard_logging_object (e.g. s3_v2) + # would then silently drop the request while spend tracking + # still succeeds via the response_cost fallback. Rebuild from an + # empty response object instead, mirroring the failure-path + # fallback. See #32019. + verbose_logger.error( + "standard_logging_object build failed for a streaming response; " + "retrying with an empty response object" + ) + try: + self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload( + {}, # mutable-ok: empty response object, mirrors the failure-path fallback + start_time, # pyright: ignore[reportUnknownArgumentType] # untyped _success_handler_helper_fn return + end_time, # pyright: ignore[reportUnknownArgumentType] # untyped _success_handler_helper_fn return + ) + if self.model_call_details["standard_logging_object"] is None: + verbose_logger.error( + "standard_logging_object rebuild failed for a streaming response; " + "loggers gated on standard_logging_object will skip this request" + ) + except Exception: # noqa: BLE001 # same rule as above: never block later callbacks + verbose_logger.exception( + "LiteLLM.LoggingError: [Non-Blocking] Exception rebuilding the degraded standard " + "logging payload for a streaming response" + ) # print standard logging payload self._emit_stored_standard_logging_payload() diff --git a/tests/unit/litellm_core_utils/test_litellm_logging_slo_fallback.py b/tests/unit/litellm_core_utils/test_litellm_logging_slo_fallback.py new file mode 100644 index 00000000000..4410af7ef87 --- /dev/null +++ b/tests/unit/litellm_core_utils/test_litellm_logging_slo_fallback.py @@ -0,0 +1,181 @@ +""" +Regression tests for https://github.com/BerriAI/litellm/issues/32019. + +Streaming /v1/messages responses were silently dropped by loggers gated on +``standard_logging_object`` (e.g. ``s3_v2``) whenever StandardLoggingPayload +construction failed: ``get_standard_logging_object_payload`` swallows the +exception and returns ``None``, spend tracking still succeeds via the +``response_cost`` fallback, and no error surfaces to the caller. +""" + +import logging +from datetime import datetime + +import pytest + +import litellm +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.types.utils import ModelResponse + +DEGRADED_PAYLOAD = {"trace_id": "degraded-fallback"} + + +@pytest.mark.asyncio +async def test_streaming_slo_build_failure_falls_back_to_degraded_payload(monkeypatch): + """When the SLP build returns None for an assembled streaming response, + async_success_handler must rebuild it from an empty response object so + SLP-gated loggers still record the request.""" + logging_obj = Logging( + model="claude-3-5-sonnet-20240620", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="anthropic_messages", + start_time=datetime.now(), + litellm_call_id="test-32019", + function_id="test-32019", + ) + build_calls = [] + + def fake_build(self, init_response_obj, start_time, end_time): + build_calls.append(init_response_obj) + if init_response_obj == {}: + return DEGRADED_PAYLOAD + return None # simulates get_standard_logging_object_payload swallowing an exception + + monkeypatch.setattr(Logging, "_build_standard_logging_payload", fake_build) + monkeypatch.setattr(litellm, "_async_success_callback", []) + monkeypatch.setattr(litellm, "success_callback", []) + + result = ModelResponse(model="claude-3-5-sonnet-20240620") + + await logging_obj.async_success_handler( + result, + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + ) + + assert logging_obj.model_call_details["standard_logging_object"] == DEGRADED_PAYLOAD + assert build_calls == [result, {}] + + +@pytest.mark.asyncio +async def test_streaming_slo_build_success_does_not_rebuild(monkeypatch): + """The degraded rebuild must not run when the SLP builds normally.""" + logging_obj = Logging( + model="claude-3-5-sonnet-20240620", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="anthropic_messages", + start_time=datetime.now(), + litellm_call_id="test-32019-ok", + function_id="test-32019-ok", + ) + build_calls = [] + healthy_payload = {"trace_id": "healthy"} + + def fake_build(self, init_response_obj, start_time, end_time): + build_calls.append(init_response_obj) + return healthy_payload + + monkeypatch.setattr(Logging, "_build_standard_logging_payload", fake_build) + monkeypatch.setattr(litellm, "_async_success_callback", []) + monkeypatch.setattr(litellm, "success_callback", []) + + result = ModelResponse(model="claude-3-5-sonnet-20240620") + + await logging_obj.async_success_handler( + result, + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + ) + + assert logging_obj.model_call_details["standard_logging_object"] == healthy_payload + assert len(build_calls) == 1 + + +@pytest.mark.asyncio +async def test_streaming_slo_double_build_failure_is_logged(monkeypatch, caplog): + """If the empty-response rebuild also fails, the persistent failure must be + surfaced with a second error log instead of silently leaving the payload None.""" + logging_obj = Logging( + model="claude-3-5-sonnet-20240620", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="anthropic_messages", + start_time=datetime.now(), + litellm_call_id="test-32019-double", + function_id="test-32019-double", + ) + build_calls = [] + + def fake_build(self, init_response_obj, start_time, end_time) -> None: + # Returning None is the point: both the primary build and the + # empty-response retry fail. + build_calls.append(init_response_obj) + + monkeypatch.setattr(Logging, "_build_standard_logging_payload", fake_build) + monkeypatch.setattr(litellm, "_async_success_callback", []) + monkeypatch.setattr(litellm, "success_callback", []) + + result = ModelResponse(model="claude-3-5-sonnet-20240620") + + with caplog.at_level(logging.ERROR, logger="LiteLLM"): + await logging_obj.async_success_handler( + result, + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + ) + + assert logging_obj.model_call_details["standard_logging_object"] is None + assert build_calls == [result, {}] + assert "rebuild failed" in caplog.text + + +@pytest.mark.asyncio +async def test_streaming_slo_rebuild_raising_does_not_block_callbacks(monkeypatch, caplog): + """The rebuild is wrapped for the same reason main wraps the primary build: + a payload build must never stop the later callbacks from running. If the + empty-response retry raises, the success callback still fires.""" + logging_obj = Logging( + model="claude-3-5-sonnet-20240620", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="anthropic_messages", + start_time=datetime.now(), + litellm_call_id="test-32019-raise", + function_id="test-32019-raise", + ) + build_calls = [] + + def fake_build(self, init_response_obj, start_time, end_time): + build_calls.append(init_response_obj) + if init_response_obj == {}: + raise RuntimeError("rebuild blew up") + return None + + called_back = [] + + async def async_success_callback(kwargs, completion_response, start_time, end_time): + called_back.append(completion_response) + + monkeypatch.setattr(Logging, "_build_standard_logging_payload", fake_build) + monkeypatch.setattr(litellm, "_async_success_callback", [async_success_callback]) + monkeypatch.setattr(litellm, "success_callback", []) + + result = ModelResponse(model="claude-3-5-sonnet-20240620") + + with caplog.at_level(logging.ERROR, logger="LiteLLM"): + await logging_obj.async_success_handler( + result, + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + ) + + assert build_calls == [result, {}] + assert logging_obj.model_call_details["standard_logging_object"] is None + assert "Exception rebuilding the degraded standard logging payload" in caplog.text + assert len(called_back) == 1