From a6890c91cb5ba6283bc5c183f80f59ff5ae26e69 Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 27 Oct 2026 01:22:58 -0700 Subject: [PATCH] fix(otel v2): cut image data from the trace root output and stamp the root after every other post-call hook --- litellm/integrations/custom_logger.py | 11 ++-- litellm/integrations/otel/root_io_logger.py | 4 +- litellm/proxy/utils.py | 6 +- .../integrations/otel/test_root_io_logger.py | 17 +++++ .../test_proxy_logging_hook_detection.py | 65 ++++++++++++++++++- 5 files changed, 91 insertions(+), 12 deletions(-) diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 06d299775a4..e7c5f7b3f35 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -85,13 +85,14 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac consulted, so a subclass that does not override ``async_pre_call_hook`` inherits nothing. """ - relays_served_stream: bool = False + reads_served_response: bool = False """ - Whether this callback's ``async_post_call_streaming_iterator_hook`` reads the stream as the caller is served it. + Whether this callback's post-call hooks read the response as the caller is served it. - The proxy chains the streaming iterator hooks in callback order, so a hook otherwise reads the chunks before a - later hook or post-call policy rewrites them, as a guardrail that masks the reply does. A callback with this set to - True is chained after all of them, and its hook must yield every chunk unchanged. + The proxy runs the post-call hooks in callback order, so a hook otherwise reads the response before a later hook + or post-call policy rewrites or refuses it, as a callback or guardrail that masks the reply does. A callback with + this set to True has its ``async_post_call_success_hook`` run and its ``async_post_call_streaming_iterator_hook`` + chained after all of them, and neither may change the response. """ def __init__( diff --git a/litellm/integrations/otel/root_io_logger.py b/litellm/integrations/otel/root_io_logger.py index c8abe1e1a41..f3c6c2a9bd7 100644 --- a/litellm/integrations/otel/root_io_logger.py +++ b/litellm/integrations/otel/root_io_logger.py @@ -41,7 +41,7 @@ class RootIOOpenTelemetryV2(OpenTelemetryV2): only through a callback whose own class defines that hook. """ - relays_served_stream: bool = True + reads_served_response: bool = True async def async_post_call_success_hook( self, @@ -160,7 +160,7 @@ class OpenInferenceRootIOOpenTelemetryV2(RootIOOpenTelemetryV2): ("openinference.span.kind", "CHAIN"), ("input.value", _without_base64_payloads(rendered_input)), ("input.mime_type", _JSON_MIME_TYPE if rendered_input is not None else None), - ("output.value", output), + ("output.value", _without_base64_payloads(output)), ("output.mime_type", _JSON_MIME_TYPE), ) ) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 163d9b467cf..c652d6eb108 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -4116,7 +4116,7 @@ class ProxyLogging: ############ Handle CustomLogger ############################### ################################################################# - for callback in other_callbacks: + for callback in sorted(other_callbacks, key=lambda callback: callback.reads_served_response): callback_response: LLMResponseTypes | None = await callback.async_post_call_success_hook( user_api_key_dict=user_api_key_dict, data=data, response=response ) @@ -4472,7 +4472,7 @@ class ProxyLogging: pipeline_gated_names: Final = _pipeline_step_guardrail_names(post_call_pipelines) guarded_layers: Final[list[AsyncGenerator[object, None]]] = [] # mutable-ok: closed on disconnect stream_rewriters: Final = tuple( - override for override in caps.iterator_overrides if not override[0].relays_served_stream + override for override in caps.iterator_overrides if not override[0].reads_served_response ) for resolved_callback, kind in stream_rewriters: if isinstance(resolved_callback, CustomGuardrail): @@ -4538,7 +4538,7 @@ class ProxyLogging: served_streams: Final = tuple( accumulate( - (callback for callback, _ in caps.iterator_overrides if callback.relays_served_stream), + (callback for callback, _ in caps.iterator_overrides if callback.reads_served_response), partial(self._relay_served_stream, user_api_key_dict=user_api_key_dict, request_data=request_data), initial=current_response, ) diff --git a/tests/unit/integrations/otel/test_root_io_logger.py b/tests/unit/integrations/otel/test_root_io_logger.py index d0a4f6daec6..c742eee72be 100644 --- a/tests/unit/integrations/otel/test_root_io_logger.py +++ b/tests/unit/integrations/otel/test_root_io_logger.py @@ -355,6 +355,23 @@ def test_root_input_cuts_an_image_data_uri_as_the_logged_messages_do(): assert root_input != messages +def test_root_output_cuts_an_image_data_uri_as_the_logged_messages_do(): + logger, exporter = _logger() + image: Final = { + "type": "image_url", + "image_url": {"url": "data:image/png;base64," + "iVBORw0KGgo" * 100}, + "index": 0, + } + reply: Final = ModelResponse(choices=[Choices(message=Message(role="assistant", content="a cat", images=[image]))]) + + _run_request(logger, CHAT_DATA, "acompletion", reply) + + root_output: Final = json.loads(_root_attrs(exporter)["output.value"]) + served: Final = [choice.message.model_dump(exclude_none=True) for choice in reply.choices] + assert root_output == truncate_base64_in_messages(served) + assert root_output != served + + @pytest.mark.parametrize( ("mappers", "expected"), [ diff --git a/tests/unit/proxy/test_proxy_logging_hook_detection.py b/tests/unit/proxy/test_proxy_logging_hook_detection.py index 32e0d41825d..9ee96833d44 100644 --- a/tests/unit/proxy/test_proxy_logging_hook_detection.py +++ b/tests/unit/proxy/test_proxy_logging_hook_detection.py @@ -1,3 +1,5 @@ +from typing import Final + import pytest from fastapi import HTTPException @@ -8,7 +10,7 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.utils import ProxyLogging from litellm.types.guardrails import GuardrailEventHooks -from litellm.types.utils import CallTypesLiteral +from litellm.types.utils import CallTypesLiteral, Choices, Message, ModelResponse def test_has_post_call_response_headers_callbacks_ignores_empty_callbacks( @@ -410,7 +412,7 @@ class _MasksReply(CustomLogger): class _ReadsServedStream(CustomLogger): - relays_served_stream = True + reads_served_response = True def __init__(self) -> None: super().__init__() @@ -446,6 +448,65 @@ async def test_post_call_stream_served_stream_relay_reads_what_a_later_hook_serv assert reader.read == served +class _MasksResponse(CustomLogger): + async def async_post_call_success_hook( + self, data: dict[str, object], user_api_key_dict: UserAPIKeyAuth, response: object + ) -> ModelResponse: + return ModelResponse(choices=[Choices(message=Message(role="assistant", content=""))]) + + +class _RefusesResponse(CustomLogger): + async def async_post_call_success_hook( + self, data: dict[str, object], user_api_key_dict: UserAPIKeyAuth, response: object + ) -> None: + raise HTTPException(status_code=400, detail="refused") + + +class _ReadsServedResponse(CustomLogger): + reads_served_response = True + + def __init__(self) -> None: + super().__init__() + self.read: Final[list[object]] = [] # mutable-ok: records each response the hook reads + + async def async_post_call_success_hook( + self, data: dict[str, object], user_api_key_dict: UserAPIKeyAuth, response: object + ) -> None: + self.read.append(response) + + +async def _post_call(callbacks: list[CustomLogger], monkeypatch: pytest.MonkeyPatch) -> object: + monkeypatch.setattr(litellm, "callbacks", callbacks) + return await ProxyLogging(user_api_key_cache=DualCache()).post_call_success_hook( + data={"model": "gpt-4o-mini", "metadata": {}}, + response=ModelResponse(choices=[Choices(message=Message(role="assistant", content="John Smith"))]), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-9876", request_route="/chat/completions"), + ) + + +@pytest.mark.asyncio +async def test_post_call_success_hook_runs_a_served_response_reader_after_a_later_listed_rewrite( + monkeypatch: pytest.MonkeyPatch, +) -> None: + reader: Final = _ReadsServedResponse() + + served: Final = await _post_call([reader, _MasksResponse()], monkeypatch) + + assert reader.read == [served] + + +@pytest.mark.asyncio +async def test_post_call_success_hook_skips_a_served_response_reader_when_a_later_listed_hook_refuses( + monkeypatch: pytest.MonkeyPatch, +) -> None: + reader: Final = _ReadsServedResponse() + + with pytest.raises(HTTPException): + await _post_call([reader, _RefusesResponse()], monkeypatch) + + assert reader.read == [] + + @pytest.mark.asyncio async def test_unified_guardrail_iterator_accepts_explicit_guardrail(): """