fix(otel v2): cut image data from the trace root output and stamp the root after every other post-call hook

This commit is contained in:
yucheng 2026-10-27 01:22:58 -07:00
parent 47b8f404aa
commit a6890c91cb
5 changed files with 91 additions and 12 deletions

View file

@ -85,13 +85,14 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
consulted, so a subclass that does not override ``async_pre_call_hook`` inherits nothing.
"""
relays_served_stream: bool = False
reads_served_response: bool = False
"""
Whether this callback's ``async_post_call_streaming_iterator_hook`` reads the stream as the caller is served it.
Whether this callback's post-call hooks read the response as the caller is served it.
The proxy chains the streaming iterator hooks in callback order, so a hook otherwise reads the chunks before a
later hook or post-call policy rewrites them, as a guardrail that masks the reply does. A callback with this set to
True is chained after all of them, and its hook must yield every chunk unchanged.
The proxy runs the post-call hooks in callback order, so a hook otherwise reads the response before a later hook
or post-call policy rewrites or refuses it, as a callback or guardrail that masks the reply does. A callback with
this set to True has its ``async_post_call_success_hook`` run and its ``async_post_call_streaming_iterator_hook``
chained after all of them, and neither may change the response.
"""
def __init__(

View file

@ -41,7 +41,7 @@ class RootIOOpenTelemetryV2(OpenTelemetryV2):
only through a callback whose own class defines that hook.
"""
relays_served_stream: bool = True
reads_served_response: bool = True
async def async_post_call_success_hook(
self,
@ -160,7 +160,7 @@ class OpenInferenceRootIOOpenTelemetryV2(RootIOOpenTelemetryV2):
("openinference.span.kind", "CHAIN"),
("input.value", _without_base64_payloads(rendered_input)),
("input.mime_type", _JSON_MIME_TYPE if rendered_input is not None else None),
("output.value", output),
("output.value", _without_base64_payloads(output)),
("output.mime_type", _JSON_MIME_TYPE),
)
)

View file

@ -4116,7 +4116,7 @@ class ProxyLogging:
############ Handle CustomLogger ###############################
#################################################################
for callback in other_callbacks:
for callback in sorted(other_callbacks, key=lambda callback: callback.reads_served_response):
callback_response: LLMResponseTypes | None = await callback.async_post_call_success_hook(
user_api_key_dict=user_api_key_dict, data=data, response=response
)
@ -4472,7 +4472,7 @@ class ProxyLogging:
pipeline_gated_names: Final = _pipeline_step_guardrail_names(post_call_pipelines)
guarded_layers: Final[list[AsyncGenerator[object, None]]] = [] # mutable-ok: closed on disconnect
stream_rewriters: Final = tuple(
override for override in caps.iterator_overrides if not override[0].relays_served_stream
override for override in caps.iterator_overrides if not override[0].reads_served_response
)
for resolved_callback, kind in stream_rewriters:
if isinstance(resolved_callback, CustomGuardrail):
@ -4538,7 +4538,7 @@ class ProxyLogging:
served_streams: Final = tuple(
accumulate(
(callback for callback, _ in caps.iterator_overrides if callback.relays_served_stream),
(callback for callback, _ in caps.iterator_overrides if callback.reads_served_response),
partial(self._relay_served_stream, user_api_key_dict=user_api_key_dict, request_data=request_data),
initial=current_response,
)

View file

@ -355,6 +355,23 @@ def test_root_input_cuts_an_image_data_uri_as_the_logged_messages_do():
assert root_input != messages
def test_root_output_cuts_an_image_data_uri_as_the_logged_messages_do():
logger, exporter = _logger()
image: Final = {
"type": "image_url",
"image_url": {"url": "data:image/png;base64," + "iVBORw0KGgo" * 100},
"index": 0,
}
reply: Final = ModelResponse(choices=[Choices(message=Message(role="assistant", content="a cat", images=[image]))])
_run_request(logger, CHAT_DATA, "acompletion", reply)
root_output: Final = json.loads(_root_attrs(exporter)["output.value"])
served: Final = [choice.message.model_dump(exclude_none=True) for choice in reply.choices]
assert root_output == truncate_base64_in_messages(served)
assert root_output != served
@pytest.mark.parametrize(
("mappers", "expected"),
[

View file

@ -1,3 +1,5 @@
from typing import Final
import pytest
from fastapi import HTTPException
@ -8,7 +10,7 @@ from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.utils import ProxyLogging
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.utils import CallTypesLiteral
from litellm.types.utils import CallTypesLiteral, Choices, Message, ModelResponse
def test_has_post_call_response_headers_callbacks_ignores_empty_callbacks(
@ -410,7 +412,7 @@ class _MasksReply(CustomLogger):
class _ReadsServedStream(CustomLogger):
relays_served_stream = True
reads_served_response = True
def __init__(self) -> None:
super().__init__()
@ -446,6 +448,65 @@ async def test_post_call_stream_served_stream_relay_reads_what_a_later_hook_serv
assert reader.read == served
class _MasksResponse(CustomLogger):
async def async_post_call_success_hook(
self, data: dict[str, object], user_api_key_dict: UserAPIKeyAuth, response: object
) -> ModelResponse:
return ModelResponse(choices=[Choices(message=Message(role="assistant", content="<PERSON>"))])
class _RefusesResponse(CustomLogger):
async def async_post_call_success_hook(
self, data: dict[str, object], user_api_key_dict: UserAPIKeyAuth, response: object
) -> None:
raise HTTPException(status_code=400, detail="refused")
class _ReadsServedResponse(CustomLogger):
reads_served_response = True
def __init__(self) -> None:
super().__init__()
self.read: Final[list[object]] = [] # mutable-ok: records each response the hook reads
async def async_post_call_success_hook(
self, data: dict[str, object], user_api_key_dict: UserAPIKeyAuth, response: object
) -> None:
self.read.append(response)
async def _post_call(callbacks: list[CustomLogger], monkeypatch: pytest.MonkeyPatch) -> object:
monkeypatch.setattr(litellm, "callbacks", callbacks)
return await ProxyLogging(user_api_key_cache=DualCache()).post_call_success_hook(
data={"model": "gpt-4o-mini", "metadata": {}},
response=ModelResponse(choices=[Choices(message=Message(role="assistant", content="John Smith"))]),
user_api_key_dict=UserAPIKeyAuth(api_key="sk-9876", request_route="/chat/completions"),
)
@pytest.mark.asyncio
async def test_post_call_success_hook_runs_a_served_response_reader_after_a_later_listed_rewrite(
monkeypatch: pytest.MonkeyPatch,
) -> None:
reader: Final = _ReadsServedResponse()
served: Final = await _post_call([reader, _MasksResponse()], monkeypatch)
assert reader.read == [served]
@pytest.mark.asyncio
async def test_post_call_success_hook_skips_a_served_response_reader_when_a_later_listed_hook_refuses(
monkeypatch: pytest.MonkeyPatch,
) -> None:
reader: Final = _ReadsServedResponse()
with pytest.raises(HTTPException):
await _post_call([reader, _RefusesResponse()], monkeypatch)
assert reader.read == []
@pytest.mark.asyncio
async def test_unified_guardrail_iterator_accepts_explicit_guardrail():
"""