mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(otel v2): cut image data from the trace root output and stamp the root after every other post-call hook
This commit is contained in:
parent
47b8f404aa
commit
a6890c91cb
5 changed files with 91 additions and 12 deletions
|
|
@ -85,13 +85,14 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
|
|||
consulted, so a subclass that does not override ``async_pre_call_hook`` inherits nothing.
|
||||
"""
|
||||
|
||||
relays_served_stream: bool = False
|
||||
reads_served_response: bool = False
|
||||
"""
|
||||
Whether this callback's ``async_post_call_streaming_iterator_hook`` reads the stream as the caller is served it.
|
||||
Whether this callback's post-call hooks read the response as the caller is served it.
|
||||
|
||||
The proxy chains the streaming iterator hooks in callback order, so a hook otherwise reads the chunks before a
|
||||
later hook or post-call policy rewrites them, as a guardrail that masks the reply does. A callback with this set to
|
||||
True is chained after all of them, and its hook must yield every chunk unchanged.
|
||||
The proxy runs the post-call hooks in callback order, so a hook otherwise reads the response before a later hook
|
||||
or post-call policy rewrites or refuses it, as a callback or guardrail that masks the reply does. A callback with
|
||||
this set to True has its ``async_post_call_success_hook`` run and its ``async_post_call_streaming_iterator_hook``
|
||||
chained after all of them, and neither may change the response.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ class RootIOOpenTelemetryV2(OpenTelemetryV2):
|
|||
only through a callback whose own class defines that hook.
|
||||
"""
|
||||
|
||||
relays_served_stream: bool = True
|
||||
reads_served_response: bool = True
|
||||
|
||||
async def async_post_call_success_hook(
|
||||
self,
|
||||
|
|
@ -160,7 +160,7 @@ class OpenInferenceRootIOOpenTelemetryV2(RootIOOpenTelemetryV2):
|
|||
("openinference.span.kind", "CHAIN"),
|
||||
("input.value", _without_base64_payloads(rendered_input)),
|
||||
("input.mime_type", _JSON_MIME_TYPE if rendered_input is not None else None),
|
||||
("output.value", output),
|
||||
("output.value", _without_base64_payloads(output)),
|
||||
("output.mime_type", _JSON_MIME_TYPE),
|
||||
)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -4116,7 +4116,7 @@ class ProxyLogging:
|
|||
############ Handle CustomLogger ###############################
|
||||
#################################################################
|
||||
|
||||
for callback in other_callbacks:
|
||||
for callback in sorted(other_callbacks, key=lambda callback: callback.reads_served_response):
|
||||
callback_response: LLMResponseTypes | None = await callback.async_post_call_success_hook(
|
||||
user_api_key_dict=user_api_key_dict, data=data, response=response
|
||||
)
|
||||
|
|
@ -4472,7 +4472,7 @@ class ProxyLogging:
|
|||
pipeline_gated_names: Final = _pipeline_step_guardrail_names(post_call_pipelines)
|
||||
guarded_layers: Final[list[AsyncGenerator[object, None]]] = [] # mutable-ok: closed on disconnect
|
||||
stream_rewriters: Final = tuple(
|
||||
override for override in caps.iterator_overrides if not override[0].relays_served_stream
|
||||
override for override in caps.iterator_overrides if not override[0].reads_served_response
|
||||
)
|
||||
for resolved_callback, kind in stream_rewriters:
|
||||
if isinstance(resolved_callback, CustomGuardrail):
|
||||
|
|
@ -4538,7 +4538,7 @@ class ProxyLogging:
|
|||
|
||||
served_streams: Final = tuple(
|
||||
accumulate(
|
||||
(callback for callback, _ in caps.iterator_overrides if callback.relays_served_stream),
|
||||
(callback for callback, _ in caps.iterator_overrides if callback.reads_served_response),
|
||||
partial(self._relay_served_stream, user_api_key_dict=user_api_key_dict, request_data=request_data),
|
||||
initial=current_response,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -355,6 +355,23 @@ def test_root_input_cuts_an_image_data_uri_as_the_logged_messages_do():
|
|||
assert root_input != messages
|
||||
|
||||
|
||||
def test_root_output_cuts_an_image_data_uri_as_the_logged_messages_do():
|
||||
logger, exporter = _logger()
|
||||
image: Final = {
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "data:image/png;base64," + "iVBORw0KGgo" * 100},
|
||||
"index": 0,
|
||||
}
|
||||
reply: Final = ModelResponse(choices=[Choices(message=Message(role="assistant", content="a cat", images=[image]))])
|
||||
|
||||
_run_request(logger, CHAT_DATA, "acompletion", reply)
|
||||
|
||||
root_output: Final = json.loads(_root_attrs(exporter)["output.value"])
|
||||
served: Final = [choice.message.model_dump(exclude_none=True) for choice in reply.choices]
|
||||
assert root_output == truncate_base64_in_messages(served)
|
||||
assert root_output != served
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mappers", "expected"),
|
||||
[
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
|
|
@ -8,7 +10,7 @@ from litellm.integrations.custom_logger import CustomLogger
|
|||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
from litellm.types.utils import CallTypesLiteral
|
||||
from litellm.types.utils import CallTypesLiteral, Choices, Message, ModelResponse
|
||||
|
||||
|
||||
def test_has_post_call_response_headers_callbacks_ignores_empty_callbacks(
|
||||
|
|
@ -410,7 +412,7 @@ class _MasksReply(CustomLogger):
|
|||
|
||||
|
||||
class _ReadsServedStream(CustomLogger):
|
||||
relays_served_stream = True
|
||||
reads_served_response = True
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
|
|
@ -446,6 +448,65 @@ async def test_post_call_stream_served_stream_relay_reads_what_a_later_hook_serv
|
|||
assert reader.read == served
|
||||
|
||||
|
||||
class _MasksResponse(CustomLogger):
|
||||
async def async_post_call_success_hook(
|
||||
self, data: dict[str, object], user_api_key_dict: UserAPIKeyAuth, response: object
|
||||
) -> ModelResponse:
|
||||
return ModelResponse(choices=[Choices(message=Message(role="assistant", content="<PERSON>"))])
|
||||
|
||||
|
||||
class _RefusesResponse(CustomLogger):
|
||||
async def async_post_call_success_hook(
|
||||
self, data: dict[str, object], user_api_key_dict: UserAPIKeyAuth, response: object
|
||||
) -> None:
|
||||
raise HTTPException(status_code=400, detail="refused")
|
||||
|
||||
|
||||
class _ReadsServedResponse(CustomLogger):
|
||||
reads_served_response = True
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.read: Final[list[object]] = [] # mutable-ok: records each response the hook reads
|
||||
|
||||
async def async_post_call_success_hook(
|
||||
self, data: dict[str, object], user_api_key_dict: UserAPIKeyAuth, response: object
|
||||
) -> None:
|
||||
self.read.append(response)
|
||||
|
||||
|
||||
async def _post_call(callbacks: list[CustomLogger], monkeypatch: pytest.MonkeyPatch) -> object:
|
||||
monkeypatch.setattr(litellm, "callbacks", callbacks)
|
||||
return await ProxyLogging(user_api_key_cache=DualCache()).post_call_success_hook(
|
||||
data={"model": "gpt-4o-mini", "metadata": {}},
|
||||
response=ModelResponse(choices=[Choices(message=Message(role="assistant", content="John Smith"))]),
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-9876", request_route="/chat/completions"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_call_success_hook_runs_a_served_response_reader_after_a_later_listed_rewrite(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
reader: Final = _ReadsServedResponse()
|
||||
|
||||
served: Final = await _post_call([reader, _MasksResponse()], monkeypatch)
|
||||
|
||||
assert reader.read == [served]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_call_success_hook_skips_a_served_response_reader_when_a_later_listed_hook_refuses(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
reader: Final = _ReadsServedResponse()
|
||||
|
||||
with pytest.raises(HTTPException):
|
||||
await _post_call([reader, _RefusesResponse()], monkeypatch)
|
||||
|
||||
assert reader.read == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unified_guardrail_iterator_accepts_explicit_guardrail():
|
||||
"""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue