fix(otel v2): render the trace root input and output off the event loop

Test the root output keys per vocabulary instead of the logger class, and type the root io test helpers
This commit is contained in:
yucheng 2026-10-27 02:07:22 -07:00
parent a6890c91cb
commit 93454cb7e3
2 changed files with 121 additions and 64 deletions

View file

@ -12,6 +12,7 @@ from litellm.integrations.otel.mappers.base import AttributeMap
from litellm.integrations.otel.mappers.utils import drop_none_pairs
from litellm.integrations.otel.model.request_io import request_input, response_output, stream_output
from litellm.integrations.otel.plumbing.context import request_root_span
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.logging_utils import (
truncate_base64_in_messages, # pyright: ignore[reportUnknownVariableType] # the helper types its messages as bare list and dict
@ -49,7 +50,7 @@ class RootIOOpenTelemetryV2(OpenTelemetryV2):
user_api_key_dict: "UserAPIKeyAuth",
response: object,
) -> None:
self._stamp_root_io(data, lambda: response_output(response))
await self._stamp_root_io(data, lambda: response_output(response))
async def _relay_stream(
self,
@ -64,23 +65,29 @@ class RootIOOpenTelemetryV2(OpenTelemetryV2):
async for chunk in response:
relayed.append(chunk)
yield chunk
self._stamp_root_io(request_data, lambda: stream_output(tuple(relayed), request_data))
await self._stamp_root_io(request_data, lambda: stream_output(tuple(relayed), request_data))
def _stamp_root_io(self, data: Mapping[str, object], render_output: Callable[[], str | None]) -> None:
async def _stamp_root_io(self, data: Mapping[str, object], render_output: Callable[[], str | None]) -> None:
if not self._capture_span_content():
return
root: Final = request_root_span()
if root is None or not root.is_recording() or not self._stamps_root(root):
return
try:
output: Final = render_output()
if output is None:
return
rendered_input: Final = request_input(data)
attributes: Final = await asyncify(self._rendered_root_io)(data, render_output)
except Exception: # noqa: BLE001 # telemetry must never fail the request it describes
verbose_logger.debug("otel v2: could not render the request root input or output", exc_info=True)
return
root.set_attributes(self._root_io_attributes(rendered_input, output))
if attributes is not None:
root.set_attributes(attributes)
def _rendered_root_io(
self, data: Mapping[str, object], render_output: Callable[[], str | None]
) -> AttributeMap | None:
"""The root's input and output attributes, rendered off the event loop, or None when the response has no
message to show."""
output: Final = render_output()
return None if output is None else self._root_io_attributes(request_input(data), output)
def _stamps_root(self, root: "Span") -> bool:
"""Whether this logger writes the input and output onto ``root``."""
@ -147,9 +154,9 @@ class OpenInferenceRootIOOpenTelemetryV2(RootIOOpenTelemetryV2):
async for chunk in self._relay_stream(response, request_data):
yield chunk
def _stamp_root_io(self, data: Mapping[str, object], render_output: Callable[[], str | None]) -> None:
async def _stamp_root_io(self, data: Mapping[str, object], render_output: Callable[[], str | None]) -> None:
if not _withholds_messages(self, data):
super()._stamp_root_io(data, render_output)
await super()._stamp_root_io(data, render_output)
def _stamps_root(self, root: "Span") -> bool:
return isinstance(root, ReadableSpan) and root.resource is self.tracer_provider.resource

View file

@ -3,7 +3,8 @@ Arize AX and Phoenix show as the trace's, when this logger's provider started th
import asyncio
import json
from collections.abc import AsyncIterator, Mapping, Sequence
import threading
from collections.abc import AsyncGenerator, AsyncIterator, Iterator, Mapping, Sequence
from datetime import datetime
from types import MappingProxyType
from typing import Final
@ -13,14 +14,16 @@ import pytest
pytest.importorskip("opentelemetry")
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter # noqa: E402
from opentelemetry.trace import Span # noqa: E402
import litellm # noqa: E402
from litellm.caching.dual_cache import DualCache # noqa: E402
from litellm.integrations.custom_logger import CustomLogger # noqa: E402
from litellm.integrations.otel.langfuse_logger import LangfuseContentOpenTelemetryV2 # noqa: E402
from litellm.integrations.otel import root_io_logger # noqa: E402
from litellm.integrations.otel.logger import OpenTelemetryV2, build_otel_v2_logger # noqa: E402
from litellm.integrations.otel.model.config import OpenTelemetryV2Config, is_otel_v2_enabled # noqa: E402
from litellm.integrations.otel.model.destination import OtelDestination # noqa: E402
from litellm.integrations.otel.model.request_io import response_output # noqa: E402
from litellm.integrations.otel.model.spans import LITELLM_PROXY_REQUEST_SPAN_NAME, SpanRole # noqa: E402
from litellm.integrations.otel.plumbing import context as otel_context # noqa: E402
from litellm.integrations.otel.plumbing import providers # noqa: E402
@ -32,6 +35,7 @@ from litellm.proxy._types import UserAPIKeyAuth # noqa: E402
from litellm.proxy.utils import ProxyLogging # noqa: E402
from litellm.types.llms.openai import ResponsesAPIResponse # noqa: E402
from litellm.types.utils import ( # noqa: E402
CallTypesLiteral,
Choices,
Delta,
Embedding,
@ -55,7 +59,7 @@ PONG: Final = ModelResponse(choices=[Choices(message=Message(role="assistant", c
@pytest.fixture(autouse=True)
def _reset_request_root_span():
def _reset_request_root_span() -> Iterator[None]:
otel_context._request_root_span.set(None)
yield
otel_context._request_root_span.set(None)
@ -66,33 +70,43 @@ def _logger(
capture: str = "span_only",
mappers: Sequence[str] = OPENINFERENCE,
settings: Mapping[str, object] = MappingProxyType({}),
):
cfg = OpenTelemetryV2Config(exporter="in_memory", mapper_names=list(mappers), capture_message_content=capture)
exporter = InMemorySpanExporter()
tracer_provider = providers.build_tracer_provider(cfg, exporter=exporter)
) -> tuple[OpenTelemetryV2, InMemorySpanExporter]:
cfg: Final = OpenTelemetryV2Config(
exporter="in_memory", mapper_names=list(mappers), capture_message_content=capture
)
exporter: Final = InMemorySpanExporter()
tracer_provider: Final = providers.build_tracer_provider(cfg, exporter=exporter)
return build_otel_v2_logger(config=cfg, tracer_provider=tracer_provider, settings=settings), exporter
def _start_root(logger):
root = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME, None)
def _start_root(logger: OpenTelemetryV2) -> Span:
root: Final = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME, None)
set_request_root_span(root)
return root
def _root_attrs(exporter) -> dict:
by_name = {span.name: span for span in exporter.get_finished_spans()}
def _root_attrs(exporter: InMemorySpanExporter) -> dict[str, object]:
by_name: Final = {span.name: span for span in exporter.get_finished_spans()}
return dict(by_name[LITELLM_PROXY_REQUEST_SPAN_NAME].attributes or {})
def _run_request(logger, data: dict, call_type: str, response: object, root_owner=None) -> None:
root = _start_root(root_owner or logger)
def _run_request(
logger: OpenTelemetryV2,
data: dict[str, object],
call_type: CallTypesLiteral,
response: object,
root_owner: OpenTelemetryV2 | None = None,
) -> None:
root: Final = _start_root(root_owner or logger)
asyncio.run(logger.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), data, call_type))
asyncio.run(logger.async_post_call_success_hook(data=data, user_api_key_dict=UserAPIKeyAuth(), response=response))
root.end()
async def _relay(logger, chunks: Sequence[object], data: dict) -> list[object]:
async def source() -> AsyncIterator[object]:
async def _relay(
logger: OpenTelemetryV2, chunks: Sequence[ModelResponseStream], data: dict[str, object]
) -> list[ModelResponseStream]:
async def source() -> AsyncIterator[ModelResponseStream]:
for chunk in chunks:
yield chunk
@ -108,7 +122,7 @@ def _chat_chunk(content: str | None, finish_reason: str | None = None) -> ModelR
)
def test_chat_request_stamps_the_root_kind_input_and_output():
def test_chat_request_stamps_the_root_kind_input_and_output() -> None:
logger, exporter = _logger()
_run_request(logger, CHAT_DATA, "acompletion", PONG)
@ -121,7 +135,7 @@ def test_chat_request_stamps_the_root_kind_input_and_output():
assert attrs["output.mime_type"] == "application/json"
def test_responses_request_folds_instructions_into_the_root_input():
def test_responses_request_folds_instructions_into_the_root_input() -> None:
logger, exporter = _logger()
data: Final = {"model": "gpt-5.4-mini", "instructions": "be terse", "input": "ping"}
response: Final = ResponsesAPIResponse(
@ -148,10 +162,10 @@ def test_responses_request_folds_instructions_into_the_root_input():
assert json.loads(attrs["output.value"])[0]["content"][0]["text"] == "pong"
def test_chat_stream_relays_chunks_untouched_and_stamps_the_assembled_output():
def test_chat_stream_relays_chunks_untouched_and_stamps_the_assembled_output() -> None:
logger, exporter = _logger()
chunks: Final = (_chat_chunk("po"), _chat_chunk("ng"), _chat_chunk(None, finish_reason="stop"))
root = _start_root(logger)
root: Final = _start_root(logger)
asyncio.run(logger.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), CHAT_DATA, "acompletion"))
relayed: Final = asyncio.run(_relay(logger, chunks, CHAT_DATA))
@ -163,7 +177,34 @@ def test_chat_stream_relays_chunks_untouched_and_stamps_the_assembled_output():
assert [(turn["role"], turn["content"]) for turn in json.loads(attrs["output.value"])] == [("assistant", "pong")]
def test_a_root_another_callback_started_is_left_alone():
def test_root_output_renders_while_the_event_loop_serves_other_requests(monkeypatch: pytest.MonkeyPatch) -> None:
logger, exporter = _logger()
loop_served_another_request: Final = threading.Event()
def render_after_another_request(response: object) -> str | None:
return response_output(response) if loop_served_another_request.wait(timeout=2) else None
async def another_request() -> None:
loop_served_another_request.set()
async def serve_both() -> None:
await asyncio.gather(
logger.async_post_call_success_hook(data=CHAT_DATA, user_api_key_dict=UserAPIKeyAuth(), response=PONG),
another_request(),
)
monkeypatch.setattr(root_io_logger, "response_output", render_after_another_request)
root: Final = _start_root(logger)
asyncio.run(serve_both())
root.end()
assert [(turn["role"], turn["content"]) for turn in json.loads(_root_attrs(exporter)["output.value"])] == [
("assistant", "pong")
]
def test_a_root_another_callback_started_is_left_alone() -> None:
owner, owner_exporter = _logger(mappers=("genai",))
logger, _ = _logger()
@ -172,9 +213,9 @@ def test_a_root_another_callback_started_is_left_alone():
assert set(ROOT_IO_ATTRS).isdisjoint(_root_attrs(owner_exporter))
def test_root_already_ended_is_left_alone():
def test_root_already_ended_is_left_alone() -> None:
logger, exporter = _logger()
root = _start_root(logger)
root: Final = _start_root(logger)
root.end()
asyncio.run(logger.async_post_call_success_hook(data=CHAT_DATA, user_api_key_dict=UserAPIKeyAuth(), response=PONG))
@ -194,7 +235,9 @@ def test_root_already_ended_is_left_alone():
],
ids=["no-message-body", "unrenderable"],
)
def test_a_response_without_a_message_stamps_nothing(data: dict, call_type: str, response: object):
def test_a_response_without_a_message_stamps_nothing(
data: dict[str, object], call_type: CallTypesLiteral, response: object
) -> None:
logger, exporter = _logger()
_run_request(logger, data, call_type, response)
@ -202,7 +245,7 @@ def test_a_response_without_a_message_stamps_nothing(data: dict, call_type: str,
assert set(ROOT_IO_ATTRS).isdisjoint(_root_attrs(exporter))
def test_root_io_requires_content_capture():
def test_root_io_requires_content_capture() -> None:
logger, exporter = _logger(capture="no_content")
_run_request(logger, CHAT_DATA, "acompletion", PONG)
@ -210,7 +253,7 @@ def test_root_io_requires_content_capture():
assert set(ROOT_IO_ATTRS).isdisjoint(_root_attrs(exporter))
def test_content_on_destination_stamps_the_root_when_global_capture_is_off():
def test_content_on_destination_stamps_the_root_when_global_capture_is_off() -> None:
logger, exporter = _logger(capture="no_content")
destination: Final = OtelDestination(
endpoint="https://team.example.com/v1/traces",
@ -226,8 +269,8 @@ def test_content_on_destination_stamps_the_root_when_global_capture_is_off():
assert json.loads(_root_attrs(exporter)["input.value"]) == [{"role": "user", "content": "ping"}]
def _logging_obj(dynamic_params: dict, *, no_log: bool = False) -> Logging:
logging_obj = Logging(
def _logging_obj(dynamic_params: dict[str, object], *, no_log: bool = False) -> Logging:
logging_obj: Final = Logging(
model="gpt-5.4-mini",
messages=CHAT_DATA["messages"],
stream=False,
@ -241,7 +284,7 @@ def _logging_obj(dynamic_params: dict, *, no_log: bool = False) -> Logging:
return logging_obj
def test_root_io_is_withheld_when_message_logging_is_turned_off(monkeypatch: pytest.MonkeyPatch):
def test_root_io_is_withheld_when_message_logging_is_turned_off(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "turn_off_message_logging", True)
logger, exporter = _logger()
@ -250,11 +293,11 @@ def test_root_io_is_withheld_when_message_logging_is_turned_off(monkeypatch: pyt
assert set(ROOT_IO_ATTRS).isdisjoint(_root_attrs(exporter))
def test_root_io_is_withheld_when_the_request_header_asks_for_redaction():
def test_root_io_is_withheld_when_the_request_header_asks_for_redaction() -> None:
logger, exporter = _logger()
data: Final = {**CHAT_DATA, "metadata": {"headers": {"x-litellm-enable-message-redaction": "true"}}}
chunks: Final = (_chat_chunk("po"), _chat_chunk("ng"), _chat_chunk(None, finish_reason="stop"))
root = _start_root(logger)
root: Final = _start_root(logger)
relayed: Final = asyncio.run(_relay(logger, chunks, data))
root.end()
@ -264,7 +307,7 @@ def test_root_io_is_withheld_when_the_request_header_asks_for_redaction():
@pytest.mark.parametrize(("turn_off", "stamped"), [(True, False), (False, True)], ids=["redacted", "not-redacted"])
def test_root_io_follows_the_request_redaction_setting_on_its_logging_object(turn_off: bool, stamped: bool):
def test_root_io_follows_the_request_redaction_setting_on_its_logging_object(turn_off: bool, stamped: bool) -> None:
logger, exporter = _logger()
data: Final = {**CHAT_DATA, "litellm_logging_obj": _logging_obj({"turn_off_message_logging": turn_off})}
@ -278,7 +321,7 @@ def test_root_io_follows_the_request_redaction_setting_on_its_logging_object(tur
[{"message_logging": False}, {"turn_off_message_logging": True}],
ids=["message-logging-false", "turn-off-message-logging"],
)
def test_root_io_is_withheld_for_a_logger_with_message_logging_off(settings: dict):
def test_root_io_is_withheld_for_a_logger_with_message_logging_off(settings: dict[str, object]) -> None:
logger, exporter = _logger(settings=settings)
_run_request(logger, CHAT_DATA, "acompletion", PONG)
@ -286,7 +329,7 @@ def test_root_io_is_withheld_for_a_logger_with_message_logging_off(settings: dic
assert set(ROOT_IO_ATTRS).isdisjoint(_root_attrs(exporter))
def _no_log_request(on_logging_object: bool) -> dict:
def _no_log_request(on_logging_object: bool) -> dict[str, object]:
return (
{**CHAT_DATA, "litellm_logging_obj": _logging_obj({}, no_log=True)}
if on_logging_object
@ -295,7 +338,7 @@ def _no_log_request(on_logging_object: bool) -> dict:
@pytest.mark.parametrize("on_logging_object", [False, True], ids=["request-body", "logging-object"])
def test_root_io_is_withheld_for_a_no_log_request(on_logging_object: bool):
def test_root_io_is_withheld_for_a_no_log_request(on_logging_object: bool) -> None:
logger, exporter = _logger()
_run_request(logger, _no_log_request(on_logging_object), "acompletion", PONG)
@ -306,7 +349,7 @@ def test_root_io_is_withheld_for_a_no_log_request(on_logging_object: bool):
@pytest.mark.parametrize("on_logging_object", [False, True], ids=["request-body", "logging-object"])
def test_root_io_is_stamped_for_a_no_log_request_when_the_no_log_param_is_disabled(
on_logging_object: bool, monkeypatch: pytest.MonkeyPatch
):
) -> None:
monkeypatch.setattr(litellm, "global_disable_no_log_param", True)
logger, exporter = _logger()
@ -327,7 +370,7 @@ class _SkipsLoggerInit(CustomLogger):
)
def test_root_io_follows_another_callback_with_message_logging_off(
sibling: CustomLogger, stamped: bool, monkeypatch: pytest.MonkeyPatch
):
) -> None:
monkeypatch.setattr(litellm, "callbacks", [sibling])
logger, exporter = _logger()
@ -336,7 +379,7 @@ def test_root_io_follows_another_callback_with_message_logging_off(
assert ("input.value" in _root_attrs(exporter)) is stamped
def test_root_input_cuts_an_image_data_uri_as_the_logged_messages_do():
def test_root_input_cuts_an_image_data_uri_as_the_logged_messages_do() -> None:
logger, exporter = _logger()
messages: Final = [
{
@ -355,7 +398,7 @@ def test_root_input_cuts_an_image_data_uri_as_the_logged_messages_do():
assert root_input != messages
def test_root_output_cuts_an_image_data_uri_as_the_logged_messages_do():
def test_root_output_cuts_an_image_data_uri_as_the_logged_messages_do() -> None:
logger, exporter = _logger()
image: Final = {
"type": "image_url",
@ -373,19 +416,21 @@ def test_root_output_cuts_an_image_data_uri_as_the_logged_messages_do():
@pytest.mark.parametrize(
("mappers", "expected"),
("mappers", "output_keys"),
[
(OPENINFERENCE, OpenInferenceRootIOOpenTelemetryV2),
(("genai", "openinference", "weave"), OpenTelemetryV2),
(("genai", "langfuse"), LangfuseContentOpenTelemetryV2),
(("genai",), OpenTelemetryV2),
(OPENINFERENCE, {"output.value"}),
(("genai", "openinference", "weave"), set()),
(("genai", "langfuse"), {"langfuse.observation.output"}),
(("genai",), set()),
],
ids=["arize-phoenix", "weave", "langfuse", "genai"],
)
def test_logger_class_follows_the_configured_vocabularies(mappers: tuple[str, ...], expected: type) -> None:
logger, _ = _logger(mappers=mappers)
def test_root_output_follows_the_configured_vocabularies(mappers: tuple[str, ...], output_keys: set[str]) -> None:
logger, exporter = _logger(mappers=mappers)
assert type(logger) is expected
_run_request(logger, CHAT_DATA, "acompletion", PONG)
assert {"output.value", "langfuse.observation.output"} & set(_root_attrs(exporter)) == output_keys
@pytest.mark.parametrize(
@ -412,13 +457,18 @@ def test_openinference_logger_keeps_the_pre_call_hook_that_seeds_metadata_baggag
class _MasksReply(CustomLogger):
async def async_post_call_streaming_iterator_hook(self, user_api_key_dict, response, request_data):
async def async_post_call_streaming_iterator_hook(
self,
user_api_key_dict: UserAPIKeyAuth,
response: AsyncIterator[ModelResponseStream],
request_data: dict[str, object],
) -> AsyncGenerator[ModelResponseStream, None]:
async for chunk in response:
choice = chunk.choices[0]
choice: Final = chunk.choices[0]
yield _chat_chunk("<PERSON>" if choice.delta.content else None, finish_reason=choice.finish_reason)
async def _serve(chunks: Sequence[ModelResponseStream], data: dict) -> list[object]:
async def _serve(chunks: Sequence[ModelResponseStream], data: dict[str, object]) -> list[ModelResponseStream]:
async def source() -> AsyncIterator[ModelResponseStream]:
for chunk in chunks:
yield chunk
@ -443,7 +493,7 @@ def test_stream_root_output_is_what_a_later_masking_hook_served(
) -> None:
logger, exporter = _logger(mappers=mappers)
monkeypatch.setattr(litellm, "callbacks", [logger, _MasksReply()])
root = _start_root(logger)
root: Final = _start_root(logger)
served: Final = asyncio.run(
_serve((_chat_chunk("John Smith"), _chat_chunk(None, finish_reason="stop")), {**CHAT_DATA})
@ -471,15 +521,15 @@ def test_arize_and_phoenix_presets_build_a_logger_that_stamps_the_root(
monkeypatch.setenv(name, value)
is_otel_v2_enabled.cache_clear()
loggers: list = []
loggers: Final[list[CustomLogger]] = []
try:
built = _maybe_construct_otel_v2(callback, loggers)
built: Final = _maybe_construct_otel_v2(callback, loggers)
assert isinstance(built, OpenInferenceRootIOOpenTelemetryV2)
root = _start_root(built)
root: Final = _start_root(built)
asyncio.run(
built.async_post_call_success_hook(data=CHAT_DATA, user_api_key_dict=UserAPIKeyAuth(), response=PONG)
)
attrs = dict(root.attributes or {})
attrs: Final = dict(root.attributes or {})
assert json.loads(attrs["input.value"]) == [{"role": "user", "content": "ping"}]
assert attrs["openinference.span.kind"] == "CHAIN"
finally: