diff --git a/litellm/integrations/otel/root_io_logger.py b/litellm/integrations/otel/root_io_logger.py index f3c6c2a9bd7..7069b113d09 100644 --- a/litellm/integrations/otel/root_io_logger.py +++ b/litellm/integrations/otel/root_io_logger.py @@ -12,6 +12,7 @@ from litellm.integrations.otel.mappers.base import AttributeMap from litellm.integrations.otel.mappers.utils import drop_none_pairs from litellm.integrations.otel.model.request_io import request_input, response_output, stream_output from litellm.integrations.otel.plumbing.context import request_root_span +from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.logging_utils import ( truncate_base64_in_messages, # pyright: ignore[reportUnknownVariableType] # the helper types its messages as bare list and dict @@ -49,7 +50,7 @@ class RootIOOpenTelemetryV2(OpenTelemetryV2): user_api_key_dict: "UserAPIKeyAuth", response: object, ) -> None: - self._stamp_root_io(data, lambda: response_output(response)) + await self._stamp_root_io(data, lambda: response_output(response)) async def _relay_stream( self, @@ -64,23 +65,29 @@ class RootIOOpenTelemetryV2(OpenTelemetryV2): async for chunk in response: relayed.append(chunk) yield chunk - self._stamp_root_io(request_data, lambda: stream_output(tuple(relayed), request_data)) + await self._stamp_root_io(request_data, lambda: stream_output(tuple(relayed), request_data)) - def _stamp_root_io(self, data: Mapping[str, object], render_output: Callable[[], str | None]) -> None: + async def _stamp_root_io(self, data: Mapping[str, object], render_output: Callable[[], str | None]) -> None: if not self._capture_span_content(): return root: Final = request_root_span() if root is None or not root.is_recording() or not self._stamps_root(root): return try: - output: Final = render_output() - if output is None: - return - rendered_input: Final = request_input(data) + attributes: Final = await asyncify(self._rendered_root_io)(data, render_output) except Exception: # noqa: BLE001 # telemetry must never fail the request it describes verbose_logger.debug("otel v2: could not render the request root input or output", exc_info=True) return - root.set_attributes(self._root_io_attributes(rendered_input, output)) + if attributes is not None: + root.set_attributes(attributes) + + def _rendered_root_io( + self, data: Mapping[str, object], render_output: Callable[[], str | None] + ) -> AttributeMap | None: + """The root's input and output attributes, rendered off the event loop, or None when the response has no + message to show.""" + output: Final = render_output() + return None if output is None else self._root_io_attributes(request_input(data), output) def _stamps_root(self, root: "Span") -> bool: """Whether this logger writes the input and output onto ``root``.""" @@ -147,9 +154,9 @@ class OpenInferenceRootIOOpenTelemetryV2(RootIOOpenTelemetryV2): async for chunk in self._relay_stream(response, request_data): yield chunk - def _stamp_root_io(self, data: Mapping[str, object], render_output: Callable[[], str | None]) -> None: + async def _stamp_root_io(self, data: Mapping[str, object], render_output: Callable[[], str | None]) -> None: if not _withholds_messages(self, data): - super()._stamp_root_io(data, render_output) + await super()._stamp_root_io(data, render_output) def _stamps_root(self, root: "Span") -> bool: return isinstance(root, ReadableSpan) and root.resource is self.tracer_provider.resource diff --git a/tests/unit/integrations/otel/test_root_io_logger.py b/tests/unit/integrations/otel/test_root_io_logger.py index c742eee72be..c13f2598955 100644 --- a/tests/unit/integrations/otel/test_root_io_logger.py +++ b/tests/unit/integrations/otel/test_root_io_logger.py @@ -3,7 +3,8 @@ Arize AX and Phoenix show as the trace's, when this logger's provider started th import asyncio import json -from collections.abc import AsyncIterator, Mapping, Sequence +import threading +from collections.abc import AsyncGenerator, AsyncIterator, Iterator, Mapping, Sequence from datetime import datetime from types import MappingProxyType from typing import Final @@ -13,14 +14,16 @@ import pytest pytest.importorskip("opentelemetry") from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter # noqa: E402 +from opentelemetry.trace import Span # noqa: E402 import litellm # noqa: E402 from litellm.caching.dual_cache import DualCache # noqa: E402 from litellm.integrations.custom_logger import CustomLogger # noqa: E402 -from litellm.integrations.otel.langfuse_logger import LangfuseContentOpenTelemetryV2 # noqa: E402 +from litellm.integrations.otel import root_io_logger # noqa: E402 from litellm.integrations.otel.logger import OpenTelemetryV2, build_otel_v2_logger # noqa: E402 from litellm.integrations.otel.model.config import OpenTelemetryV2Config, is_otel_v2_enabled # noqa: E402 from litellm.integrations.otel.model.destination import OtelDestination # noqa: E402 +from litellm.integrations.otel.model.request_io import response_output # noqa: E402 from litellm.integrations.otel.model.spans import LITELLM_PROXY_REQUEST_SPAN_NAME, SpanRole # noqa: E402 from litellm.integrations.otel.plumbing import context as otel_context # noqa: E402 from litellm.integrations.otel.plumbing import providers # noqa: E402 @@ -32,6 +35,7 @@ from litellm.proxy._types import UserAPIKeyAuth # noqa: E402 from litellm.proxy.utils import ProxyLogging # noqa: E402 from litellm.types.llms.openai import ResponsesAPIResponse # noqa: E402 from litellm.types.utils import ( # noqa: E402 + CallTypesLiteral, Choices, Delta, Embedding, @@ -55,7 +59,7 @@ PONG: Final = ModelResponse(choices=[Choices(message=Message(role="assistant", c @pytest.fixture(autouse=True) -def _reset_request_root_span(): +def _reset_request_root_span() -> Iterator[None]: otel_context._request_root_span.set(None) yield otel_context._request_root_span.set(None) @@ -66,33 +70,43 @@ def _logger( capture: str = "span_only", mappers: Sequence[str] = OPENINFERENCE, settings: Mapping[str, object] = MappingProxyType({}), -): - cfg = OpenTelemetryV2Config(exporter="in_memory", mapper_names=list(mappers), capture_message_content=capture) - exporter = InMemorySpanExporter() - tracer_provider = providers.build_tracer_provider(cfg, exporter=exporter) +) -> tuple[OpenTelemetryV2, InMemorySpanExporter]: + cfg: Final = OpenTelemetryV2Config( + exporter="in_memory", mapper_names=list(mappers), capture_message_content=capture + ) + exporter: Final = InMemorySpanExporter() + tracer_provider: Final = providers.build_tracer_provider(cfg, exporter=exporter) return build_otel_v2_logger(config=cfg, tracer_provider=tracer_provider, settings=settings), exporter -def _start_root(logger): - root = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME, None) +def _start_root(logger: OpenTelemetryV2) -> Span: + root: Final = logger._emitter.start_span(SpanRole.PROXY_REQUEST, LITELLM_PROXY_REQUEST_SPAN_NAME, None) set_request_root_span(root) return root -def _root_attrs(exporter) -> dict: - by_name = {span.name: span for span in exporter.get_finished_spans()} +def _root_attrs(exporter: InMemorySpanExporter) -> dict[str, object]: + by_name: Final = {span.name: span for span in exporter.get_finished_spans()} return dict(by_name[LITELLM_PROXY_REQUEST_SPAN_NAME].attributes or {}) -def _run_request(logger, data: dict, call_type: str, response: object, root_owner=None) -> None: - root = _start_root(root_owner or logger) +def _run_request( + logger: OpenTelemetryV2, + data: dict[str, object], + call_type: CallTypesLiteral, + response: object, + root_owner: OpenTelemetryV2 | None = None, +) -> None: + root: Final = _start_root(root_owner or logger) asyncio.run(logger.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), data, call_type)) asyncio.run(logger.async_post_call_success_hook(data=data, user_api_key_dict=UserAPIKeyAuth(), response=response)) root.end() -async def _relay(logger, chunks: Sequence[object], data: dict) -> list[object]: - async def source() -> AsyncIterator[object]: +async def _relay( + logger: OpenTelemetryV2, chunks: Sequence[ModelResponseStream], data: dict[str, object] +) -> list[ModelResponseStream]: + async def source() -> AsyncIterator[ModelResponseStream]: for chunk in chunks: yield chunk @@ -108,7 +122,7 @@ def _chat_chunk(content: str | None, finish_reason: str | None = None) -> ModelR ) -def test_chat_request_stamps_the_root_kind_input_and_output(): +def test_chat_request_stamps_the_root_kind_input_and_output() -> None: logger, exporter = _logger() _run_request(logger, CHAT_DATA, "acompletion", PONG) @@ -121,7 +135,7 @@ def test_chat_request_stamps_the_root_kind_input_and_output(): assert attrs["output.mime_type"] == "application/json" -def test_responses_request_folds_instructions_into_the_root_input(): +def test_responses_request_folds_instructions_into_the_root_input() -> None: logger, exporter = _logger() data: Final = {"model": "gpt-5.4-mini", "instructions": "be terse", "input": "ping"} response: Final = ResponsesAPIResponse( @@ -148,10 +162,10 @@ def test_responses_request_folds_instructions_into_the_root_input(): assert json.loads(attrs["output.value"])[0]["content"][0]["text"] == "pong" -def test_chat_stream_relays_chunks_untouched_and_stamps_the_assembled_output(): +def test_chat_stream_relays_chunks_untouched_and_stamps_the_assembled_output() -> None: logger, exporter = _logger() chunks: Final = (_chat_chunk("po"), _chat_chunk("ng"), _chat_chunk(None, finish_reason="stop")) - root = _start_root(logger) + root: Final = _start_root(logger) asyncio.run(logger.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), CHAT_DATA, "acompletion")) relayed: Final = asyncio.run(_relay(logger, chunks, CHAT_DATA)) @@ -163,7 +177,34 @@ def test_chat_stream_relays_chunks_untouched_and_stamps_the_assembled_output(): assert [(turn["role"], turn["content"]) for turn in json.loads(attrs["output.value"])] == [("assistant", "pong")] -def test_a_root_another_callback_started_is_left_alone(): +def test_root_output_renders_while_the_event_loop_serves_other_requests(monkeypatch: pytest.MonkeyPatch) -> None: + logger, exporter = _logger() + loop_served_another_request: Final = threading.Event() + + def render_after_another_request(response: object) -> str | None: + return response_output(response) if loop_served_another_request.wait(timeout=2) else None + + async def another_request() -> None: + loop_served_another_request.set() + + async def serve_both() -> None: + await asyncio.gather( + logger.async_post_call_success_hook(data=CHAT_DATA, user_api_key_dict=UserAPIKeyAuth(), response=PONG), + another_request(), + ) + + monkeypatch.setattr(root_io_logger, "response_output", render_after_another_request) + root: Final = _start_root(logger) + + asyncio.run(serve_both()) + root.end() + + assert [(turn["role"], turn["content"]) for turn in json.loads(_root_attrs(exporter)["output.value"])] == [ + ("assistant", "pong") + ] + + +def test_a_root_another_callback_started_is_left_alone() -> None: owner, owner_exporter = _logger(mappers=("genai",)) logger, _ = _logger() @@ -172,9 +213,9 @@ def test_a_root_another_callback_started_is_left_alone(): assert set(ROOT_IO_ATTRS).isdisjoint(_root_attrs(owner_exporter)) -def test_root_already_ended_is_left_alone(): +def test_root_already_ended_is_left_alone() -> None: logger, exporter = _logger() - root = _start_root(logger) + root: Final = _start_root(logger) root.end() asyncio.run(logger.async_post_call_success_hook(data=CHAT_DATA, user_api_key_dict=UserAPIKeyAuth(), response=PONG)) @@ -194,7 +235,9 @@ def test_root_already_ended_is_left_alone(): ], ids=["no-message-body", "unrenderable"], ) -def test_a_response_without_a_message_stamps_nothing(data: dict, call_type: str, response: object): +def test_a_response_without_a_message_stamps_nothing( + data: dict[str, object], call_type: CallTypesLiteral, response: object +) -> None: logger, exporter = _logger() _run_request(logger, data, call_type, response) @@ -202,7 +245,7 @@ def test_a_response_without_a_message_stamps_nothing(data: dict, call_type: str, assert set(ROOT_IO_ATTRS).isdisjoint(_root_attrs(exporter)) -def test_root_io_requires_content_capture(): +def test_root_io_requires_content_capture() -> None: logger, exporter = _logger(capture="no_content") _run_request(logger, CHAT_DATA, "acompletion", PONG) @@ -210,7 +253,7 @@ def test_root_io_requires_content_capture(): assert set(ROOT_IO_ATTRS).isdisjoint(_root_attrs(exporter)) -def test_content_on_destination_stamps_the_root_when_global_capture_is_off(): +def test_content_on_destination_stamps_the_root_when_global_capture_is_off() -> None: logger, exporter = _logger(capture="no_content") destination: Final = OtelDestination( endpoint="https://team.example.com/v1/traces", @@ -226,8 +269,8 @@ def test_content_on_destination_stamps_the_root_when_global_capture_is_off(): assert json.loads(_root_attrs(exporter)["input.value"]) == [{"role": "user", "content": "ping"}] -def _logging_obj(dynamic_params: dict, *, no_log: bool = False) -> Logging: - logging_obj = Logging( +def _logging_obj(dynamic_params: dict[str, object], *, no_log: bool = False) -> Logging: + logging_obj: Final = Logging( model="gpt-5.4-mini", messages=CHAT_DATA["messages"], stream=False, @@ -241,7 +284,7 @@ def _logging_obj(dynamic_params: dict, *, no_log: bool = False) -> Logging: return logging_obj -def test_root_io_is_withheld_when_message_logging_is_turned_off(monkeypatch: pytest.MonkeyPatch): +def test_root_io_is_withheld_when_message_logging_is_turned_off(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "turn_off_message_logging", True) logger, exporter = _logger() @@ -250,11 +293,11 @@ def test_root_io_is_withheld_when_message_logging_is_turned_off(monkeypatch: pyt assert set(ROOT_IO_ATTRS).isdisjoint(_root_attrs(exporter)) -def test_root_io_is_withheld_when_the_request_header_asks_for_redaction(): +def test_root_io_is_withheld_when_the_request_header_asks_for_redaction() -> None: logger, exporter = _logger() data: Final = {**CHAT_DATA, "metadata": {"headers": {"x-litellm-enable-message-redaction": "true"}}} chunks: Final = (_chat_chunk("po"), _chat_chunk("ng"), _chat_chunk(None, finish_reason="stop")) - root = _start_root(logger) + root: Final = _start_root(logger) relayed: Final = asyncio.run(_relay(logger, chunks, data)) root.end() @@ -264,7 +307,7 @@ def test_root_io_is_withheld_when_the_request_header_asks_for_redaction(): @pytest.mark.parametrize(("turn_off", "stamped"), [(True, False), (False, True)], ids=["redacted", "not-redacted"]) -def test_root_io_follows_the_request_redaction_setting_on_its_logging_object(turn_off: bool, stamped: bool): +def test_root_io_follows_the_request_redaction_setting_on_its_logging_object(turn_off: bool, stamped: bool) -> None: logger, exporter = _logger() data: Final = {**CHAT_DATA, "litellm_logging_obj": _logging_obj({"turn_off_message_logging": turn_off})} @@ -278,7 +321,7 @@ def test_root_io_follows_the_request_redaction_setting_on_its_logging_object(tur [{"message_logging": False}, {"turn_off_message_logging": True}], ids=["message-logging-false", "turn-off-message-logging"], ) -def test_root_io_is_withheld_for_a_logger_with_message_logging_off(settings: dict): +def test_root_io_is_withheld_for_a_logger_with_message_logging_off(settings: dict[str, object]) -> None: logger, exporter = _logger(settings=settings) _run_request(logger, CHAT_DATA, "acompletion", PONG) @@ -286,7 +329,7 @@ def test_root_io_is_withheld_for_a_logger_with_message_logging_off(settings: dic assert set(ROOT_IO_ATTRS).isdisjoint(_root_attrs(exporter)) -def _no_log_request(on_logging_object: bool) -> dict: +def _no_log_request(on_logging_object: bool) -> dict[str, object]: return ( {**CHAT_DATA, "litellm_logging_obj": _logging_obj({}, no_log=True)} if on_logging_object @@ -295,7 +338,7 @@ def _no_log_request(on_logging_object: bool) -> dict: @pytest.mark.parametrize("on_logging_object", [False, True], ids=["request-body", "logging-object"]) -def test_root_io_is_withheld_for_a_no_log_request(on_logging_object: bool): +def test_root_io_is_withheld_for_a_no_log_request(on_logging_object: bool) -> None: logger, exporter = _logger() _run_request(logger, _no_log_request(on_logging_object), "acompletion", PONG) @@ -306,7 +349,7 @@ def test_root_io_is_withheld_for_a_no_log_request(on_logging_object: bool): @pytest.mark.parametrize("on_logging_object", [False, True], ids=["request-body", "logging-object"]) def test_root_io_is_stamped_for_a_no_log_request_when_the_no_log_param_is_disabled( on_logging_object: bool, monkeypatch: pytest.MonkeyPatch -): +) -> None: monkeypatch.setattr(litellm, "global_disable_no_log_param", True) logger, exporter = _logger() @@ -327,7 +370,7 @@ class _SkipsLoggerInit(CustomLogger): ) def test_root_io_follows_another_callback_with_message_logging_off( sibling: CustomLogger, stamped: bool, monkeypatch: pytest.MonkeyPatch -): +) -> None: monkeypatch.setattr(litellm, "callbacks", [sibling]) logger, exporter = _logger() @@ -336,7 +379,7 @@ def test_root_io_follows_another_callback_with_message_logging_off( assert ("input.value" in _root_attrs(exporter)) is stamped -def test_root_input_cuts_an_image_data_uri_as_the_logged_messages_do(): +def test_root_input_cuts_an_image_data_uri_as_the_logged_messages_do() -> None: logger, exporter = _logger() messages: Final = [ { @@ -355,7 +398,7 @@ def test_root_input_cuts_an_image_data_uri_as_the_logged_messages_do(): assert root_input != messages -def test_root_output_cuts_an_image_data_uri_as_the_logged_messages_do(): +def test_root_output_cuts_an_image_data_uri_as_the_logged_messages_do() -> None: logger, exporter = _logger() image: Final = { "type": "image_url", @@ -373,19 +416,21 @@ def test_root_output_cuts_an_image_data_uri_as_the_logged_messages_do(): @pytest.mark.parametrize( - ("mappers", "expected"), + ("mappers", "output_keys"), [ - (OPENINFERENCE, OpenInferenceRootIOOpenTelemetryV2), - (("genai", "openinference", "weave"), OpenTelemetryV2), - (("genai", "langfuse"), LangfuseContentOpenTelemetryV2), - (("genai",), OpenTelemetryV2), + (OPENINFERENCE, {"output.value"}), + (("genai", "openinference", "weave"), set()), + (("genai", "langfuse"), {"langfuse.observation.output"}), + (("genai",), set()), ], ids=["arize-phoenix", "weave", "langfuse", "genai"], ) -def test_logger_class_follows_the_configured_vocabularies(mappers: tuple[str, ...], expected: type) -> None: - logger, _ = _logger(mappers=mappers) +def test_root_output_follows_the_configured_vocabularies(mappers: tuple[str, ...], output_keys: set[str]) -> None: + logger, exporter = _logger(mappers=mappers) - assert type(logger) is expected + _run_request(logger, CHAT_DATA, "acompletion", PONG) + + assert {"output.value", "langfuse.observation.output"} & set(_root_attrs(exporter)) == output_keys @pytest.mark.parametrize( @@ -412,13 +457,18 @@ def test_openinference_logger_keeps_the_pre_call_hook_that_seeds_metadata_baggag class _MasksReply(CustomLogger): - async def async_post_call_streaming_iterator_hook(self, user_api_key_dict, response, request_data): + async def async_post_call_streaming_iterator_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + response: AsyncIterator[ModelResponseStream], + request_data: dict[str, object], + ) -> AsyncGenerator[ModelResponseStream, None]: async for chunk in response: - choice = chunk.choices[0] + choice: Final = chunk.choices[0] yield _chat_chunk("" if choice.delta.content else None, finish_reason=choice.finish_reason) -async def _serve(chunks: Sequence[ModelResponseStream], data: dict) -> list[object]: +async def _serve(chunks: Sequence[ModelResponseStream], data: dict[str, object]) -> list[ModelResponseStream]: async def source() -> AsyncIterator[ModelResponseStream]: for chunk in chunks: yield chunk @@ -443,7 +493,7 @@ def test_stream_root_output_is_what_a_later_masking_hook_served( ) -> None: logger, exporter = _logger(mappers=mappers) monkeypatch.setattr(litellm, "callbacks", [logger, _MasksReply()]) - root = _start_root(logger) + root: Final = _start_root(logger) served: Final = asyncio.run( _serve((_chat_chunk("John Smith"), _chat_chunk(None, finish_reason="stop")), {**CHAT_DATA}) @@ -471,15 +521,15 @@ def test_arize_and_phoenix_presets_build_a_logger_that_stamps_the_root( monkeypatch.setenv(name, value) is_otel_v2_enabled.cache_clear() - loggers: list = [] + loggers: Final[list[CustomLogger]] = [] try: - built = _maybe_construct_otel_v2(callback, loggers) + built: Final = _maybe_construct_otel_v2(callback, loggers) assert isinstance(built, OpenInferenceRootIOOpenTelemetryV2) - root = _start_root(built) + root: Final = _start_root(built) asyncio.run( built.async_post_call_success_hook(data=CHAT_DATA, user_api_key_dict=UserAPIKeyAuth(), response=PONG) ) - attrs = dict(root.attributes or {}) + attrs: Final = dict(root.attributes or {}) assert json.loads(attrs["input.value"]) == [{"role": "user", "content": "ping"}] assert attrs["openinference.span.kind"] == "CHAIN" finally: