fix(otel-v2): emit GenAI message-content events for event_only / span_and_event

The V2 OpenTelemetry integration honored only the span half of
OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT. The mode drove
capture_span_content (which stamps gen_ai.input.messages /
gen_ai.output.messages), but nothing consumed the event half of the OTel
ContentCapturingMode contract; enable_events was dead and the engine emitted no
GenAI events. So span_and_event produced span attributes but no events, and
event_only produced neither, dropping the prompt and completion entirely.

Wire the capture mode to event emission: under event_only and span_and_event the
engine now records one gen_ai.{role}.message event per request message and one
gen_ai.choice per response choice on the LLM-call span, each carrying the
provider, role, and textual content. They are span events so they reach whatever
trace exporter is already configured, including the console exporter used to
reproduce the issue.

To keep event_only from leaking content onto span attributes, the retained
bodies are split from their span destination: the payload parser keeps the raw
prompt/response whenever either sink is active (capture_content), and a new
content_on_span flag gates whether the mappers may stamp them as attributes, so
all five content vocabularies stay correct across every mode. no_content and
span_only behavior is unchanged.

Fixes #30956
This commit is contained in:
akashkathole7 2026-06-22 17:09:31 +05:30
parent 6437b812be
commit a982517eb7
14 changed files with 325 additions and 17 deletions

View file

@ -10,6 +10,7 @@ from opentelemetry.trace.status import Status, StatusCode
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
from litellm.integrations.otel.mappers import resolve_mappers
from litellm.integrations.otel.mappers.base import AttributeMapper, SpanData
from litellm.integrations.otel.model.events import llm_message_events
from litellm.integrations.otel.model.payloads import (
GuardrailSpanData,
LLMCallSpanData,
@ -165,6 +166,14 @@ class SpanEmitter:
for mapper in self._mappers:
for key, value in mapper.map(data).items():
span.set_attribute(key, value)
# The event half of ``capture_message_content`` (``event_only`` /
# ``span_and_event``): record prompt/response bodies as span events so
# they reach the configured trace exporter even when content is kept off
# the span attributes (``event_only``). Span-attribute content is the
# mappers' job above and is gated separately by ``content_on_span``.
if self._config.capture_event_content and isinstance(data, LLMCallSpanData):
for name, attributes in llm_message_events(data):
span.add_event(name, attributes)
error = (
data.error
if isinstance(

View file

@ -321,7 +321,9 @@ class OpenTelemetryV2(CustomLogger):
carrier.span.end(end_time=to_ns(end_time))
return None
data = LLMCallSpanData.from_standard_logging_payload(
payload, capture_content=self.config.capture_span_content
payload,
capture_content=self.config.capture_content,
content_on_span=self.config.capture_span_content,
)
end_time_ns = to_ns(end_time)
if carrier.span is not None:

View file

@ -52,8 +52,10 @@ class GenAIMapper:
else None
),
GenAI.REQUEST_SEED: lambda d: d.request_params.seed,
GenAI.INPUT_MESSAGES: lambda d: serialize_messages(d.messages_in),
GenAI.OUTPUT_MESSAGES: lambda d: serialize_messages(output_messages(d)),
GenAI.INPUT_MESSAGES: lambda d: serialize_messages(d.span_messages_in),
GenAI.OUTPUT_MESSAGES: lambda d: serialize_messages(
output_messages(d.span_choices_out)
),
GenAI.RESPONSE_MODEL: lambda d: d.response_model,
GenAI.RESPONSE_ID: lambda d: d.response_id,
GenAI.RESPONSE_FINISH_REASONS: lambda d: (

View file

@ -57,8 +57,10 @@ class LangfuseMapper:
"langfuse.observation.model.parameters": lambda d: json_if(
collect(LangfuseMapper._MODEL_PARAMS, d.request_params)
),
"langfuse.observation.input": lambda d: serialize_messages(d.messages_in),
"langfuse.observation.output": lambda d: serialize_messages(output_messages(d)),
"langfuse.observation.input": lambda d: serialize_messages(d.span_messages_in),
"langfuse.observation.output": lambda d: serialize_messages(
output_messages(d.span_choices_out)
),
"langfuse.observation.usage_details": lambda d: json_if(
collect(LangfuseMapper._USAGE_FIELDS, d.usage)
),

View file

@ -42,10 +42,12 @@ class LangtraceMapper:
_BLOB_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = {
"llm.prompts": lambda d: (
json_or_none(list(d.messages_in)) if d.messages_in else None
json_or_none(list(d.span_messages_in)) if d.span_messages_in else None
),
"llm.completions": lambda d: (
json_or_none(output_messages(d)) if d.choices_out else None
json_or_none(output_messages(d.span_choices_out))
if d.span_choices_out
else None
),
}

View file

@ -82,9 +82,11 @@ class OpenInferenceMapper:
return {
**collect(cls._LLM_CALL_ATTRS, data),
**collect(cls._BLOB_ATTRS, data),
**cls._messages("llm.input_messages", "input.value", data.messages_in),
**cls._messages("llm.input_messages", "input.value", data.span_messages_in),
**cls._messages(
"llm.output_messages", "output.value", output_messages(data)
"llm.output_messages",
"output.value",
output_messages(data.span_choices_out),
),
**cls._tools(data),
}

View file

@ -9,7 +9,6 @@ import json
from typing import Callable, Mapping, Sequence
from litellm.integrations.otel.mappers.base import AttributeMap, AttrValue
from litellm.integrations.otel.model.payloads import LLMCallSpanData
def drop_none(values: Mapping[str, AttrValue | None]) -> AttributeMap:
@ -71,6 +70,6 @@ def message_content(message: object) -> str | None:
return None
def output_messages(data: LLMCallSpanData) -> list:
def output_messages(choices: Sequence[Mapping[str, object]]) -> list[object]:
"""The ``message`` payload of each response choice."""
return [c.get("message") for c in data.choices_out if isinstance(c, dict)]
return [c.get("message") for c in choices]

View file

@ -29,7 +29,7 @@ class WeaveMapper:
_BLOB_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = {
# Weave treats the response choices as the "output" payload.
"weave.output": lambda d: (
json_or_none(list(d.choices_out)) if d.choices_out else None
json_or_none(list(d.span_choices_out)) if d.span_choices_out else None
),
}

View file

@ -288,6 +288,30 @@ class OpenTelemetryV2Config(BaseSettings):
CaptureMessageContent.SPAN_AND_EVENT,
)
@property
def capture_event_content(self) -> bool:
"""Whether prompt/response content may be emitted as span events.
The event half of the OTel ``ContentCapturingMode`` contract: under
``event_only`` and ``span_and_event`` the message bodies are recorded as
``gen_ai.*`` events on the LLM-call span, complementing (or, for
``event_only``, standing in for) the ``gen_ai.input.messages`` /
``gen_ai.output.messages`` span attributes that ``capture_span_content``
governs. Defaults off, like its span counterpart.
"""
return self.capture_message_content in (
CaptureMessageContent.EVENT_ONLY,
CaptureMessageContent.SPAN_AND_EVENT,
)
@property
def capture_content(self) -> bool:
"""Whether message bodies are retained at all — as span attributes, span
events, or both. The payload parser keeps the raw prompt/response only
when this is true, so a request can never force content past either sink
while every capture mode is off."""
return self.capture_span_content or self.capture_event_content
@classmethod
def from_env(cls) -> "OpenTelemetryV2Config":
return cls()

View file

@ -0,0 +1,73 @@
"""GenAI message events: the event half of ``capture_message_content``.
The OTel GenAI ``ContentCapturingMode`` records prompt/response content either as
span attributes (``gen_ai.input.messages`` / ``gen_ai.output.messages``, owned by
the mappers) or as events, or both. This module owns the event representation:
one ``gen_ai.{role}.message`` event per request message and one ``gen_ai.choice``
event per response choice, stamped on the LLM-call span so they ride whatever
trace exporter the operator already configured — the console exporter included —
without a separate logs pipeline.
"""
from typing import Mapping, cast
from litellm.integrations.otel.mappers.base import AttributeMap
from litellm.integrations.otel.mappers.utils import drop_none, message_content
from litellm.integrations.otel.model.payloads import LLMCallSpanData
from litellm.integrations.otel.model.semconv import GenAI, GenAIEvent
from litellm.integrations.otel.model.utils import as_str
def _role(message: object) -> str | None:
"""The chat-message ``role``, or ``None`` when absent or not a dict."""
if not isinstance(message, dict):
return None
role = cast(Mapping[str, object], message).get("role")
return role if isinstance(role, str) and role else None
def llm_message_events(
data: LLMCallSpanData,
) -> tuple[tuple[str, AttributeMap], ...]:
"""``(event_name, attributes)`` pairs for an LLM call's prompt and response.
One ``gen_ai.{role}.message`` event per request message and one
``gen_ai.choice`` per response choice, each carrying the provider, role, and
(when present) the textual content. The caller stamps them as span events.
"""
provider = data.provider or None
inputs = tuple(_input_event(provider, m) for m in data.messages_in)
outputs = tuple(
_choice_event(provider, idx, c) for idx, c in enumerate(data.choices_out)
)
return inputs + outputs
def _input_event(
provider: str | None, message: Mapping[str, object]
) -> tuple[str, AttributeMap]:
role = _role(message) or "user"
attrs = drop_none(
{
GenAI.PROVIDER_NAME: provider,
"role": role,
"content": message_content(message),
}
)
return GenAIEvent.message(role), attrs
def _choice_event(
provider: str | None, index: int, choice: Mapping[str, object]
) -> tuple[str, AttributeMap]:
message = choice.get("message")
attrs = drop_none(
{
GenAI.PROVIDER_NAME: provider,
"index": index,
"finish_reason": as_str(choice.get("finish_reason")),
"role": _role(message),
"content": message_content(message),
}
)
return GenAIEvent.CHOICE, attrs

View file

@ -308,11 +308,31 @@ class LLMCallSpanData:
# dataclass stays hashable and frozen.
messages_in: tuple[Mapping[str, object], ...] = ()
choices_out: tuple[Mapping[str, object], ...] = ()
# Whether the retained bodies may be stamped as span *attributes*. False
# under ``event_only`` capture, where the bodies are kept for span events
# only and must not leak onto ``gen_ai.input.messages`` / ``...output...``
# (or any vendor vocabulary's) span attributes.
content_on_span: bool = True
system_fingerprint: str | None = None
@property
def span_messages_in(self) -> tuple[Mapping[str, object], ...]:
"""Request messages exposed to span-attribute mappers — empty when the
captured content is bound to events only."""
return self.messages_in if self.content_on_span else ()
@property
def span_choices_out(self) -> tuple[Mapping[str, object], ...]:
"""Response choices exposed to span-attribute mappers — empty when the
captured content is bound to events only."""
return self.choices_out if self.content_on_span else ()
@classmethod
def from_standard_logging_payload(
cls, payload: "StandardLoggingPayload", capture_content: bool = False
cls,
payload: "StandardLoggingPayload",
capture_content: bool = False,
content_on_span: bool | None = None,
) -> "LLMCallSpanData":
params = cast(Mapping[str, object], payload.get("model_parameters") or {})
# The single parse of the request's metadata — the request-vs-provider
@ -328,9 +348,12 @@ class LLMCallSpanData:
choices_out = _dicts(response.get("choices"))
# ``finish_reasons`` is metadata, not content, so derive it from
# ``choices_out`` before gating. The raw message/choice bodies are only
# retained when content capture is enabled (see ``capture_span_content``);
# otherwise the content-bearing mappers receive empty sequences and emit
# no prompt/response text.
# retained when content capture is enabled — for span attributes, span
# events, or both (see ``capture_content``). ``content_on_span`` then
# decides whether the bodies reach span attributes; under ``event_only``
# they are retained but kept off the span, for events alone. With capture
# off the content-bearing mappers receive empty sequences and emit no
# prompt/response text.
finish_reasons = _finish_reasons(choices_out)
return cls(
operation=resolve_operation(as_str(payload.get("call_type"))),
@ -356,6 +379,9 @@ class LLMCallSpanData:
tools=_extract_tools(params),
messages_in=_dicts(payload.get("messages")) if capture_content else (),
choices_out=choices_out if capture_content else (),
content_on_span=(
capture_content if content_on_span is None else content_on_span
),
system_fingerprint=as_str(response.get("system_fingerprint")),
)

View file

@ -92,6 +92,20 @@ class GenAI:
PROMPT_NAME: Final = "gen_ai.prompt.name"
class GenAIEvent:
"""OTel GenAI event names. Prompt/response content is recorded as these
events on the LLM-call span under the ``event_only`` / ``span_and_event``
capture modes — the event half of ``ContentCapturingMode``."""
CHOICE: Final = "gen_ai.choice"
@staticmethod
def message(role: str) -> str:
"""Per-message event name, e.g. ``gen_ai.user.message``. The standard
roles (system/user/assistant/tool) yield the canonical semconv names."""
return f"gen_ai.{role}.message"
class MCP:
"""OTel GenAI MCP (Model Context Protocol) span-attribute keys.

View file

@ -268,6 +268,67 @@ def test_genai_mapper_omits_messages_when_content_not_captured():
assert GenAI.OUTPUT_MESSAGES not in attrs
def _captured_llm_call(content_on_span=True):
return LLMCallSpanData(
operation=GenAIOperation.CHAT,
provider="openai",
request_model="gpt-4o",
response_model="gpt-4o-2024",
response_id="resp_1",
request_params=LLMRequestParams(),
usage=LLMUsage(),
finish_reasons=("stop",),
error=None,
response_cost=None,
server=None,
identity=RequestIdentity(call_id="c1"),
messages_in=(
{"role": "system", "content": "Be concise."},
{"role": "user", "content": "What's the weather?"},
),
choices_out=(
{
"finish_reason": "stop",
"message": {"role": "assistant", "content": "Sunny."},
},
),
content_on_span=content_on_span,
)
def test_genai_mapper_withholds_span_content_when_event_only():
"""``content_on_span=False`` (the ``event_only`` mode) keeps retained bodies
off the span attributes even though they're present for the event path."""
attrs = GenAIMapper().map(_captured_llm_call(content_on_span=False))
assert GenAI.INPUT_MESSAGES not in attrs
assert GenAI.OUTPUT_MESSAGES not in attrs
def test_llm_message_events_carry_role_and_content():
from litellm.integrations.otel.model.events import llm_message_events
events = llm_message_events(_captured_llm_call())
names = [name for name, _ in events]
assert names == ["gen_ai.system.message", "gen_ai.user.message", "gen_ai.choice"]
by_name = {name: attrs for name, attrs in events}
assert by_name["gen_ai.system.message"]["role"] == "system"
assert by_name["gen_ai.system.message"]["content"] == "Be concise."
assert by_name["gen_ai.user.message"]["content"] == "What's the weather?"
choice = by_name["gen_ai.choice"]
assert choice["index"] == 0
assert choice["finish_reason"] == "stop"
assert choice["role"] == "assistant"
assert choice["content"] == "Sunny."
assert choice[GenAI.PROVIDER_NAME] == "openai"
def test_llm_message_events_empty_without_bodies():
from litellm.integrations.otel.model.events import llm_message_events
assert llm_message_events(_full_llm_call()) == ()
def test_genai_mapper_cost_breakdown():
from litellm.integrations.otel.model.semconv import LiteLLM

View file

@ -238,6 +238,98 @@ def test_idempotent_on_repeat_callback():
assert len(exporter.get_finished_spans()) == 1
# --------------------------------------------------------------------------- #
# Message-content capture modes (issue #30956)
# --------------------------------------------------------------------------- #
def _logger_for_mode(mode):
cfg = OpenTelemetryV2Config(
exporter="in_memory",
legacy_compat=False,
capture_message_content=mode,
)
exporter = InMemorySpanExporter()
tracer_provider = providers.build_tracer_provider(cfg, exporter=exporter)
return OpenTelemetryV2(config=cfg, tracer_provider=tracer_provider), exporter
def _content_payload():
return _payload(
messages=[
{"role": "system", "content": "Be concise."},
{"role": "user", "content": "What's the capital of France?"},
],
response={
"id": "resp_1",
"model": "gpt-4o-2024",
"choices": [
{
"finish_reason": "stop",
"message": {"role": "assistant", "content": "Paris."},
}
],
},
)
def _emit_content_span(mode):
logger, exporter = _logger_for_mode(mode)
_emit_llm(logger, _kwargs(_content_payload()))
(span,) = exporter.get_finished_spans()
return span
def test_span_and_event_emits_both_attributes_and_events():
"""Regression for #30956: ``span_and_event`` must put message content on the
span attributes AND emit semantic events. Before the fix, only the span
attributes were produced and no events were emitted."""
span = _emit_content_span("span_and_event")
attrs = dict(span.attributes or {})
assert "What's the capital of France?" in attrs[GenAI.INPUT_MESSAGES]
assert "Paris." in attrs[GenAI.OUTPUT_MESSAGES]
events = {e.name: dict(e.attributes or {}) for e in span.events}
assert set(events) == {
"gen_ai.system.message",
"gen_ai.user.message",
"gen_ai.choice",
}
assert events["gen_ai.user.message"]["content"] == "What's the capital of France?"
assert events["gen_ai.choice"]["content"] == "Paris."
assert events["gen_ai.choice"]["finish_reason"] == "stop"
def test_event_only_emits_events_without_span_content():
"""Regression for #30956: ``event_only`` must still capture content — as
events — instead of dropping it. Before the fix this mode produced neither
span attributes nor events, so the content was lost entirely."""
span = _emit_content_span("event_only")
attrs = dict(span.attributes or {})
assert GenAI.INPUT_MESSAGES not in attrs
assert GenAI.OUTPUT_MESSAGES not in attrs
events = {e.name for e in span.events}
assert events == {"gen_ai.system.message", "gen_ai.user.message", "gen_ai.choice"}
def test_span_only_emits_no_message_events():
span = _emit_content_span("span_only")
attrs = dict(span.attributes or {})
assert GenAI.INPUT_MESSAGES in attrs
assert [e for e in span.events if e.name.startswith("gen_ai.")] == []
def test_no_content_emits_neither_attributes_nor_events():
span = _emit_content_span("no_content")
attrs = dict(span.attributes or {})
assert GenAI.INPUT_MESSAGES not in attrs
assert GenAI.OUTPUT_MESSAGES not in attrs
assert [e for e in span.events if e.name.startswith("gen_ai.")] == []
# --------------------------------------------------------------------------- #
# MCP tool-call spans
# --------------------------------------------------------------------------- #