fix(otel): record full error message on standard exception event in otel v2 (#30380)

The v2 span engine only stamped error.type and stuffed the message into the
span status description; it never recorded the standard OTel exception event.
Backends that dynamic-map unknown string fields (e.g. Elasticsearch) index the
message as a keyword capped at ignore_above:1024, truncating it. Emit the full
message under the recognized exception.message semconv field via a span event so
it is mapped as full text instead.

Co-authored-by: Claude <noreply@anthropic.com>
(cherry picked from commit 3b84150137)
This commit is contained in:
Yassin Kortam 2026-06-13 11:42:43 -07:00 committed by Yuneng Jiang
parent ba5e46b709
commit ca7d360d87
No known key found for this signature in database
3 changed files with 111 additions and 4 deletions

View file

@ -17,7 +17,7 @@ from litellm.integrations.otel.model.payloads import (
ServiceSpanData,
)
from litellm.integrations.otel.plumbing.providers import to_otel_span_kind
from litellm.integrations.otel.model.semconv import Error
from litellm.integrations.otel.model.semconv import Error, ExceptionEvent
from litellm.integrations.otel.model.spans import (
SPAN_REGISTRY,
SpanRole,
@ -179,9 +179,17 @@ class SpanEmitter:
else None
)
if error and (error.error_type or error.message):
span.set_attribute(Error.TYPE, error.error_type or "error")
span.set_status(
Status(StatusCode.ERROR, error.message or error.error_type or "error")
error_type = error.error_type or "error"
message = error.message or error.error_type or "error"
span.set_attribute(Error.TYPE, error_type)
span.set_status(Status(StatusCode.ERROR, message))
# Carry the full message on the standard ``exception`` event so backends
# map it as full text under ``exception.message``. Setting it as a bare
# string attribute instead lets backends like Elasticsearch dynamic-map
# it to a ``keyword`` capped at 1024 chars, truncating the message.
span.add_event(
ExceptionEvent.NAME,
{ExceptionEvent.TYPE: error_type, ExceptionEvent.MESSAGE: message},
)
# On success leave the status UNSET (the semconv default) rather than
# forcing OK — that matches the FastAPI server span and avoids implying a

View file

@ -146,6 +146,21 @@ class Error:
TYPE: Final = "error.type"
class ExceptionEvent:
"""OTel exception-event name and attribute keys (semconv ``exception.*``).
The full error message rides ``exception.message`` on a span event rather than
a custom string attribute. Backends recognise these semantic-convention names
and map them as full text; an unrecognised key (e.g. ``error_message``) falls
into the default dynamic template, which truncates strings to a 1024-char
``keyword``.
"""
NAME: Final = "exception"
TYPE: Final = "exception.type"
MESSAGE: Final = "exception.message"
class Server:
ADDRESS: Final = "server.address"
PORT: Final = "server.port"

View file

@ -410,6 +410,90 @@ def test_emitter_without_call_id_is_not_deduped():
assert len(exporter.get_finished_spans()) == 2
def _emit_error_span(message, error_type="litellm.APIError"):
from litellm.integrations.otel.emitter import SpanEmitter
cfg = OpenTelemetryV2Config(exporter="in_memory")
provider, exporter = providers.in_memory_provider(cfg)
engine = SpanEmitter(providers.get_tracer(provider, "t"), cfg)
data = LLMCallSpanData(
operation=GenAIOperation.CHAT,
provider="openai",
request_model="gpt-4o",
response_model=None,
response_id=None,
request_params=LLMRequestParams(),
usage=LLMUsage(),
finish_reasons=(),
error=SpanError(error_type=error_type, message=message),
response_cost=None,
server=None,
identity=RequestIdentity(call_id=None),
)
engine.emit(SpanRole.LLM_CALL, data)
(span,) = exporter.get_finished_spans()
return span
def _exception_event(span):
from litellm.integrations.otel.model.semconv import ExceptionEvent
events = [e for e in span.events if e.name == ExceptionEvent.NAME]
assert len(events) == 1, "expected exactly one exception event"
return events[0]
def test_error_message_recorded_as_full_exception_event_untruncated():
"""Regression for the Elasticsearch keyword/ignore_above:1024 truncation.
A long error message must survive intact on the standard ``exception``
event under ``exception.message`` not get dropped onto a bare string
attribute that backends dynamic-map to a 1024-char ``keyword``. The SDK
must not truncate it either, so a 5000-char message stays 5000 chars.
"""
from litellm.integrations.otel.model.semconv import Error, ExceptionEvent
long_message = "boom: " + "x" * 5000
span = _emit_error_span(long_message, error_type="litellm.APIError")
event = _exception_event(span)
assert event.attributes[ExceptionEvent.MESSAGE] == long_message
assert len(event.attributes[ExceptionEvent.MESSAGE]) == len(long_message) > 1024
assert event.attributes[ExceptionEvent.TYPE] == "litellm.APIError"
# error.type stays a low-cardinality attribute; the message does NOT become a
# bare string attribute (which is what got truncated).
assert span.attributes[Error.TYPE] == "litellm.APIError"
assert ExceptionEvent.MESSAGE not in span.attributes
assert span.status.description == long_message
def test_success_span_records_no_exception_event():
from litellm.integrations.otel.emitter import SpanEmitter
from litellm.integrations.otel.model.semconv import ExceptionEvent
cfg = OpenTelemetryV2Config(exporter="in_memory")
provider, exporter = providers.in_memory_provider(cfg)
engine = SpanEmitter(providers.get_tracer(provider, "t"), cfg)
data = LLMCallSpanData(
operation=GenAIOperation.CHAT,
provider="openai",
request_model="gpt-4o",
response_model="gpt-4o",
response_id="resp-1",
request_params=LLMRequestParams(),
usage=LLMUsage(),
finish_reasons=("stop",),
error=None,
response_cost=None,
server=None,
identity=RequestIdentity(call_id=None),
)
engine.emit(SpanRole.LLM_CALL, data)
(span,) = exporter.get_finished_spans()
assert all(e.name != ExceptionEvent.NAME for e in span.events)
# --- service taxonomy: which calls become spans, and of what kind ----------- #