mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-12 23:01:41 +00:00
fix(otel): record full error message on standard exception event in otel v2 (#30380)
The v2 span engine only stamped error.type and stuffed the message into the
span status description; it never recorded the standard OTel exception event.
Backends that dynamic-map unknown string fields (e.g. Elasticsearch) index the
message as a keyword capped at ignore_above:1024, truncating it. Emit the full
message under the recognized exception.message semconv field via a span event so
it is mapped as full text instead.
Co-authored-by: Claude <noreply@anthropic.com>
(cherry picked from commit 3b84150137)
This commit is contained in:
parent
ba5e46b709
commit
ca7d360d87
3 changed files with 111 additions and 4 deletions
|
|
@ -17,7 +17,7 @@ from litellm.integrations.otel.model.payloads import (
|
|||
ServiceSpanData,
|
||||
)
|
||||
from litellm.integrations.otel.plumbing.providers import to_otel_span_kind
|
||||
from litellm.integrations.otel.model.semconv import Error
|
||||
from litellm.integrations.otel.model.semconv import Error, ExceptionEvent
|
||||
from litellm.integrations.otel.model.spans import (
|
||||
SPAN_REGISTRY,
|
||||
SpanRole,
|
||||
|
|
@ -179,9 +179,17 @@ class SpanEmitter:
|
|||
else None
|
||||
)
|
||||
if error and (error.error_type or error.message):
|
||||
span.set_attribute(Error.TYPE, error.error_type or "error")
|
||||
span.set_status(
|
||||
Status(StatusCode.ERROR, error.message or error.error_type or "error")
|
||||
error_type = error.error_type or "error"
|
||||
message = error.message or error.error_type or "error"
|
||||
span.set_attribute(Error.TYPE, error_type)
|
||||
span.set_status(Status(StatusCode.ERROR, message))
|
||||
# Carry the full message on the standard ``exception`` event so backends
|
||||
# map it as full text under ``exception.message``. Setting it as a bare
|
||||
# string attribute instead lets backends like Elasticsearch dynamic-map
|
||||
# it to a ``keyword`` capped at 1024 chars, truncating the message.
|
||||
span.add_event(
|
||||
ExceptionEvent.NAME,
|
||||
{ExceptionEvent.TYPE: error_type, ExceptionEvent.MESSAGE: message},
|
||||
)
|
||||
# On success leave the status UNSET (the semconv default) rather than
|
||||
# forcing OK — that matches the FastAPI server span and avoids implying a
|
||||
|
|
|
|||
|
|
@ -146,6 +146,21 @@ class Error:
|
|||
TYPE: Final = "error.type"
|
||||
|
||||
|
||||
class ExceptionEvent:
|
||||
"""OTel exception-event name and attribute keys (semconv ``exception.*``).
|
||||
|
||||
The full error message rides ``exception.message`` on a span event rather than
|
||||
a custom string attribute. Backends recognise these semantic-convention names
|
||||
and map them as full text; an unrecognised key (e.g. ``error_message``) falls
|
||||
into the default dynamic template, which truncates strings to a 1024-char
|
||||
``keyword``.
|
||||
"""
|
||||
|
||||
NAME: Final = "exception"
|
||||
TYPE: Final = "exception.type"
|
||||
MESSAGE: Final = "exception.message"
|
||||
|
||||
|
||||
class Server:
|
||||
ADDRESS: Final = "server.address"
|
||||
PORT: Final = "server.port"
|
||||
|
|
|
|||
|
|
@ -410,6 +410,90 @@ def test_emitter_without_call_id_is_not_deduped():
|
|||
assert len(exporter.get_finished_spans()) == 2
|
||||
|
||||
|
||||
def _emit_error_span(message, error_type="litellm.APIError"):
|
||||
from litellm.integrations.otel.emitter import SpanEmitter
|
||||
|
||||
cfg = OpenTelemetryV2Config(exporter="in_memory")
|
||||
provider, exporter = providers.in_memory_provider(cfg)
|
||||
engine = SpanEmitter(providers.get_tracer(provider, "t"), cfg)
|
||||
data = LLMCallSpanData(
|
||||
operation=GenAIOperation.CHAT,
|
||||
provider="openai",
|
||||
request_model="gpt-4o",
|
||||
response_model=None,
|
||||
response_id=None,
|
||||
request_params=LLMRequestParams(),
|
||||
usage=LLMUsage(),
|
||||
finish_reasons=(),
|
||||
error=SpanError(error_type=error_type, message=message),
|
||||
response_cost=None,
|
||||
server=None,
|
||||
identity=RequestIdentity(call_id=None),
|
||||
)
|
||||
engine.emit(SpanRole.LLM_CALL, data)
|
||||
(span,) = exporter.get_finished_spans()
|
||||
return span
|
||||
|
||||
|
||||
def _exception_event(span):
|
||||
from litellm.integrations.otel.model.semconv import ExceptionEvent
|
||||
|
||||
events = [e for e in span.events if e.name == ExceptionEvent.NAME]
|
||||
assert len(events) == 1, "expected exactly one exception event"
|
||||
return events[0]
|
||||
|
||||
|
||||
def test_error_message_recorded_as_full_exception_event_untruncated():
|
||||
"""Regression for the Elasticsearch keyword/ignore_above:1024 truncation.
|
||||
|
||||
A long error message must survive intact on the standard ``exception``
|
||||
event under ``exception.message`` — not get dropped onto a bare string
|
||||
attribute that backends dynamic-map to a 1024-char ``keyword``. The SDK
|
||||
must not truncate it either, so a 5000-char message stays 5000 chars.
|
||||
"""
|
||||
from litellm.integrations.otel.model.semconv import Error, ExceptionEvent
|
||||
|
||||
long_message = "boom: " + "x" * 5000
|
||||
span = _emit_error_span(long_message, error_type="litellm.APIError")
|
||||
|
||||
event = _exception_event(span)
|
||||
assert event.attributes[ExceptionEvent.MESSAGE] == long_message
|
||||
assert len(event.attributes[ExceptionEvent.MESSAGE]) == len(long_message) > 1024
|
||||
assert event.attributes[ExceptionEvent.TYPE] == "litellm.APIError"
|
||||
|
||||
# error.type stays a low-cardinality attribute; the message does NOT become a
|
||||
# bare string attribute (which is what got truncated).
|
||||
assert span.attributes[Error.TYPE] == "litellm.APIError"
|
||||
assert ExceptionEvent.MESSAGE not in span.attributes
|
||||
assert span.status.description == long_message
|
||||
|
||||
|
||||
def test_success_span_records_no_exception_event():
|
||||
from litellm.integrations.otel.emitter import SpanEmitter
|
||||
from litellm.integrations.otel.model.semconv import ExceptionEvent
|
||||
|
||||
cfg = OpenTelemetryV2Config(exporter="in_memory")
|
||||
provider, exporter = providers.in_memory_provider(cfg)
|
||||
engine = SpanEmitter(providers.get_tracer(provider, "t"), cfg)
|
||||
data = LLMCallSpanData(
|
||||
operation=GenAIOperation.CHAT,
|
||||
provider="openai",
|
||||
request_model="gpt-4o",
|
||||
response_model="gpt-4o",
|
||||
response_id="resp-1",
|
||||
request_params=LLMRequestParams(),
|
||||
usage=LLMUsage(),
|
||||
finish_reasons=("stop",),
|
||||
error=None,
|
||||
response_cost=None,
|
||||
server=None,
|
||||
identity=RequestIdentity(call_id=None),
|
||||
)
|
||||
engine.emit(SpanRole.LLM_CALL, data)
|
||||
(span,) = exporter.get_finished_spans()
|
||||
assert all(e.name != ExceptionEvent.NAME for e in span.events)
|
||||
|
||||
|
||||
# --- service taxonomy: which calls become spans, and of what kind ----------- #
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue