From 27b5cfd20dead4f9ec2b8a3639f87fe12bdd1a25 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Wed, 8 Jul 2026 13:44:48 -0700 Subject: [PATCH] fix(otel): restore error.* span attributes on v2 error spans (LIT-4179) (#32524) The v2 emitter has never stamped error.message / error.code / error.stack_trace / error.llm_provider as span attributes; only error.type reached the wire. Backends that flatten span attributes into label indexes (Elastic APM labels.error_*, Datadog span tags) lost these four fields when v2 became the active integration on v1.90+ for otel_v2-flagged deployments. The pre-existing exception span event carrying the full message (LIT-3758) is unchanged; the message now rides both places at once, matching v1s shape. SpanError grows three optional detail fields; _parse_error threads them from StandardLoggingPayloadErrorInformation; the emitters error branch stamps them via a new module-level helper, guarded per field so guardrail-shape errors are not polluted with empty attributes. New semconv constants mirror open_inference.ErrorAttributes byte-for-byte, so v1 and v2 consumers read the same keys. Regression tests extend the mapped test files under tests/test_litellm/integrations/otel/. pytest reports 243 passed. (cherry picked from commit 85d1fe6e2a535e9edfc1ae0b0854eb204573c7ba) --- litellm/integrations/otel/__init__.py | 2 + litellm/integrations/otel/emitter.py | 35 +++++- litellm/integrations/otel/model/payloads.py | 6 + litellm/integrations/otel/model/semconv.py | 20 ++++ .../otel/test_otel_v2_components.py | 110 ++++++++++++++++-- .../otel/test_otel_v2_sources_of_truth.py | 47 +++++++- 6 files changed, 203 insertions(+), 17 deletions(-) diff --git a/litellm/integrations/otel/__init__.py b/litellm/integrations/otel/__init__.py index 7f78f7156b4..5e167e006ff 100644 --- a/litellm/integrations/otel/__init__.py +++ b/litellm/integrations/otel/__init__.py @@ -52,6 +52,7 @@ from litellm.integrations.otel.model.semconv import ( GenAIProvider, JsonRpc, LiteLLM, + LiteLLMError, MCPMethod, Metric, Network, @@ -87,6 +88,7 @@ __all__ = [ "HTTP", "JsonRpc", "LiteLLM", + "LiteLLMError", "MCP", "MCPMethod", "Metric", diff --git a/litellm/integrations/otel/emitter.py b/litellm/integrations/otel/emitter.py index 8441cbae834..46aa166a8bb 100644 --- a/litellm/integrations/otel/emitter.py +++ b/litellm/integrations/otel/emitter.py @@ -16,9 +16,10 @@ from litellm.integrations.otel.model.payloads import ( MCPListToolsSpanData, MCPToolCallSpanData, ServiceSpanData, + SpanError, ) from litellm.integrations.otel.plumbing.providers import to_otel_span_kind -from litellm.integrations.otel.model.semconv import Error, ExceptionEvent +from litellm.integrations.otel.model.semconv import Error, ExceptionEvent, LiteLLMError from litellm.integrations.otel.model.spans import ( SPAN_REGISTRY, SpanRole, @@ -49,6 +50,27 @@ _NAME_BUILDERS: dict[SpanRole, Callable[..., str]] = { _DEDUP_CACHE_MAX = 10_000 +def _stamp_otel_error_attributes(span: Span, error_type: str, resolved_message: str) -> None: + """Stamp the OTel-semconv error attributes (``error.type`` + ``error.message``). + ``error_type`` and ``resolved_message`` are ``finish_span``'s already-computed + fallback chains, so the pair on the status, event, and attributes stays in + lockstep.""" + span.set_attribute(Error.TYPE, error_type) + span.set_attribute(Error.MESSAGE, resolved_message) + + +def _stamp_litellm_error_attributes(span: Span, error: SpanError) -> None: + """Stamp litellm-specific error detail attributes. Emitted only when the + corresponding field is populated so guardrail-shape errors carrying only a + message aren't polluted with empty detail keys.""" + if error.code: + span.set_attribute(LiteLLMError.CODE, error.code) + if error.stack_trace: + span.set_attribute(LiteLLMError.STACK_TRACE, error.stack_trace) + if error.llm_provider: + span.set_attribute(LiteLLMError.LLM_PROVIDER, error.llm_provider) + + class SpanEmitter: def __init__( self, @@ -190,12 +212,13 @@ class SpanEmitter: if error and (error.error_type or error.message): error_type = error.error_type or "error" message = error.message or error.error_type or "error" - span.set_attribute(Error.TYPE, error_type) + _stamp_otel_error_attributes(span, error_type, message) + _stamp_litellm_error_attributes(span, error) span.set_status(Status(StatusCode.ERROR, message)) - # Carry the full message on the standard ``exception`` event so backends - # map it as full text under ``exception.message``. Setting it as a bare - # string attribute instead lets backends like Elasticsearch dynamic-map - # it to a ``keyword`` capped at 1024 chars, truncating the message. + # Also emit the semconv ``exception`` event so backends that + # dynamic-map unknown string span attrs to ``keyword`` (e.g. + # Elasticsearch with a 1024-char ``ignore_above``) still see the + # full untruncated message on the recognized event field. span.add_event( ExceptionEvent.NAME, {ExceptionEvent.TYPE: error_type, ExceptionEvent.MESSAGE: message}, diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index fcd710492f0..4a8f01858b5 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -141,6 +141,9 @@ class LLMCost: class SpanError: error_type: str | None = None message: str | None = None + code: str | None = None + stack_trace: str | None = None + llm_provider: str | None = None @dataclass(frozen=True) @@ -571,6 +574,9 @@ def _parse_error(payload: "StandardLoggingPayload") -> SpanError | None: return SpanError( error_type=as_str(info.get("error_class")) or as_str(info.get("error_code")), message=as_str(info.get("error_message")) or as_str(payload.get("error_str")), + code=as_str(info.get("error_code")), + stack_trace=as_str(info.get("traceback")), + llm_provider=as_str(info.get("llm_provider")), ) diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py index 4e725ae0a29..69d1e454655 100644 --- a/litellm/integrations/otel/model/semconv.py +++ b/litellm/integrations/otel/model/semconv.py @@ -144,7 +144,27 @@ class Client: class Error: + """OTel-defined error attribute keys, from the semconv ``error.*`` registry. + ``MESSAGE`` is marked *Deprecated* upstream in favor of domain-specific + error message keys plus ``exception.message`` on the exception event, but + is still defined and stamped by litellm's v1 integration; keeping it here + for byte-for-byte parity.""" + TYPE: Final = "error.type" + MESSAGE: Final = "error.message" + + +class LiteLLMError: + """LiteLLM-specific error attribute keys. Emitted under the ``error.*`` + namespace (not ``litellm.*``) for byte-for-byte compat with the v1 + integration in ``opentelemetry.py``; consumers reading these keys on v1 + spans read the same keys on v2 spans. OTel semconv does not define any of + these three, and per its extension rules a namespace may carry additional + vendor keys as long as they don't collide with defined names.""" + + CODE: Final = "error.code" + STACK_TRACE: Final = "error.stack_trace" + LLM_PROVIDER: Final = "error.llm_provider" class ExceptionEvent: diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_components.py b/tests/test_litellm/integrations/otel/test_otel_v2_components.py index 19eef284b91..298047ec18b 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_components.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_components.py @@ -579,13 +579,10 @@ def _exception_event(span): def test_error_message_recorded_as_full_exception_event_untruncated(): - """Regression for the Elasticsearch keyword/ignore_above:1024 truncation. - - A long error message must survive intact on the standard ``exception`` - event under ``exception.message`` — not get dropped onto a bare string - attribute that backends dynamic-map to a 1024-char ``keyword``. The SDK - must not truncate it either, so a 5000-char message stays 5000 chars. - """ + """The ``exception`` event carries the full untruncated message under + ``exception.message`` so backends that dynamic-map unknown string span + attrs to ``keyword`` (e.g. Elasticsearch with a 1024-char ``ignore_above``) + still see it in full via the semconv-recognized event field.""" from litellm.integrations.otel.model.semconv import Error, ExceptionEvent long_message = "boom: " + "x" * 5000 @@ -596,13 +593,108 @@ def test_error_message_recorded_as_full_exception_event_untruncated(): assert len(event.attributes[ExceptionEvent.MESSAGE]) == len(long_message) > 1024 assert event.attributes[ExceptionEvent.TYPE] == "litellm.APIError" - # error.type stays a low-cardinality attribute; the message does NOT become a - # bare string attribute (which is what got truncated). + # error.type stays a low-cardinality attribute; the exception EVENT field + # ``exception.message`` never becomes a bare string attribute. assert span.attributes[Error.TYPE] == "litellm.APIError" assert ExceptionEvent.MESSAGE not in span.attributes assert span.status.description == long_message +def test_error_details_stamped_as_span_attributes_for_labels_ingest(): + """OTel-defined keys and litellm-specific detail keys both ride span + attributes so backends that flatten attrs into label indexes (Elastic APM + ``labels.*``, Datadog span tags) render them. The exception event with the + full untruncated message stays alongside — both places, matching v1's + shape.""" + from litellm.integrations.otel.model.semconv import Error, ExceptionEvent, LiteLLMError + from litellm.integrations.otel.emitter import SpanEmitter + + cfg = OpenTelemetryV2Config(exporter="in_memory") + provider, exporter = providers.in_memory_provider(cfg) + engine = SpanEmitter(providers.get_tracer(provider, "t"), cfg) + data = LLMCallSpanData( + operation=GenAIOperation.CHAT, + provider="openai", + request_model="gpt-4o", + response_model=None, + response_id=None, + request_params=LLMRequestParams(), + usage=LLMUsage(), + finish_reasons=(), + error=SpanError( + error_type="litellm.BadRequestError", + message="400: violated moderation policy", + code="400", + stack_trace="File proxy_server.py line 8570 ...", + llm_provider="openai", + ), + response_cost=None, + server=None, + identity=RequestIdentity(call_id=None), + ) + engine.emit(SpanRole.LLM_CALL, data) + (span,) = exporter.get_finished_spans() + + # OTel-defined keys (from the ``error.*`` semconv registry). + assert span.attributes[Error.TYPE] == "litellm.BadRequestError" + assert span.attributes[Error.MESSAGE] == "400: violated moderation policy" + # LiteLLM-specific detail keys — vendor-namespaced under ``error.*`` + # for v1-parity, not defined by OTel semconv. + assert span.attributes[LiteLLMError.CODE] == "400" + assert span.attributes[LiteLLMError.STACK_TRACE] == "File proxy_server.py line 8570 ..." + assert span.attributes[LiteLLMError.LLM_PROVIDER] == "openai" + + # The exception event carries the same message on the span too. + event = _exception_event(span) + assert event.attributes[ExceptionEvent.MESSAGE] == "400: violated moderation policy" + + +def test_error_details_omitted_when_span_error_carries_only_message(): + """A guardrail-shape error (message only, no code/traceback/provider) must + not pollute the span with empty-string detail attributes. Only the keys + that carry real data land.""" + from litellm.integrations.otel.model.semconv import Error, LiteLLMError + + span = _emit_error_span("guardrail rejected", error_type="ContentFilter") + + assert span.attributes[Error.TYPE] == "ContentFilter" + assert span.attributes[Error.MESSAGE] == "guardrail rejected" + # LiteLLM-specific detail keys aren't stamped when the SpanError doesn't + # carry them. + assert LiteLLMError.CODE not in span.attributes + assert LiteLLMError.STACK_TRACE not in span.attributes + assert LiteLLMError.LLM_PROVIDER not in span.attributes + + +def test_v2_error_attribute_keys_match_v1_error_attributes_byte_for_byte(): + """v1 (``opentelemetry.py``) and v2 (``otel/`` package) stamp identical + span-attribute keys so consumers reading ``labels.error_message`` don't + care which integration produced the span. Renaming either side is a + breaking change for downstream dashboards; this test locks the vocabulary.""" + from litellm.integrations._types.open_inference import ErrorAttributes + from litellm.integrations.otel.model.semconv import Error, LiteLLMError + + assert Error.TYPE == ErrorAttributes.ERROR_TYPE + assert Error.MESSAGE == ErrorAttributes.ERROR_MESSAGE + assert LiteLLMError.CODE == ErrorAttributes.ERROR_CODE + assert LiteLLMError.STACK_TRACE == ErrorAttributes.ERROR_STACK_TRACE + assert LiteLLMError.LLM_PROVIDER == ErrorAttributes.ERROR_LLM_PROVIDER + + +def test_error_message_falls_back_to_error_type_when_message_absent(): + """A ``SpanError(error_type=..., message=None)`` still renders on the span: + the resolved message is the error_type, and it lands on ``error.message``, + the exception event, and the span-status description in lockstep so a + single-source-of-truth view isn't inconsistent.""" + from litellm.integrations.otel.model.semconv import Error, ExceptionEvent + + span = _emit_error_span(message=None, error_type="RateLimitError") + + assert span.attributes[Error.MESSAGE] == "RateLimitError" + assert _exception_event(span).attributes[ExceptionEvent.MESSAGE] == "RateLimitError" + assert span.status.description == "RateLimitError" + + def test_success_span_records_no_exception_event(): from litellm.integrations.otel.emitter import SpanEmitter from litellm.integrations.otel.model.semconv import ExceptionEvent diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py index 834a484090f..89aa73a6066 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py @@ -144,11 +144,13 @@ def _all_constants(cls): def test_attribute_keys_are_unique_across_namespaces(): - from litellm.integrations.otel import MCP, Client, JsonRpc, Network + from litellm.integrations.otel import MCP, Client, JsonRpc, LiteLLMError, Network # prefixes are allowed to be substrings; exact keys must not collide. + # ``LiteLLMError`` shares the ``error.*`` prefix with ``Error`` by design + # (v1-parity); the assert below is the guarantee they never overlap. exact = set() - for cls in (GenAI, Error, Server, HTTP, DB, MCP, JsonRpc, Network, Client): + for cls in (GenAI, Error, LiteLLMError, Server, HTTP, DB, MCP, JsonRpc, Network, Client): for key in _all_constants(cls): assert key not in exact, f"duplicate attribute key {key}" exact.add(key) @@ -342,6 +344,47 @@ def test_llm_call_adapter_failure_path(): assert data.error.message == "429 slow down" +def test_llm_call_adapter_carries_error_detail_fields(): + """``_parse_error`` threads the full detail set from ``error_information`` + (``error_code``, ``traceback``, ``llm_provider``) onto ``SpanError`` so the + emitter can stamp them as span attributes.""" + payload = _sample_payload( + status="failure", + error_information={ + "error_class": "BadRequestError", + "error_message": "400 violated moderation policy", + "error_code": "400", + "traceback": "File proxy_server.py line 8570 ...", + "llm_provider": "openai", + }, + ) + data = LLMCallSpanData.from_standard_logging_payload(payload) + assert data.error is not None + assert data.error.error_type == "BadRequestError" + assert data.error.message == "400 violated moderation policy" + assert data.error.code == "400" + assert data.error.stack_trace == "File proxy_server.py line 8570 ..." + assert data.error.llm_provider == "openai" + + +def test_llm_call_adapter_error_details_default_to_none_when_absent(): + """Guardrail-shape payloads carry only ``error_class`` + ``error_message``. + The detail fields must stay ``None`` so the emitter's ``if error.code:`` + guards skip stamping empty attributes.""" + payload = _sample_payload( + status="failure", + error_information={ + "error_class": "ContentFilter", + "error_message": "guardrail rejected", + }, + ) + data = LLMCallSpanData.from_standard_logging_payload(payload) + assert data.error is not None + assert data.error.code is None + assert data.error.stack_trace is None + assert data.error.llm_provider is None + + def test_adapter_is_resilient_to_minimal_payload(): data = LLMCallSpanData.from_standard_logging_payload({}) assert data.request_model == ""