fix(otel): put the error message on every failed span status (#44185)

* fix(otel): put the error message on the failed span status

The legacy OTel logger (used by otel, langfuse_otel, arize and others) ended a failed call with Status(StatusCode.ERROR) and no description, so backends that read the span status, such as Langfuse's status message, showed an empty error. The text was only in the error.message attribute. Use the same message as that attribute, as the OTel v2 emitter already does.

* fix(otel): take the failure status message from _record_exception_on_span

Instead of reading the logging payload a second time in _handle_failure,
_record_exception_on_span returns the error message it already records
as error.message, and _handle_failure puts it on the ERROR status. The
other callers ignore the return value, so their spans are unchanged.
Its input is now a typed view of what it reads (exception,
error_information, error_str) instead of a bare dict.

* fix(otel): put the error message on every failed span status

Covers the proxy SERVER span, management endpoint spans, service spans, and Arize Phoenix spans, so no ERROR span ships with an empty status description

* fix(otel): cut the error status to the span attribute length limit and describe dangling server spans

The status description now uses the error text as stored on the span, so OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT applies to it too. A server span the proxy closes on a 4xx or 5xx outside the request path gets the same description, and an empty error message falls back to the error class

* test(otel): type the validation-handler status test

* test(otel): cover the Phoenix parent span and error_str length-limit status

---------

Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
Max Azatian 2026-10-20 08:35:47 +02:00 • committed by GitHub
parent f03f98ac22
commit f43f2d8c15
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 343 additions and 55 deletions

View file

@ -354,17 +354,16 @@ class ArizePhoenixLogger(OpenTelemetry):
_project_name, tracer = self._resolve_tracer_for_kwargs(kwargs)
ctx, parent_span = self._get_phoenix_context(kwargs, tracer=tracer)
status: Final = Status(StatusCode.OK if success else StatusCode.ERROR)
status_code: Final = StatusCode.OK if success else StatusCode.ERROR
span: Final = tracer.start_span(
name=self._get_span_name(kwargs),
start_time=self._to_ns(start_time),
context=ctx,
)
span.set_status(status)
self.set_attributes(span, kwargs, response_obj)
if not success:
self._record_exception_on_span(span=span, kwargs=kwargs)
span_error_message: Final = None if success else self._record_exception_on_span(span=span, kwargs=kwargs)
span.set_status(Status(status_code, span_error_message))
if success:
self._maybe_log_raw_request(kwargs, response_obj, start_time, end_time, span)
@ -373,10 +372,11 @@ class ArizePhoenixLogger(OpenTelemetry):
self._create_guardrail_span(kwargs=kwargs, context=ctx)
if parent_span is not None:
parent_span.set_status(status)
self.set_attributes(parent_span, kwargs, response_obj)
if not success:
self._record_exception_on_span(span=parent_span, kwargs=kwargs)
parent_error_message: Final = (
None if success else self._record_exception_on_span(span=parent_span, kwargs=kwargs)
)
parent_span.set_status(Status(status_code, parent_error_message))
parent_span.end(end_time=self._to_ns(end_time))
if success:

View file

@ -53,6 +53,7 @@ from litellm.types.utils import (
StandardCallbackDynamicParams,
StandardLoggingMetadata,
StandardLoggingPayload,
StandardLoggingPayloadErrorInformation,
)
# OpenTelemetry imports moved to individual functions to avoid import errors when not installed
@ -104,8 +105,26 @@ class _ResponseWithUsageView(TypedDict, total=False):
usage: "_UsageCompletionTokensView | None"
class _FailurePayloadView(TypedDict, total=False):
error_information: ReadOnly[StandardLoggingPayloadErrorInformation | None]
error_str: ReadOnly[str | None]
class _FailureKwargsView(TypedDict, total=False):
exception: ReadOnly[BaseException | None]
standard_logging_object: ReadOnly[_FailurePayloadView | None]
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
def _text_as_stored_on_span(span: Span, key: str, text: str) -> str:
from opentelemetry.sdk.trace import ReadableSpan
stored: Final = span.attributes.get(key) if isinstance(span, ReadableSpan) and span.attributes else None
return stored if isinstance(stored, str) else text
# Cap on credential-scoped providers held at once; each one owns an exporter thread.
_MAX_DYNAMIC_TRACER_PROVIDERS: Final = 256
@ -922,7 +941,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
value=value,
)
service_logging_span.set_status(Status(StatusCode.ERROR))
service_logging_span.set_status(
Status(
StatusCode.ERROR, _text_as_stored_on_span(service_logging_span, "error", error) if error else None
)
)
service_logging_span.end(end_time=_end_time_ns)
async def async_post_call_failure_hook(
@ -937,8 +960,6 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
parent_otel_span: Final = user_api_key_dict.parent_otel_span
if parent_otel_span is not None:
parent_otel_span.set_status(Status(StatusCode.ERROR))
# Stamp team attributes onto the SERVER (root) span too, so the
# trace root is team-filterable on the failure path like the
# child exception span below.
@ -960,13 +981,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
original_exception=original_exception,
traceback_str=traceback_str,
)
self._record_exception_on_span(
server_error_message: Final = self._record_exception_on_span(
span=parent_otel_span,
kwargs={
"exception": original_exception,
"standard_logging_object": {"error_information": error_information},
},
)
parent_otel_span.set_status(Status(StatusCode.ERROR, server_error_message))
# _record_exception_on_span only stamps when error_code is set;
# bare TypeError etc. has none, and the span is about to be ended.
@ -995,7 +1017,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
team_id=user_api_key_dict.team_id,
team_alias=user_api_key_dict.team_alias,
)
exception_logging_span.set_status(Status(StatusCode.ERROR))
exception_logging_span.set_status(Status(StatusCode.ERROR, server_error_message))
exception_logging_span.end(end_time=self._to_ns(datetime.now()))
# Emit guardrail spans for any guardrail invocations that
@ -2199,11 +2221,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
if self._gen_ai_semconv_latest_experimental:
span_kwargs["kind"] = self.span_kind.CLIENT
span = otel_tracer.start_span(**span_kwargs)
span.set_status(Status(StatusCode.ERROR))
self.set_attributes(span, kwargs, response_obj)
# Record exception information using OTEL standard method
self._record_exception_on_span(span=span, kwargs=kwargs)
span_error_message: Final = self._record_exception_on_span(span=span, kwargs=kwargs)
span.set_status(Status(StatusCode.ERROR, span_error_message))
span.end(end_time=self._to_ns(end_time))
else:
@ -2212,9 +2234,9 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
# Only set attributes if the span is still recording (not closed)
# Note: parent_otel_span is guaranteed to be not None here
if parent_otel_span.is_recording():
parent_otel_span.set_status(Status(StatusCode.ERROR))
self.set_attributes(parent_otel_span, kwargs, response_obj)
self._record_exception_on_span(span=parent_otel_span, kwargs=kwargs)
parent_error_message: Final = self._record_exception_on_span(span=parent_otel_span, kwargs=kwargs)
parent_otel_span.set_status(Status(StatusCode.ERROR, parent_error_message))
# Create span for guardrail information — ensure proper parenting (Issue #5)
guardrail_ctx: Final = self._resolve_guardrail_context(
@ -2232,13 +2254,15 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
):
parent_otel_span.end(end_time=self._to_ns(end_time))
def _record_exception_on_span(self, span: Span, kwargs: dict):
def _record_exception_on_span(self, span: Span, kwargs: _FailureKwargsView) -> str | None:
"""
Record exception information on the span using OTEL standard methods.
This extracts error information from StandardLoggingPayload and:
1. Uses span.record_exception() for the actual exception object (OTEL standard)
2. Sets structured error attributes from StandardLoggingPayloadErrorInformation
Returns the error message it recorded, if any, for the span status description.
"""
try:
from litellm.integrations._types.open_inference import (
@ -2253,10 +2277,10 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
span.record_exception(exception)
# Get StandardLoggingPayload for structured error information
standard_logging_payload: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object")
standard_logging_payload: Final = kwargs.get("standard_logging_object")
if standard_logging_payload is None:
return
return None
# Extract error_information from StandardLoggingPayload
error_information: Final = standard_logging_payload.get("error_information")
@ -2270,59 +2294,67 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
key=ErrorAttributes.ERROR_MESSAGE,
value=error_str,
)
return
return _text_as_stored_on_span(span, ErrorAttributes.ERROR_MESSAGE, error_str) if error_str else None
# Set structured error attributes from StandardLoggingPayloadErrorInformation
if error_information.get("error_code"):
error_code: Final = error_information.get("error_code")
if error_code:
self.safe_set_attribute(
span=span,
key=ErrorAttributes.ERROR_CODE,
value=error_information["error_code"],
value=error_code,
)
# Also expose under the OTel-standard name as an int
# (error_code is a str, may be non-numeric).
_error_code_val: Final = error_information["error_code"]
if _error_code_val is not None:
try:
self.safe_set_attribute(
span=span,
key=HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE,
value=int(_error_code_val),
)
except (ValueError, TypeError):
pass
try:
self.safe_set_attribute(
span=span,
key=HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE,
value=int(error_code),
)
except (ValueError, TypeError):
pass
if error_information.get("error_class"):
error_class: Final = error_information.get("error_class")
if error_class:
self.safe_set_attribute(
span=span,
key=ErrorAttributes.ERROR_TYPE,
value=error_information["error_class"],
value=error_class,
)
if error_information.get("error_message"):
error_message: Final = error_information.get("error_message")
if error_message:
self.safe_set_attribute(
span=span,
key=ErrorAttributes.ERROR_MESSAGE,
value=error_information["error_message"],
value=error_message,
)
if error_information.get("llm_provider"):
llm_provider: Final = error_information.get("llm_provider")
if llm_provider:
self.safe_set_attribute(
span=span,
key=ErrorAttributes.ERROR_LLM_PROVIDER,
value=error_information["llm_provider"],
value=llm_provider,
)
if error_information.get("traceback"):
traceback_text: Final = error_information.get("traceback")
if traceback_text:
self.safe_set_attribute(
span=span,
key=ErrorAttributes.ERROR_STACK_TRACE,
value=error_information["traceback"],
value=traceback_text,
)
except Exception as e:
verbose_logger.exception("OpenTelemetry: Error recording exception on span: %s", str(e))
return None
else:
if error_message:
return _text_as_stored_on_span(span, ErrorAttributes.ERROR_MESSAGE, error_message)
return error_class or None
def set_tools_attributes(self, span: Span, tools):
import json
@ -3518,8 +3550,6 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
key="exception",
value=str(_exception),
)
management_endpoint_span.set_status(Status(StatusCode.ERROR))
management_endpoint_span.end(end_time=_end_time_ns)
# The management wrapper has no other hook that closes the SERVER span.
from litellm.litellm_core_utils.litellm_logging import (
@ -3529,14 +3559,16 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
error_information: Final = StandardLoggingPayloadSetup.get_error_information(
original_exception=_exception,
)
parent_otel_span.set_status(Status(StatusCode.ERROR))
self._record_exception_on_span(
server_error_message: Final = self._record_exception_on_span(
span=parent_otel_span,
kwargs={
"exception": _exception,
"standard_logging_object": {"error_information": error_information},
},
)
management_endpoint_span.set_status(Status(StatusCode.ERROR, server_error_message))
management_endpoint_span.end(end_time=_end_time_ns)
parent_otel_span.set_status(Status(StatusCode.ERROR, server_error_message))
parent_otel_span.end(end_time=_end_time_ns)
def create_litellm_proxy_request_started_span(
@ -3596,19 +3628,20 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
span: Span | None,
exception: Exception | None,
status_code: int,
) -> None:
) -> str | None:
"""Stamp structured ``error.*`` attributes on the SERVER span from the
exception returned to the client, with ``error.code`` pinned to the real
response status. Idempotent (overwrites); emits no exception event."""
response status. Idempotent (overwrites); emits no exception event.
Returns the error text for the span status description."""
if span is None or exception is None:
return
return None
from litellm.litellm_core_utils.litellm_logging import (
StandardLoggingPayloadSetup,
)
error_information: Final = StandardLoggingPayloadSetup.get_error_information(original_exception=exception)
error_information["error_code"] = str(status_code)
self._record_exception_on_span(
return self._record_exception_on_span(
span=span,
kwargs={"standard_logging_object": {"error_information": error_information}},
)

View file

@ -2240,9 +2240,14 @@ def _close_dangling_otel_server_span(request: Request, status_code: int, exc: Ex
open_telemetry_logger.record_error_attributes_on_span(parent_otel_span, exc, status_code)
return
open_telemetry_logger.set_response_status_code_attribute(parent_otel_span, status_code)
if status_code >= 400:
open_telemetry_logger.record_error_attributes_on_span(parent_otel_span, exc, status_code)
parent_otel_span.set_status(Status(StatusCode.ERROR if status_code >= 400 else StatusCode.OK))
parent_otel_span.set_status(
Status(
StatusCode.ERROR,
open_telemetry_logger.record_error_attributes_on_span(parent_otel_span, exc, status_code),
)
if status_code >= 400
else Status(StatusCode.OK)
)
parent_otel_span.end()
except Exception as e:
verbose_proxy_logger.debug("Error closing dangling OTEL SERVER span: %s", str(e))

View file

@ -2,7 +2,15 @@ import asyncio, importlib, litellm, logging, os, unittest
from unittest.mock import MagicMock, patch
import pytest
from datetime import datetime, timedelta, timezone
from typing import Final
from opentelemetry.sdk.trace import SpanLimits, TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
from opentelemetry.trace import StatusCode
from litellm.integrations.opentelemetry import LITELLM_REQUEST_SPAN_NAME
from litellm.integrations.arize.arize_phoenix import (
ArizePhoenixConfig,
ArizePhoenixLogger,
@ -852,6 +860,47 @@ class TestPhoenixTraceHandling:
assert span.resource.attributes.get("model_id") == "unified-proj"
@pytest.mark.parametrize(("max_attribute_length", "expected_description"), [(None, "model not found"), (5, "model")])
def test_failed_proxy_request_puts_the_error_message_on_every_phoenix_span_status(
max_attribute_length: int | None, expected_description: str
) -> None:
exporter: Final = InMemorySpanExporter()
provider: Final = TracerProvider(span_limits=SpanLimits(max_attribute_length=max_attribute_length))
provider.add_span_processor(SimpleSpanProcessor(exporter))
logger: Final = ArizePhoenixLogger(callback_name="arize_phoenix", tracer_provider=provider)
start: Final = datetime(2026, 1, 1, tzinfo=timezone.utc)
logger._handle_failure(
{
"model": "gpt-4",
"messages": [{"role": "user", "content": "Hello"}],
"optional_params": {},
"litellm_params": {
"custom_llm_provider": "openai",
"proxy_server_request": {"url": "/chat/completions", "method": "POST", "headers": {}},
},
"standard_logging_object": {
"id": "test-id",
"call_type": "completion",
"metadata": {},
"error_information": {"error_class": "NotFoundError", "error_message": "model not found"},
},
"exception": Exception("model not found"),
},
None,
start,
start + timedelta(seconds=1),
)
statuses: Final = {
span.name: (span.status.status_code, span.status.description) for span in exporter.get_finished_spans()
}
assert statuses == {
"litellm_proxy_request": (StatusCode.ERROR, expected_description),
LITELLM_REQUEST_SPAN_NAME: (StatusCode.ERROR, expected_description),
}
class TestGetArizePhoenixConfigProjectName:
@patch.dict(
"os.environ", {"PHOENIX_PROJECT_NAME": "phoenix-config-proj"}, clear=True

View file

@ -3,10 +3,13 @@ that fail after auth but before the route handler runs (e.g. /model/new
TypeError or RequestValidationError)."""
import asyncio
from collections.abc import Callable
from typing import Final
import pytest
from fastapi import HTTPException, Request
from fastapi.exceptions import RequestValidationError
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
import litellm.proxy.proxy_server as proxy_server_module
from litellm.proxy._types import ProxyException
@ -133,10 +136,30 @@ def test_openai_exception_handler_stamps_structured_error_on_span(
expected_url_path=path,
where=f"openai_exception_handler ({path})",
)
attrs = get_server_span(wired_otel).attributes
server_span = get_server_span(wired_otel)
attrs = server_span.attributes
assert attrs.get(ErrorAttributes.ERROR_MESSAGE) == msg
assert attrs.get(ErrorAttributes.ERROR_TYPE) == "ProxyException"
assert attrs.get(ErrorAttributes.ERROR_CODE) == "401"
assert server_span.status.description == msg
def test_validation_handler_puts_the_error_message_on_the_span_status(
wired_otel: InMemorySpanExporter, server_span_factory: Callable[[str], object]
) -> None:
path: Final = "/team/new"
request: Final = _fake_request(parent_otel_span=server_span_factory(path), path=path)
exc: Final = RequestValidationError(
errors=[{"type": "float_parsing", "loc": ("body", "max_budget"), "msg": "Input should be a valid number"}]
)
response: Final = asyncio.run(otel_request_validation_exception_handler(request, exc))
assert response.status_code == 422
server_span: Final = get_server_span(wired_otel)
description: Final = server_span.status.description or ""
assert "Input should be a valid number" in description
assert description == server_span.attributes.get(ErrorAttributes.ERROR_MESSAGE)
def test_unhandled_handler_reraises_known_exceptions(wired_otel, server_span_factory):

View file

@ -25,7 +25,7 @@ from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider
from opentelemetry.sdk._logs.export import InMemoryLogExporter, LogExportResult, SimpleLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import InMemoryMetricReader, MetricsData
from opentelemetry.sdk.trace import ReadableSpan, TracerProvider
from opentelemetry.sdk.trace import ReadableSpan, SpanLimits, TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExportResult
from opentelemetry.sdk.util.instrumentation import InstrumentationScope
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
@ -48,9 +48,10 @@ from litellm.integrations.opentelemetry import (
)
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.proxy import proxy_server
from litellm.proxy._types import SpanAttributes
from litellm.proxy._types import ManagementEndpointLoggingPayload, ProxyException, SpanAttributes
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
from litellm.types.services import ServiceLoggerPayload, ServiceTypes
from collections.abc import AsyncIterator
from collections.abc import AsyncIterator, Callable, Mapping
from litellm.constants import KUBERNETES_POD_ROUTING_KEY, LOGGING_WORKER_MAX_TIME_PER_COROUTINE
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.types.utils import ChatCompletionMessageToolCall, Function
@ -3214,6 +3215,9 @@ class TestOpenTelemetryExternalSpan(unittest.TestCase):
)
_FIXED_START: Final = datetime(2026, 1, 1, tzinfo=timezone.utc)
class TestOpenTelemetrySemanticConventions138(unittest.TestCase):
"""
Test suite for OpenTelemetry 1.38 Semantic Conventions compliance.
@ -3731,6 +3735,91 @@ class TestOpenTelemetrySemanticConventions138(unittest.TestCase):
error_spans = [s for s in spans if s.status.status_code == StatusCode.ERROR]
self.assertTrue(error_spans, "Expected at least one span with ERROR status")
@parameterized.expand(
[
(
"own_span_error_information",
False,
{
"error_str": "litellm.ServiceUnavailableError: upstream returned 503",
"error_information": {
"error_class": "ServiceUnavailableError",
"error_message": "upstream returned 503",
},
},
"upstream returned 503",
),
(
"own_span_error_str_only",
False,
{"error_str": "litellm.ServiceUnavailableError: upstream returned 503"},
"litellm.ServiceUnavailableError: upstream returned 503",
),
(
"own_span_error_str_only_cut_to_the_attribute_length_limit",
False,
{"error_str": "litellm.ServiceUnavailableError: upstream returned 503"},
"litellm.Serv",
12,
),
(
"own_span_empty_error_message",
False,
{"error_information": {"error_class": "ServiceUnavailableError", "error_message": ""}},
"ServiceUnavailableError",
),
(
"parent_span_error_information",
True,
{
"error_information": {
"error_class": "ServiceUnavailableError",
"error_message": "upstream returned 503",
}
},
"upstream returned 503",
),
]
)
def test_handle_failure_error_status_carries_the_error_message(
self,
_name: str,
has_parent_span: bool,
error_fields: Mapping[str, object],
expected_description: str,
max_attribute_length: int | None = None,
) -> None:
span_exporter: Final = InMemorySpanExporter()
tracer_provider: Final = TracerProvider(span_limits=SpanLimits(max_attribute_length=max_attribute_length))
tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter))
otel: Final = OpenTelemetry(tracer_provider=tracer_provider)
otel.tracer = tracer_provider.get_tracer("litellm")
active_parent_span: Final = (
trace.use_span(tracer_provider.get_tracer("proxy").start_span("parent_span"), end_on_exit=True)
if has_parent_span
else contextlib.nullcontext()
)
start: Final = _FIXED_START
end: Final = start + timedelta(seconds=1)
kwargs: Final = {
"model": "gpt-4",
"messages": [{"role": "user", "content": "Hello"}],
"optional_params": {},
"litellm_params": {"custom_llm_provider": "openai"},
"standard_logging_object": {"id": "test-id", "call_type": "completion", "metadata": {}, **error_fields},
"exception": Exception("upstream returned 503"),
}
with patch.dict(os.environ, {"USE_OTEL_LITELLM_REQUEST_SPAN": "false"}), active_parent_span:
otel._handle_failure(kwargs, None, start, end)
error_spans: Final = [
s for s in span_exporter.get_finished_spans() if s.status.status_code == trace.StatusCode.ERROR
]
self.assertEqual(len(error_spans), 1)
self.assertEqual(error_spans[0].name, "parent_span" if has_parent_span else "litellm_request")
self.assertEqual(error_spans[0].status.description, expected_description)
class TestRawSpanAttributeIsolation(unittest.TestCase):
"""Issue #3: raw_gen_ai_request span should only contain provider-specific
@ -5439,6 +5528,95 @@ class TestOpenTelemetryFailureHookStampsServerSpan(unittest.TestCase):
)
_PROXY_KEY_ERROR: Final = "Authentication Error, invalid proxy key"
def _proxy_key_exception() -> ProxyException:
return ProxyException(message=_PROXY_KEY_ERROR, type="auth_error", param="None", code=401)
def _fail_proxy_request(provider: TracerProvider) -> None:
otel: Final = OpenTelemetry(tracer_provider=provider)
asyncio.run(
otel.async_post_call_failure_hook(
request_data={},
original_exception=_proxy_key_exception(),
user_api_key_dict=UserAPIKeyAuth(parent_otel_span=otel.tracer.start_span("Received Proxy Server Request")),
)
)
def _fail_management_request(provider: TracerProvider) -> None:
otel: Final = OpenTelemetry(tracer_provider=provider)
now: Final = _FIXED_START
asyncio.run(
otel.async_management_endpoint_failure_hook(
logging_payload=ManagementEndpointLoggingPayload(
route="/key/generate", request_data={}, exception=_proxy_key_exception(), start_time=now, end_time=now
),
parent_otel_span=otel.tracer.start_span("Received Proxy Server Request"),
)
)
def _fail_service_call(provider: TracerProvider) -> None:
otel: Final = OpenTelemetry(tracer_provider=provider)
now: Final = _FIXED_START
asyncio.run(
otel.async_service_failure_hook(
payload=ServiceLoggerPayload(
is_error=True,
error="redis connection refused",
service=ServiceTypes.REDIS,
duration=0.1,
call_type="async_get_cache",
event_metadata=None,
),
error="redis connection refused",
parent_otel_span=otel.tracer.start_span("Received Proxy Server Request"),
start_time=now,
end_time=now,
)
)
_FAILED_SPANS: Final = pytest.mark.parametrize(
("fail", "expected_description"),
[
(_fail_proxy_request, _PROXY_KEY_ERROR),
(_fail_management_request, _PROXY_KEY_ERROR),
(_fail_service_call, "redis connection refused"),
],
ids=["proxy_request", "management_request", "service_call"],
)
def _error_status_descriptions(fail: Callable[[TracerProvider], None], provider: TracerProvider) -> set[str | None]:
exporter: Final = InMemorySpanExporter()
provider.add_span_processor(SimpleSpanProcessor(exporter))
fail(provider)
error_spans: Final = [s for s in exporter.get_finished_spans() if s.status.status_code == trace.StatusCode.ERROR]
assert error_spans
return {s.status.description for s in error_spans}
@_FAILED_SPANS
def test_every_error_span_carries_the_error_message_in_its_status(
fail: Callable[[TracerProvider], None], expected_description: str
) -> None:
assert _error_status_descriptions(fail, TracerProvider()) == {expected_description}
@_FAILED_SPANS
def test_error_status_is_cut_to_the_span_attribute_length_limit(
fail: Callable[[TracerProvider], None], expected_description: str
) -> None:
limit: Final = 12
provider: Final = TracerProvider(span_limits=SpanLimits(max_attribute_length=limit))
assert _error_status_descriptions(fail, provider) == {expected_description[:limit]}
class TestOpenTelemetrySetProxyRequestRouteAttributes(unittest.TestCase):
"""http.route (template) + url.path (literal) must land on the SERVER
span. The logging handlers write the litellm_request child span, so