diff --git a/litellm/integrations/arize/arize_phoenix.py b/litellm/integrations/arize/arize_phoenix.py index ce585ad26c4..150d814f811 100644 --- a/litellm/integrations/arize/arize_phoenix.py +++ b/litellm/integrations/arize/arize_phoenix.py @@ -354,17 +354,16 @@ class ArizePhoenixLogger(OpenTelemetry): _project_name, tracer = self._resolve_tracer_for_kwargs(kwargs) ctx, parent_span = self._get_phoenix_context(kwargs, tracer=tracer) - status: Final = Status(StatusCode.OK if success else StatusCode.ERROR) + status_code: Final = StatusCode.OK if success else StatusCode.ERROR span: Final = tracer.start_span( name=self._get_span_name(kwargs), start_time=self._to_ns(start_time), context=ctx, ) - span.set_status(status) self.set_attributes(span, kwargs, response_obj) - if not success: - self._record_exception_on_span(span=span, kwargs=kwargs) + span_error_message: Final = None if success else self._record_exception_on_span(span=span, kwargs=kwargs) + span.set_status(Status(status_code, span_error_message)) if success: self._maybe_log_raw_request(kwargs, response_obj, start_time, end_time, span) @@ -373,10 +372,11 @@ class ArizePhoenixLogger(OpenTelemetry): self._create_guardrail_span(kwargs=kwargs, context=ctx) if parent_span is not None: - parent_span.set_status(status) self.set_attributes(parent_span, kwargs, response_obj) - if not success: - self._record_exception_on_span(span=parent_span, kwargs=kwargs) + parent_error_message: Final = ( + None if success else self._record_exception_on_span(span=parent_span, kwargs=kwargs) + ) + parent_span.set_status(Status(status_code, parent_error_message)) parent_span.end(end_time=self._to_ns(end_time)) if success: diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 90392d9a45d..0d0e309034a 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -53,6 +53,7 @@ from litellm.types.utils import ( StandardCallbackDynamicParams, StandardLoggingMetadata, StandardLoggingPayload, + StandardLoggingPayloadErrorInformation, ) # OpenTelemetry imports moved to individual functions to avoid import errors when not installed @@ -104,8 +105,26 @@ class _ResponseWithUsageView(TypedDict, total=False): usage: "_UsageCompletionTokensView | None" +class _FailurePayloadView(TypedDict, total=False): + error_information: ReadOnly[StandardLoggingPayloadErrorInformation | None] + error_str: ReadOnly[str | None] + + +class _FailureKwargsView(TypedDict, total=False): + exception: ReadOnly[BaseException | None] + standard_logging_object: ReadOnly[_FailurePayloadView | None] + + _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) + +def _text_as_stored_on_span(span: Span, key: str, text: str) -> str: + from opentelemetry.sdk.trace import ReadableSpan + + stored: Final = span.attributes.get(key) if isinstance(span, ReadableSpan) and span.attributes else None + return stored if isinstance(stored, str) else text + + # Cap on credential-scoped providers held at once; each one owns an exporter thread. _MAX_DYNAMIC_TRACER_PROVIDERS: Final = 256 @@ -922,7 +941,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): value=value, ) - service_logging_span.set_status(Status(StatusCode.ERROR)) + service_logging_span.set_status( + Status( + StatusCode.ERROR, _text_as_stored_on_span(service_logging_span, "error", error) if error else None + ) + ) service_logging_span.end(end_time=_end_time_ns) async def async_post_call_failure_hook( @@ -937,8 +960,6 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): parent_otel_span: Final = user_api_key_dict.parent_otel_span if parent_otel_span is not None: - parent_otel_span.set_status(Status(StatusCode.ERROR)) - # Stamp team attributes onto the SERVER (root) span too, so the # trace root is team-filterable on the failure path like the # child exception span below. @@ -960,13 +981,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): original_exception=original_exception, traceback_str=traceback_str, ) - self._record_exception_on_span( + server_error_message: Final = self._record_exception_on_span( span=parent_otel_span, kwargs={ "exception": original_exception, "standard_logging_object": {"error_information": error_information}, }, ) + parent_otel_span.set_status(Status(StatusCode.ERROR, server_error_message)) # _record_exception_on_span only stamps when error_code is set; # bare TypeError etc. has none, and the span is about to be ended. @@ -995,7 +1017,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): team_id=user_api_key_dict.team_id, team_alias=user_api_key_dict.team_alias, ) - exception_logging_span.set_status(Status(StatusCode.ERROR)) + exception_logging_span.set_status(Status(StatusCode.ERROR, server_error_message)) exception_logging_span.end(end_time=self._to_ns(datetime.now())) # Emit guardrail spans for any guardrail invocations that @@ -2199,11 +2221,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): if self._gen_ai_semconv_latest_experimental: span_kwargs["kind"] = self.span_kind.CLIENT span = otel_tracer.start_span(**span_kwargs) - span.set_status(Status(StatusCode.ERROR)) self.set_attributes(span, kwargs, response_obj) # Record exception information using OTEL standard method - self._record_exception_on_span(span=span, kwargs=kwargs) + span_error_message: Final = self._record_exception_on_span(span=span, kwargs=kwargs) + span.set_status(Status(StatusCode.ERROR, span_error_message)) span.end(end_time=self._to_ns(end_time)) else: @@ -2212,9 +2234,9 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): # Only set attributes if the span is still recording (not closed) # Note: parent_otel_span is guaranteed to be not None here if parent_otel_span.is_recording(): - parent_otel_span.set_status(Status(StatusCode.ERROR)) self.set_attributes(parent_otel_span, kwargs, response_obj) - self._record_exception_on_span(span=parent_otel_span, kwargs=kwargs) + parent_error_message: Final = self._record_exception_on_span(span=parent_otel_span, kwargs=kwargs) + parent_otel_span.set_status(Status(StatusCode.ERROR, parent_error_message)) # Create span for guardrail information — ensure proper parenting (Issue #5) guardrail_ctx: Final = self._resolve_guardrail_context( @@ -2232,13 +2254,15 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): ): parent_otel_span.end(end_time=self._to_ns(end_time)) - def _record_exception_on_span(self, span: Span, kwargs: dict): + def _record_exception_on_span(self, span: Span, kwargs: _FailureKwargsView) -> str | None: """ Record exception information on the span using OTEL standard methods. This extracts error information from StandardLoggingPayload and: 1. Uses span.record_exception() for the actual exception object (OTEL standard) 2. Sets structured error attributes from StandardLoggingPayloadErrorInformation + + Returns the error message it recorded, if any, for the span status description. """ try: from litellm.integrations._types.open_inference import ( @@ -2253,10 +2277,10 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): span.record_exception(exception) # Get StandardLoggingPayload for structured error information - standard_logging_payload: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object") + standard_logging_payload: Final = kwargs.get("standard_logging_object") if standard_logging_payload is None: - return + return None # Extract error_information from StandardLoggingPayload error_information: Final = standard_logging_payload.get("error_information") @@ -2270,59 +2294,67 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): key=ErrorAttributes.ERROR_MESSAGE, value=error_str, ) - return + return _text_as_stored_on_span(span, ErrorAttributes.ERROR_MESSAGE, error_str) if error_str else None # Set structured error attributes from StandardLoggingPayloadErrorInformation - if error_information.get("error_code"): + error_code: Final = error_information.get("error_code") + if error_code: self.safe_set_attribute( span=span, key=ErrorAttributes.ERROR_CODE, - value=error_information["error_code"], + value=error_code, ) # Also expose under the OTel-standard name as an int # (error_code is a str, may be non-numeric). - _error_code_val: Final = error_information["error_code"] - if _error_code_val is not None: - try: - self.safe_set_attribute( - span=span, - key=HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE, - value=int(_error_code_val), - ) - except (ValueError, TypeError): - pass + try: + self.safe_set_attribute( + span=span, + key=HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE, + value=int(error_code), + ) + except (ValueError, TypeError): + pass - if error_information.get("error_class"): + error_class: Final = error_information.get("error_class") + if error_class: self.safe_set_attribute( span=span, key=ErrorAttributes.ERROR_TYPE, - value=error_information["error_class"], + value=error_class, ) - if error_information.get("error_message"): + error_message: Final = error_information.get("error_message") + if error_message: self.safe_set_attribute( span=span, key=ErrorAttributes.ERROR_MESSAGE, - value=error_information["error_message"], + value=error_message, ) - if error_information.get("llm_provider"): + llm_provider: Final = error_information.get("llm_provider") + if llm_provider: self.safe_set_attribute( span=span, key=ErrorAttributes.ERROR_LLM_PROVIDER, - value=error_information["llm_provider"], + value=llm_provider, ) - if error_information.get("traceback"): + traceback_text: Final = error_information.get("traceback") + if traceback_text: self.safe_set_attribute( span=span, key=ErrorAttributes.ERROR_STACK_TRACE, - value=error_information["traceback"], + value=traceback_text, ) except Exception as e: verbose_logger.exception("OpenTelemetry: Error recording exception on span: %s", str(e)) + return None + else: + if error_message: + return _text_as_stored_on_span(span, ErrorAttributes.ERROR_MESSAGE, error_message) + return error_class or None def set_tools_attributes(self, span: Span, tools): import json @@ -3518,8 +3550,6 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): key="exception", value=str(_exception), ) - management_endpoint_span.set_status(Status(StatusCode.ERROR)) - management_endpoint_span.end(end_time=_end_time_ns) # The management wrapper has no other hook that closes the SERVER span. from litellm.litellm_core_utils.litellm_logging import ( @@ -3529,14 +3559,16 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): error_information: Final = StandardLoggingPayloadSetup.get_error_information( original_exception=_exception, ) - parent_otel_span.set_status(Status(StatusCode.ERROR)) - self._record_exception_on_span( + server_error_message: Final = self._record_exception_on_span( span=parent_otel_span, kwargs={ "exception": _exception, "standard_logging_object": {"error_information": error_information}, }, ) + management_endpoint_span.set_status(Status(StatusCode.ERROR, server_error_message)) + management_endpoint_span.end(end_time=_end_time_ns) + parent_otel_span.set_status(Status(StatusCode.ERROR, server_error_message)) parent_otel_span.end(end_time=_end_time_ns) def create_litellm_proxy_request_started_span( @@ -3596,19 +3628,20 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): span: Span | None, exception: Exception | None, status_code: int, - ) -> None: + ) -> str | None: """Stamp structured ``error.*`` attributes on the SERVER span from the exception returned to the client, with ``error.code`` pinned to the real - response status. Idempotent (overwrites); emits no exception event.""" + response status. Idempotent (overwrites); emits no exception event. + Returns the error text for the span status description.""" if span is None or exception is None: - return + return None from litellm.litellm_core_utils.litellm_logging import ( StandardLoggingPayloadSetup, ) error_information: Final = StandardLoggingPayloadSetup.get_error_information(original_exception=exception) error_information["error_code"] = str(status_code) - self._record_exception_on_span( + return self._record_exception_on_span( span=span, kwargs={"standard_logging_object": {"error_information": error_information}}, ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 341f12ba720..295f4380d7d 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2240,9 +2240,14 @@ def _close_dangling_otel_server_span(request: Request, status_code: int, exc: Ex open_telemetry_logger.record_error_attributes_on_span(parent_otel_span, exc, status_code) return open_telemetry_logger.set_response_status_code_attribute(parent_otel_span, status_code) - if status_code >= 400: - open_telemetry_logger.record_error_attributes_on_span(parent_otel_span, exc, status_code) - parent_otel_span.set_status(Status(StatusCode.ERROR if status_code >= 400 else StatusCode.OK)) + parent_otel_span.set_status( + Status( + StatusCode.ERROR, + open_telemetry_logger.record_error_attributes_on_span(parent_otel_span, exc, status_code), + ) + if status_code >= 400 + else Status(StatusCode.OK) + ) parent_otel_span.end() except Exception as e: verbose_proxy_logger.debug("Error closing dangling OTEL SERVER span: %s", str(e)) diff --git a/tests/unit/integrations/arize/test_arize_phoenix.py b/tests/unit/integrations/arize/test_arize_phoenix.py index ca32a9cf9e3..eb9e1326ced 100644 --- a/tests/unit/integrations/arize/test_arize_phoenix.py +++ b/tests/unit/integrations/arize/test_arize_phoenix.py @@ -2,7 +2,15 @@ import asyncio, importlib, litellm, logging, os, unittest from unittest.mock import MagicMock, patch import pytest +from datetime import datetime, timedelta, timezone +from typing import Final +from opentelemetry.sdk.trace import SpanLimits, TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode + +from litellm.integrations.opentelemetry import LITELLM_REQUEST_SPAN_NAME from litellm.integrations.arize.arize_phoenix import ( ArizePhoenixConfig, ArizePhoenixLogger, @@ -852,6 +860,47 @@ class TestPhoenixTraceHandling: assert span.resource.attributes.get("model_id") == "unified-proj" +@pytest.mark.parametrize(("max_attribute_length", "expected_description"), [(None, "model not found"), (5, "model")]) +def test_failed_proxy_request_puts_the_error_message_on_every_phoenix_span_status( + max_attribute_length: int | None, expected_description: str +) -> None: + exporter: Final = InMemorySpanExporter() + provider: Final = TracerProvider(span_limits=SpanLimits(max_attribute_length=max_attribute_length)) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + logger: Final = ArizePhoenixLogger(callback_name="arize_phoenix", tracer_provider=provider) + start: Final = datetime(2026, 1, 1, tzinfo=timezone.utc) + + logger._handle_failure( + { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}], + "optional_params": {}, + "litellm_params": { + "custom_llm_provider": "openai", + "proxy_server_request": {"url": "/chat/completions", "method": "POST", "headers": {}}, + }, + "standard_logging_object": { + "id": "test-id", + "call_type": "completion", + "metadata": {}, + "error_information": {"error_class": "NotFoundError", "error_message": "model not found"}, + }, + "exception": Exception("model not found"), + }, + None, + start, + start + timedelta(seconds=1), + ) + + statuses: Final = { + span.name: (span.status.status_code, span.status.description) for span in exporter.get_finished_spans() + } + assert statuses == { + "litellm_proxy_request": (StatusCode.ERROR, expected_description), + LITELLM_REQUEST_SPAN_NAME: (StatusCode.ERROR, expected_description), + } + + class TestGetArizePhoenixConfigProjectName: @patch.dict( "os.environ", {"PHOENIX_PROJECT_NAME": "phoenix-config-proj"}, clear=True diff --git a/tests/unit/integrations/open_telemetry/test_otel_exception_handler.py b/tests/unit/integrations/open_telemetry/test_otel_exception_handler.py index 56d067b16cf..08e7bda1350 100644 --- a/tests/unit/integrations/open_telemetry/test_otel_exception_handler.py +++ b/tests/unit/integrations/open_telemetry/test_otel_exception_handler.py @@ -3,10 +3,13 @@ that fail after auth but before the route handler runs (e.g. /model/new TypeError or RequestValidationError).""" import asyncio +from collections.abc import Callable +from typing import Final import pytest from fastapi import HTTPException, Request from fastapi.exceptions import RequestValidationError +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter import litellm.proxy.proxy_server as proxy_server_module from litellm.proxy._types import ProxyException @@ -133,10 +136,30 @@ def test_openai_exception_handler_stamps_structured_error_on_span( expected_url_path=path, where=f"openai_exception_handler ({path})", ) - attrs = get_server_span(wired_otel).attributes + server_span = get_server_span(wired_otel) + attrs = server_span.attributes assert attrs.get(ErrorAttributes.ERROR_MESSAGE) == msg assert attrs.get(ErrorAttributes.ERROR_TYPE) == "ProxyException" assert attrs.get(ErrorAttributes.ERROR_CODE) == "401" + assert server_span.status.description == msg + + +def test_validation_handler_puts_the_error_message_on_the_span_status( + wired_otel: InMemorySpanExporter, server_span_factory: Callable[[str], object] +) -> None: + path: Final = "/team/new" + request: Final = _fake_request(parent_otel_span=server_span_factory(path), path=path) + exc: Final = RequestValidationError( + errors=[{"type": "float_parsing", "loc": ("body", "max_budget"), "msg": "Input should be a valid number"}] + ) + + response: Final = asyncio.run(otel_request_validation_exception_handler(request, exc)) + + assert response.status_code == 422 + server_span: Final = get_server_span(wired_otel) + description: Final = server_span.status.description or "" + assert "Input should be a valid number" in description + assert description == server_span.attributes.get(ErrorAttributes.ERROR_MESSAGE) def test_unhandled_handler_reraises_known_exceptions(wired_otel, server_span_factory): diff --git a/tests/unit/integrations/test_opentelemetry.py b/tests/unit/integrations/test_opentelemetry.py index 9cc06ccaafc..d14703e48a1 100644 --- a/tests/unit/integrations/test_opentelemetry.py +++ b/tests/unit/integrations/test_opentelemetry.py @@ -25,7 +25,7 @@ from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider from opentelemetry.sdk._logs.export import InMemoryLogExporter, LogExportResult, SimpleLogRecordProcessor from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import InMemoryMetricReader, MetricsData -from opentelemetry.sdk.trace import ReadableSpan, TracerProvider +from opentelemetry.sdk.trace import ReadableSpan, SpanLimits, TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExportResult from opentelemetry.sdk.util.instrumentation import InstrumentationScope from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter @@ -48,9 +48,10 @@ from litellm.integrations.opentelemetry import ( ) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy import proxy_server -from litellm.proxy._types import SpanAttributes +from litellm.proxy._types import ManagementEndpointLoggingPayload, ProxyException, SpanAttributes +from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth from litellm.types.services import ServiceLoggerPayload, ServiceTypes -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Callable, Mapping from litellm.constants import KUBERNETES_POD_ROUTING_KEY, LOGGING_WORKER_MAX_TIME_PER_COROUTINE from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.types.utils import ChatCompletionMessageToolCall, Function @@ -3214,6 +3215,9 @@ class TestOpenTelemetryExternalSpan(unittest.TestCase): ) +_FIXED_START: Final = datetime(2026, 1, 1, tzinfo=timezone.utc) + + class TestOpenTelemetrySemanticConventions138(unittest.TestCase): """ Test suite for OpenTelemetry 1.38 Semantic Conventions compliance. @@ -3731,6 +3735,91 @@ class TestOpenTelemetrySemanticConventions138(unittest.TestCase): error_spans = [s for s in spans if s.status.status_code == StatusCode.ERROR] self.assertTrue(error_spans, "Expected at least one span with ERROR status") + @parameterized.expand( + [ + ( + "own_span_error_information", + False, + { + "error_str": "litellm.ServiceUnavailableError: upstream returned 503", + "error_information": { + "error_class": "ServiceUnavailableError", + "error_message": "upstream returned 503", + }, + }, + "upstream returned 503", + ), + ( + "own_span_error_str_only", + False, + {"error_str": "litellm.ServiceUnavailableError: upstream returned 503"}, + "litellm.ServiceUnavailableError: upstream returned 503", + ), + ( + "own_span_error_str_only_cut_to_the_attribute_length_limit", + False, + {"error_str": "litellm.ServiceUnavailableError: upstream returned 503"}, + "litellm.Serv", + 12, + ), + ( + "own_span_empty_error_message", + False, + {"error_information": {"error_class": "ServiceUnavailableError", "error_message": ""}}, + "ServiceUnavailableError", + ), + ( + "parent_span_error_information", + True, + { + "error_information": { + "error_class": "ServiceUnavailableError", + "error_message": "upstream returned 503", + } + }, + "upstream returned 503", + ), + ] + ) + def test_handle_failure_error_status_carries_the_error_message( + self, + _name: str, + has_parent_span: bool, + error_fields: Mapping[str, object], + expected_description: str, + max_attribute_length: int | None = None, + ) -> None: + span_exporter: Final = InMemorySpanExporter() + tracer_provider: Final = TracerProvider(span_limits=SpanLimits(max_attribute_length=max_attribute_length)) + tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter)) + otel: Final = OpenTelemetry(tracer_provider=tracer_provider) + otel.tracer = tracer_provider.get_tracer("litellm") + active_parent_span: Final = ( + trace.use_span(tracer_provider.get_tracer("proxy").start_span("parent_span"), end_on_exit=True) + if has_parent_span + else contextlib.nullcontext() + ) + start: Final = _FIXED_START + end: Final = start + timedelta(seconds=1) + kwargs: Final = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}], + "optional_params": {}, + "litellm_params": {"custom_llm_provider": "openai"}, + "standard_logging_object": {"id": "test-id", "call_type": "completion", "metadata": {}, **error_fields}, + "exception": Exception("upstream returned 503"), + } + + with patch.dict(os.environ, {"USE_OTEL_LITELLM_REQUEST_SPAN": "false"}), active_parent_span: + otel._handle_failure(kwargs, None, start, end) + + error_spans: Final = [ + s for s in span_exporter.get_finished_spans() if s.status.status_code == trace.StatusCode.ERROR + ] + self.assertEqual(len(error_spans), 1) + self.assertEqual(error_spans[0].name, "parent_span" if has_parent_span else "litellm_request") + self.assertEqual(error_spans[0].status.description, expected_description) + class TestRawSpanAttributeIsolation(unittest.TestCase): """Issue #3: raw_gen_ai_request span should only contain provider-specific @@ -5439,6 +5528,95 @@ class TestOpenTelemetryFailureHookStampsServerSpan(unittest.TestCase): ) +_PROXY_KEY_ERROR: Final = "Authentication Error, invalid proxy key" + + +def _proxy_key_exception() -> ProxyException: + return ProxyException(message=_PROXY_KEY_ERROR, type="auth_error", param="None", code=401) + + +def _fail_proxy_request(provider: TracerProvider) -> None: + otel: Final = OpenTelemetry(tracer_provider=provider) + asyncio.run( + otel.async_post_call_failure_hook( + request_data={}, + original_exception=_proxy_key_exception(), + user_api_key_dict=UserAPIKeyAuth(parent_otel_span=otel.tracer.start_span("Received Proxy Server Request")), + ) + ) + + +def _fail_management_request(provider: TracerProvider) -> None: + otel: Final = OpenTelemetry(tracer_provider=provider) + now: Final = _FIXED_START + asyncio.run( + otel.async_management_endpoint_failure_hook( + logging_payload=ManagementEndpointLoggingPayload( + route="/key/generate", request_data={}, exception=_proxy_key_exception(), start_time=now, end_time=now + ), + parent_otel_span=otel.tracer.start_span("Received Proxy Server Request"), + ) + ) + + +def _fail_service_call(provider: TracerProvider) -> None: + otel: Final = OpenTelemetry(tracer_provider=provider) + now: Final = _FIXED_START + asyncio.run( + otel.async_service_failure_hook( + payload=ServiceLoggerPayload( + is_error=True, + error="redis connection refused", + service=ServiceTypes.REDIS, + duration=0.1, + call_type="async_get_cache", + event_metadata=None, + ), + error="redis connection refused", + parent_otel_span=otel.tracer.start_span("Received Proxy Server Request"), + start_time=now, + end_time=now, + ) + ) + + +_FAILED_SPANS: Final = pytest.mark.parametrize( + ("fail", "expected_description"), + [ + (_fail_proxy_request, _PROXY_KEY_ERROR), + (_fail_management_request, _PROXY_KEY_ERROR), + (_fail_service_call, "redis connection refused"), + ], + ids=["proxy_request", "management_request", "service_call"], +) + + +def _error_status_descriptions(fail: Callable[[TracerProvider], None], provider: TracerProvider) -> set[str | None]: + exporter: Final = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + fail(provider) + error_spans: Final = [s for s in exporter.get_finished_spans() if s.status.status_code == trace.StatusCode.ERROR] + assert error_spans + return {s.status.description for s in error_spans} + + +@_FAILED_SPANS +def test_every_error_span_carries_the_error_message_in_its_status( + fail: Callable[[TracerProvider], None], expected_description: str +) -> None: + assert _error_status_descriptions(fail, TracerProvider()) == {expected_description} + + +@_FAILED_SPANS +def test_error_status_is_cut_to_the_span_attribute_length_limit( + fail: Callable[[TracerProvider], None], expected_description: str +) -> None: + limit: Final = 12 + provider: Final = TracerProvider(span_limits=SpanLimits(max_attribute_length=limit)) + + assert _error_status_descriptions(fail, provider) == {expected_description[:limit]} + + class TestOpenTelemetrySetProxyRequestRouteAttributes(unittest.TestCase): """http.route (template) + url.path (literal) must land on the SERVER span. The logging handlers write the litellm_request child span, so