mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-29 06:33:28 +00:00
fix(otel): put the error message on every failed span status (#44185)
* fix(otel): put the error message on the failed span status The legacy OTel logger (used by otel, langfuse_otel, arize and others) ended a failed call with Status(StatusCode.ERROR) and no description, so backends that read the span status, such as Langfuse's status message, showed an empty error. The text was only in the error.message attribute. Use the same message as that attribute, as the OTel v2 emitter already does. * fix(otel): take the failure status message from _record_exception_on_span Instead of reading the logging payload a second time in _handle_failure, _record_exception_on_span returns the error message it already records as error.message, and _handle_failure puts it on the ERROR status. The other callers ignore the return value, so their spans are unchanged. Its input is now a typed view of what it reads (exception, error_information, error_str) instead of a bare dict. * fix(otel): put the error message on every failed span status Covers the proxy SERVER span, management endpoint spans, service spans, and Arize Phoenix spans, so no ERROR span ships with an empty status description * fix(otel): cut the error status to the span attribute length limit and describe dangling server spans The status description now uses the error text as stored on the span, so OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT applies to it too. A server span the proxy closes on a 4xx or 5xx outside the request path gets the same description, and an empty error message falls back to the error class * test(otel): type the validation-handler status test * test(otel): cover the Phoenix parent span and error_str length-limit status --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
parent
f03f98ac22
commit
f43f2d8c15
6 changed files with 343 additions and 55 deletions
|
|
@ -354,17 +354,16 @@ class ArizePhoenixLogger(OpenTelemetry):
|
|||
_project_name, tracer = self._resolve_tracer_for_kwargs(kwargs)
|
||||
ctx, parent_span = self._get_phoenix_context(kwargs, tracer=tracer)
|
||||
|
||||
status: Final = Status(StatusCode.OK if success else StatusCode.ERROR)
|
||||
status_code: Final = StatusCode.OK if success else StatusCode.ERROR
|
||||
|
||||
span: Final = tracer.start_span(
|
||||
name=self._get_span_name(kwargs),
|
||||
start_time=self._to_ns(start_time),
|
||||
context=ctx,
|
||||
)
|
||||
span.set_status(status)
|
||||
self.set_attributes(span, kwargs, response_obj)
|
||||
if not success:
|
||||
self._record_exception_on_span(span=span, kwargs=kwargs)
|
||||
span_error_message: Final = None if success else self._record_exception_on_span(span=span, kwargs=kwargs)
|
||||
span.set_status(Status(status_code, span_error_message))
|
||||
|
||||
if success:
|
||||
self._maybe_log_raw_request(kwargs, response_obj, start_time, end_time, span)
|
||||
|
|
@ -373,10 +372,11 @@ class ArizePhoenixLogger(OpenTelemetry):
|
|||
self._create_guardrail_span(kwargs=kwargs, context=ctx)
|
||||
|
||||
if parent_span is not None:
|
||||
parent_span.set_status(status)
|
||||
self.set_attributes(parent_span, kwargs, response_obj)
|
||||
if not success:
|
||||
self._record_exception_on_span(span=parent_span, kwargs=kwargs)
|
||||
parent_error_message: Final = (
|
||||
None if success else self._record_exception_on_span(span=parent_span, kwargs=kwargs)
|
||||
)
|
||||
parent_span.set_status(Status(status_code, parent_error_message))
|
||||
parent_span.end(end_time=self._to_ns(end_time))
|
||||
|
||||
if success:
|
||||
|
|
|
|||
|
|
@ -53,6 +53,7 @@ from litellm.types.utils import (
|
|||
StandardCallbackDynamicParams,
|
||||
StandardLoggingMetadata,
|
||||
StandardLoggingPayload,
|
||||
StandardLoggingPayloadErrorInformation,
|
||||
)
|
||||
|
||||
# OpenTelemetry imports moved to individual functions to avoid import errors when not installed
|
||||
|
|
@ -104,8 +105,26 @@ class _ResponseWithUsageView(TypedDict, total=False):
|
|||
usage: "_UsageCompletionTokensView | None"
|
||||
|
||||
|
||||
class _FailurePayloadView(TypedDict, total=False):
|
||||
error_information: ReadOnly[StandardLoggingPayloadErrorInformation | None]
|
||||
error_str: ReadOnly[str | None]
|
||||
|
||||
|
||||
class _FailureKwargsView(TypedDict, total=False):
|
||||
exception: ReadOnly[BaseException | None]
|
||||
standard_logging_object: ReadOnly[_FailurePayloadView | None]
|
||||
|
||||
|
||||
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
|
||||
|
||||
|
||||
def _text_as_stored_on_span(span: Span, key: str, text: str) -> str:
|
||||
from opentelemetry.sdk.trace import ReadableSpan
|
||||
|
||||
stored: Final = span.attributes.get(key) if isinstance(span, ReadableSpan) and span.attributes else None
|
||||
return stored if isinstance(stored, str) else text
|
||||
|
||||
|
||||
# Cap on credential-scoped providers held at once; each one owns an exporter thread.
|
||||
_MAX_DYNAMIC_TRACER_PROVIDERS: Final = 256
|
||||
|
||||
|
|
@ -922,7 +941,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
value=value,
|
||||
)
|
||||
|
||||
service_logging_span.set_status(Status(StatusCode.ERROR))
|
||||
service_logging_span.set_status(
|
||||
Status(
|
||||
StatusCode.ERROR, _text_as_stored_on_span(service_logging_span, "error", error) if error else None
|
||||
)
|
||||
)
|
||||
service_logging_span.end(end_time=_end_time_ns)
|
||||
|
||||
async def async_post_call_failure_hook(
|
||||
|
|
@ -937,8 +960,6 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
|
||||
parent_otel_span: Final = user_api_key_dict.parent_otel_span
|
||||
if parent_otel_span is not None:
|
||||
parent_otel_span.set_status(Status(StatusCode.ERROR))
|
||||
|
||||
# Stamp team attributes onto the SERVER (root) span too, so the
|
||||
# trace root is team-filterable on the failure path like the
|
||||
# child exception span below.
|
||||
|
|
@ -960,13 +981,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
original_exception=original_exception,
|
||||
traceback_str=traceback_str,
|
||||
)
|
||||
self._record_exception_on_span(
|
||||
server_error_message: Final = self._record_exception_on_span(
|
||||
span=parent_otel_span,
|
||||
kwargs={
|
||||
"exception": original_exception,
|
||||
"standard_logging_object": {"error_information": error_information},
|
||||
},
|
||||
)
|
||||
parent_otel_span.set_status(Status(StatusCode.ERROR, server_error_message))
|
||||
|
||||
# _record_exception_on_span only stamps when error_code is set;
|
||||
# bare TypeError etc. has none, and the span is about to be ended.
|
||||
|
|
@ -995,7 +1017,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
team_id=user_api_key_dict.team_id,
|
||||
team_alias=user_api_key_dict.team_alias,
|
||||
)
|
||||
exception_logging_span.set_status(Status(StatusCode.ERROR))
|
||||
exception_logging_span.set_status(Status(StatusCode.ERROR, server_error_message))
|
||||
exception_logging_span.end(end_time=self._to_ns(datetime.now()))
|
||||
|
||||
# Emit guardrail spans for any guardrail invocations that
|
||||
|
|
@ -2199,11 +2221,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
if self._gen_ai_semconv_latest_experimental:
|
||||
span_kwargs["kind"] = self.span_kind.CLIENT
|
||||
span = otel_tracer.start_span(**span_kwargs)
|
||||
span.set_status(Status(StatusCode.ERROR))
|
||||
self.set_attributes(span, kwargs, response_obj)
|
||||
|
||||
# Record exception information using OTEL standard method
|
||||
self._record_exception_on_span(span=span, kwargs=kwargs)
|
||||
span_error_message: Final = self._record_exception_on_span(span=span, kwargs=kwargs)
|
||||
span.set_status(Status(StatusCode.ERROR, span_error_message))
|
||||
|
||||
span.end(end_time=self._to_ns(end_time))
|
||||
else:
|
||||
|
|
@ -2212,9 +2234,9 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
# Only set attributes if the span is still recording (not closed)
|
||||
# Note: parent_otel_span is guaranteed to be not None here
|
||||
if parent_otel_span.is_recording():
|
||||
parent_otel_span.set_status(Status(StatusCode.ERROR))
|
||||
self.set_attributes(parent_otel_span, kwargs, response_obj)
|
||||
self._record_exception_on_span(span=parent_otel_span, kwargs=kwargs)
|
||||
parent_error_message: Final = self._record_exception_on_span(span=parent_otel_span, kwargs=kwargs)
|
||||
parent_otel_span.set_status(Status(StatusCode.ERROR, parent_error_message))
|
||||
|
||||
# Create span for guardrail information — ensure proper parenting (Issue #5)
|
||||
guardrail_ctx: Final = self._resolve_guardrail_context(
|
||||
|
|
@ -2232,13 +2254,15 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
):
|
||||
parent_otel_span.end(end_time=self._to_ns(end_time))
|
||||
|
||||
def _record_exception_on_span(self, span: Span, kwargs: dict):
|
||||
def _record_exception_on_span(self, span: Span, kwargs: _FailureKwargsView) -> str | None:
|
||||
"""
|
||||
Record exception information on the span using OTEL standard methods.
|
||||
|
||||
This extracts error information from StandardLoggingPayload and:
|
||||
1. Uses span.record_exception() for the actual exception object (OTEL standard)
|
||||
2. Sets structured error attributes from StandardLoggingPayloadErrorInformation
|
||||
|
||||
Returns the error message it recorded, if any, for the span status description.
|
||||
"""
|
||||
try:
|
||||
from litellm.integrations._types.open_inference import (
|
||||
|
|
@ -2253,10 +2277,10 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
span.record_exception(exception)
|
||||
|
||||
# Get StandardLoggingPayload for structured error information
|
||||
standard_logging_payload: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object")
|
||||
standard_logging_payload: Final = kwargs.get("standard_logging_object")
|
||||
|
||||
if standard_logging_payload is None:
|
||||
return
|
||||
return None
|
||||
|
||||
# Extract error_information from StandardLoggingPayload
|
||||
error_information: Final = standard_logging_payload.get("error_information")
|
||||
|
|
@ -2270,59 +2294,67 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
key=ErrorAttributes.ERROR_MESSAGE,
|
||||
value=error_str,
|
||||
)
|
||||
return
|
||||
return _text_as_stored_on_span(span, ErrorAttributes.ERROR_MESSAGE, error_str) if error_str else None
|
||||
|
||||
# Set structured error attributes from StandardLoggingPayloadErrorInformation
|
||||
if error_information.get("error_code"):
|
||||
error_code: Final = error_information.get("error_code")
|
||||
if error_code:
|
||||
self.safe_set_attribute(
|
||||
span=span,
|
||||
key=ErrorAttributes.ERROR_CODE,
|
||||
value=error_information["error_code"],
|
||||
value=error_code,
|
||||
)
|
||||
|
||||
# Also expose under the OTel-standard name as an int
|
||||
# (error_code is a str, may be non-numeric).
|
||||
_error_code_val: Final = error_information["error_code"]
|
||||
if _error_code_val is not None:
|
||||
try:
|
||||
self.safe_set_attribute(
|
||||
span=span,
|
||||
key=HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE,
|
||||
value=int(_error_code_val),
|
||||
)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
try:
|
||||
self.safe_set_attribute(
|
||||
span=span,
|
||||
key=HTTP_RESPONSE_STATUS_CODE_ATTRIBUTE,
|
||||
value=int(error_code),
|
||||
)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
if error_information.get("error_class"):
|
||||
error_class: Final = error_information.get("error_class")
|
||||
if error_class:
|
||||
self.safe_set_attribute(
|
||||
span=span,
|
||||
key=ErrorAttributes.ERROR_TYPE,
|
||||
value=error_information["error_class"],
|
||||
value=error_class,
|
||||
)
|
||||
|
||||
if error_information.get("error_message"):
|
||||
error_message: Final = error_information.get("error_message")
|
||||
if error_message:
|
||||
self.safe_set_attribute(
|
||||
span=span,
|
||||
key=ErrorAttributes.ERROR_MESSAGE,
|
||||
value=error_information["error_message"],
|
||||
value=error_message,
|
||||
)
|
||||
|
||||
if error_information.get("llm_provider"):
|
||||
llm_provider: Final = error_information.get("llm_provider")
|
||||
if llm_provider:
|
||||
self.safe_set_attribute(
|
||||
span=span,
|
||||
key=ErrorAttributes.ERROR_LLM_PROVIDER,
|
||||
value=error_information["llm_provider"],
|
||||
value=llm_provider,
|
||||
)
|
||||
|
||||
if error_information.get("traceback"):
|
||||
traceback_text: Final = error_information.get("traceback")
|
||||
if traceback_text:
|
||||
self.safe_set_attribute(
|
||||
span=span,
|
||||
key=ErrorAttributes.ERROR_STACK_TRACE,
|
||||
value=error_information["traceback"],
|
||||
value=traceback_text,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
verbose_logger.exception("OpenTelemetry: Error recording exception on span: %s", str(e))
|
||||
return None
|
||||
else:
|
||||
if error_message:
|
||||
return _text_as_stored_on_span(span, ErrorAttributes.ERROR_MESSAGE, error_message)
|
||||
return error_class or None
|
||||
|
||||
def set_tools_attributes(self, span: Span, tools):
|
||||
import json
|
||||
|
|
@ -3518,8 +3550,6 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
key="exception",
|
||||
value=str(_exception),
|
||||
)
|
||||
management_endpoint_span.set_status(Status(StatusCode.ERROR))
|
||||
management_endpoint_span.end(end_time=_end_time_ns)
|
||||
|
||||
# The management wrapper has no other hook that closes the SERVER span.
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
|
|
@ -3529,14 +3559,16 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
error_information: Final = StandardLoggingPayloadSetup.get_error_information(
|
||||
original_exception=_exception,
|
||||
)
|
||||
parent_otel_span.set_status(Status(StatusCode.ERROR))
|
||||
self._record_exception_on_span(
|
||||
server_error_message: Final = self._record_exception_on_span(
|
||||
span=parent_otel_span,
|
||||
kwargs={
|
||||
"exception": _exception,
|
||||
"standard_logging_object": {"error_information": error_information},
|
||||
},
|
||||
)
|
||||
management_endpoint_span.set_status(Status(StatusCode.ERROR, server_error_message))
|
||||
management_endpoint_span.end(end_time=_end_time_ns)
|
||||
parent_otel_span.set_status(Status(StatusCode.ERROR, server_error_message))
|
||||
parent_otel_span.end(end_time=_end_time_ns)
|
||||
|
||||
def create_litellm_proxy_request_started_span(
|
||||
|
|
@ -3596,19 +3628,20 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
span: Span | None,
|
||||
exception: Exception | None,
|
||||
status_code: int,
|
||||
) -> None:
|
||||
) -> str | None:
|
||||
"""Stamp structured ``error.*`` attributes on the SERVER span from the
|
||||
exception returned to the client, with ``error.code`` pinned to the real
|
||||
response status. Idempotent (overwrites); emits no exception event."""
|
||||
response status. Idempotent (overwrites); emits no exception event.
|
||||
Returns the error text for the span status description."""
|
||||
if span is None or exception is None:
|
||||
return
|
||||
return None
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
StandardLoggingPayloadSetup,
|
||||
)
|
||||
|
||||
error_information: Final = StandardLoggingPayloadSetup.get_error_information(original_exception=exception)
|
||||
error_information["error_code"] = str(status_code)
|
||||
self._record_exception_on_span(
|
||||
return self._record_exception_on_span(
|
||||
span=span,
|
||||
kwargs={"standard_logging_object": {"error_information": error_information}},
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2240,9 +2240,14 @@ def _close_dangling_otel_server_span(request: Request, status_code: int, exc: Ex
|
|||
open_telemetry_logger.record_error_attributes_on_span(parent_otel_span, exc, status_code)
|
||||
return
|
||||
open_telemetry_logger.set_response_status_code_attribute(parent_otel_span, status_code)
|
||||
if status_code >= 400:
|
||||
open_telemetry_logger.record_error_attributes_on_span(parent_otel_span, exc, status_code)
|
||||
parent_otel_span.set_status(Status(StatusCode.ERROR if status_code >= 400 else StatusCode.OK))
|
||||
parent_otel_span.set_status(
|
||||
Status(
|
||||
StatusCode.ERROR,
|
||||
open_telemetry_logger.record_error_attributes_on_span(parent_otel_span, exc, status_code),
|
||||
)
|
||||
if status_code >= 400
|
||||
else Status(StatusCode.OK)
|
||||
)
|
||||
parent_otel_span.end()
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.debug("Error closing dangling OTEL SERVER span: %s", str(e))
|
||||
|
|
|
|||
|
|
@ -2,7 +2,15 @@ import asyncio, importlib, litellm, logging, os, unittest
|
|||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Final
|
||||
|
||||
from opentelemetry.sdk.trace import SpanLimits, TracerProvider
|
||||
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
|
||||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
|
||||
from opentelemetry.trace import StatusCode
|
||||
|
||||
from litellm.integrations.opentelemetry import LITELLM_REQUEST_SPAN_NAME
|
||||
from litellm.integrations.arize.arize_phoenix import (
|
||||
ArizePhoenixConfig,
|
||||
ArizePhoenixLogger,
|
||||
|
|
@ -852,6 +860,47 @@ class TestPhoenixTraceHandling:
|
|||
assert span.resource.attributes.get("model_id") == "unified-proj"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("max_attribute_length", "expected_description"), [(None, "model not found"), (5, "model")])
|
||||
def test_failed_proxy_request_puts_the_error_message_on_every_phoenix_span_status(
|
||||
max_attribute_length: int | None, expected_description: str
|
||||
) -> None:
|
||||
exporter: Final = InMemorySpanExporter()
|
||||
provider: Final = TracerProvider(span_limits=SpanLimits(max_attribute_length=max_attribute_length))
|
||||
provider.add_span_processor(SimpleSpanProcessor(exporter))
|
||||
logger: Final = ArizePhoenixLogger(callback_name="arize_phoenix", tracer_provider=provider)
|
||||
start: Final = datetime(2026, 1, 1, tzinfo=timezone.utc)
|
||||
|
||||
logger._handle_failure(
|
||||
{
|
||||
"model": "gpt-4",
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"optional_params": {},
|
||||
"litellm_params": {
|
||||
"custom_llm_provider": "openai",
|
||||
"proxy_server_request": {"url": "/chat/completions", "method": "POST", "headers": {}},
|
||||
},
|
||||
"standard_logging_object": {
|
||||
"id": "test-id",
|
||||
"call_type": "completion",
|
||||
"metadata": {},
|
||||
"error_information": {"error_class": "NotFoundError", "error_message": "model not found"},
|
||||
},
|
||||
"exception": Exception("model not found"),
|
||||
},
|
||||
None,
|
||||
start,
|
||||
start + timedelta(seconds=1),
|
||||
)
|
||||
|
||||
statuses: Final = {
|
||||
span.name: (span.status.status_code, span.status.description) for span in exporter.get_finished_spans()
|
||||
}
|
||||
assert statuses == {
|
||||
"litellm_proxy_request": (StatusCode.ERROR, expected_description),
|
||||
LITELLM_REQUEST_SPAN_NAME: (StatusCode.ERROR, expected_description),
|
||||
}
|
||||
|
||||
|
||||
class TestGetArizePhoenixConfigProjectName:
|
||||
@patch.dict(
|
||||
"os.environ", {"PHOENIX_PROJECT_NAME": "phoenix-config-proj"}, clear=True
|
||||
|
|
|
|||
|
|
@ -3,10 +3,13 @@ that fail after auth but before the route handler runs (e.g. /model/new
|
|||
TypeError or RequestValidationError)."""
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Callable
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
|
||||
|
||||
import litellm.proxy.proxy_server as proxy_server_module
|
||||
from litellm.proxy._types import ProxyException
|
||||
|
|
@ -133,10 +136,30 @@ def test_openai_exception_handler_stamps_structured_error_on_span(
|
|||
expected_url_path=path,
|
||||
where=f"openai_exception_handler ({path})",
|
||||
)
|
||||
attrs = get_server_span(wired_otel).attributes
|
||||
server_span = get_server_span(wired_otel)
|
||||
attrs = server_span.attributes
|
||||
assert attrs.get(ErrorAttributes.ERROR_MESSAGE) == msg
|
||||
assert attrs.get(ErrorAttributes.ERROR_TYPE) == "ProxyException"
|
||||
assert attrs.get(ErrorAttributes.ERROR_CODE) == "401"
|
||||
assert server_span.status.description == msg
|
||||
|
||||
|
||||
def test_validation_handler_puts_the_error_message_on_the_span_status(
|
||||
wired_otel: InMemorySpanExporter, server_span_factory: Callable[[str], object]
|
||||
) -> None:
|
||||
path: Final = "/team/new"
|
||||
request: Final = _fake_request(parent_otel_span=server_span_factory(path), path=path)
|
||||
exc: Final = RequestValidationError(
|
||||
errors=[{"type": "float_parsing", "loc": ("body", "max_budget"), "msg": "Input should be a valid number"}]
|
||||
)
|
||||
|
||||
response: Final = asyncio.run(otel_request_validation_exception_handler(request, exc))
|
||||
|
||||
assert response.status_code == 422
|
||||
server_span: Final = get_server_span(wired_otel)
|
||||
description: Final = server_span.status.description or ""
|
||||
assert "Input should be a valid number" in description
|
||||
assert description == server_span.attributes.get(ErrorAttributes.ERROR_MESSAGE)
|
||||
|
||||
|
||||
def test_unhandled_handler_reraises_known_exceptions(wired_otel, server_span_factory):
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider
|
|||
from opentelemetry.sdk._logs.export import InMemoryLogExporter, LogExportResult, SimpleLogRecordProcessor
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import InMemoryMetricReader, MetricsData
|
||||
from opentelemetry.sdk.trace import ReadableSpan, TracerProvider
|
||||
from opentelemetry.sdk.trace import ReadableSpan, SpanLimits, TracerProvider
|
||||
from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExportResult
|
||||
from opentelemetry.sdk.util.instrumentation import InstrumentationScope
|
||||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
|
||||
|
|
@ -48,9 +48,10 @@ from litellm.integrations.opentelemetry import (
|
|||
)
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import SpanAttributes
|
||||
from litellm.proxy._types import ManagementEndpointLoggingPayload, ProxyException, SpanAttributes
|
||||
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
|
||||
from litellm.types.services import ServiceLoggerPayload, ServiceTypes
|
||||
from collections.abc import AsyncIterator
|
||||
from collections.abc import AsyncIterator, Callable, Mapping
|
||||
from litellm.constants import KUBERNETES_POD_ROUTING_KEY, LOGGING_WORKER_MAX_TIME_PER_COROUTINE
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.types.utils import ChatCompletionMessageToolCall, Function
|
||||
|
|
@ -3214,6 +3215,9 @@ class TestOpenTelemetryExternalSpan(unittest.TestCase):
|
|||
)
|
||||
|
||||
|
||||
_FIXED_START: Final = datetime(2026, 1, 1, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
class TestOpenTelemetrySemanticConventions138(unittest.TestCase):
|
||||
"""
|
||||
Test suite for OpenTelemetry 1.38 Semantic Conventions compliance.
|
||||
|
|
@ -3731,6 +3735,91 @@ class TestOpenTelemetrySemanticConventions138(unittest.TestCase):
|
|||
error_spans = [s for s in spans if s.status.status_code == StatusCode.ERROR]
|
||||
self.assertTrue(error_spans, "Expected at least one span with ERROR status")
|
||||
|
||||
@parameterized.expand(
|
||||
[
|
||||
(
|
||||
"own_span_error_information",
|
||||
False,
|
||||
{
|
||||
"error_str": "litellm.ServiceUnavailableError: upstream returned 503",
|
||||
"error_information": {
|
||||
"error_class": "ServiceUnavailableError",
|
||||
"error_message": "upstream returned 503",
|
||||
},
|
||||
},
|
||||
"upstream returned 503",
|
||||
),
|
||||
(
|
||||
"own_span_error_str_only",
|
||||
False,
|
||||
{"error_str": "litellm.ServiceUnavailableError: upstream returned 503"},
|
||||
"litellm.ServiceUnavailableError: upstream returned 503",
|
||||
),
|
||||
(
|
||||
"own_span_error_str_only_cut_to_the_attribute_length_limit",
|
||||
False,
|
||||
{"error_str": "litellm.ServiceUnavailableError: upstream returned 503"},
|
||||
"litellm.Serv",
|
||||
12,
|
||||
),
|
||||
(
|
||||
"own_span_empty_error_message",
|
||||
False,
|
||||
{"error_information": {"error_class": "ServiceUnavailableError", "error_message": ""}},
|
||||
"ServiceUnavailableError",
|
||||
),
|
||||
(
|
||||
"parent_span_error_information",
|
||||
True,
|
||||
{
|
||||
"error_information": {
|
||||
"error_class": "ServiceUnavailableError",
|
||||
"error_message": "upstream returned 503",
|
||||
}
|
||||
},
|
||||
"upstream returned 503",
|
||||
),
|
||||
]
|
||||
)
|
||||
def test_handle_failure_error_status_carries_the_error_message(
|
||||
self,
|
||||
_name: str,
|
||||
has_parent_span: bool,
|
||||
error_fields: Mapping[str, object],
|
||||
expected_description: str,
|
||||
max_attribute_length: int | None = None,
|
||||
) -> None:
|
||||
span_exporter: Final = InMemorySpanExporter()
|
||||
tracer_provider: Final = TracerProvider(span_limits=SpanLimits(max_attribute_length=max_attribute_length))
|
||||
tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter))
|
||||
otel: Final = OpenTelemetry(tracer_provider=tracer_provider)
|
||||
otel.tracer = tracer_provider.get_tracer("litellm")
|
||||
active_parent_span: Final = (
|
||||
trace.use_span(tracer_provider.get_tracer("proxy").start_span("parent_span"), end_on_exit=True)
|
||||
if has_parent_span
|
||||
else contextlib.nullcontext()
|
||||
)
|
||||
start: Final = _FIXED_START
|
||||
end: Final = start + timedelta(seconds=1)
|
||||
kwargs: Final = {
|
||||
"model": "gpt-4",
|
||||
"messages": [{"role": "user", "content": "Hello"}],
|
||||
"optional_params": {},
|
||||
"litellm_params": {"custom_llm_provider": "openai"},
|
||||
"standard_logging_object": {"id": "test-id", "call_type": "completion", "metadata": {}, **error_fields},
|
||||
"exception": Exception("upstream returned 503"),
|
||||
}
|
||||
|
||||
with patch.dict(os.environ, {"USE_OTEL_LITELLM_REQUEST_SPAN": "false"}), active_parent_span:
|
||||
otel._handle_failure(kwargs, None, start, end)
|
||||
|
||||
error_spans: Final = [
|
||||
s for s in span_exporter.get_finished_spans() if s.status.status_code == trace.StatusCode.ERROR
|
||||
]
|
||||
self.assertEqual(len(error_spans), 1)
|
||||
self.assertEqual(error_spans[0].name, "parent_span" if has_parent_span else "litellm_request")
|
||||
self.assertEqual(error_spans[0].status.description, expected_description)
|
||||
|
||||
|
||||
class TestRawSpanAttributeIsolation(unittest.TestCase):
|
||||
"""Issue #3: raw_gen_ai_request span should only contain provider-specific
|
||||
|
|
@ -5439,6 +5528,95 @@ class TestOpenTelemetryFailureHookStampsServerSpan(unittest.TestCase):
|
|||
)
|
||||
|
||||
|
||||
_PROXY_KEY_ERROR: Final = "Authentication Error, invalid proxy key"
|
||||
|
||||
|
||||
def _proxy_key_exception() -> ProxyException:
|
||||
return ProxyException(message=_PROXY_KEY_ERROR, type="auth_error", param="None", code=401)
|
||||
|
||||
|
||||
def _fail_proxy_request(provider: TracerProvider) -> None:
|
||||
otel: Final = OpenTelemetry(tracer_provider=provider)
|
||||
asyncio.run(
|
||||
otel.async_post_call_failure_hook(
|
||||
request_data={},
|
||||
original_exception=_proxy_key_exception(),
|
||||
user_api_key_dict=UserAPIKeyAuth(parent_otel_span=otel.tracer.start_span("Received Proxy Server Request")),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _fail_management_request(provider: TracerProvider) -> None:
|
||||
otel: Final = OpenTelemetry(tracer_provider=provider)
|
||||
now: Final = _FIXED_START
|
||||
asyncio.run(
|
||||
otel.async_management_endpoint_failure_hook(
|
||||
logging_payload=ManagementEndpointLoggingPayload(
|
||||
route="/key/generate", request_data={}, exception=_proxy_key_exception(), start_time=now, end_time=now
|
||||
),
|
||||
parent_otel_span=otel.tracer.start_span("Received Proxy Server Request"),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _fail_service_call(provider: TracerProvider) -> None:
|
||||
otel: Final = OpenTelemetry(tracer_provider=provider)
|
||||
now: Final = _FIXED_START
|
||||
asyncio.run(
|
||||
otel.async_service_failure_hook(
|
||||
payload=ServiceLoggerPayload(
|
||||
is_error=True,
|
||||
error="redis connection refused",
|
||||
service=ServiceTypes.REDIS,
|
||||
duration=0.1,
|
||||
call_type="async_get_cache",
|
||||
event_metadata=None,
|
||||
),
|
||||
error="redis connection refused",
|
||||
parent_otel_span=otel.tracer.start_span("Received Proxy Server Request"),
|
||||
start_time=now,
|
||||
end_time=now,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
_FAILED_SPANS: Final = pytest.mark.parametrize(
|
||||
("fail", "expected_description"),
|
||||
[
|
||||
(_fail_proxy_request, _PROXY_KEY_ERROR),
|
||||
(_fail_management_request, _PROXY_KEY_ERROR),
|
||||
(_fail_service_call, "redis connection refused"),
|
||||
],
|
||||
ids=["proxy_request", "management_request", "service_call"],
|
||||
)
|
||||
|
||||
|
||||
def _error_status_descriptions(fail: Callable[[TracerProvider], None], provider: TracerProvider) -> set[str | None]:
|
||||
exporter: Final = InMemorySpanExporter()
|
||||
provider.add_span_processor(SimpleSpanProcessor(exporter))
|
||||
fail(provider)
|
||||
error_spans: Final = [s for s in exporter.get_finished_spans() if s.status.status_code == trace.StatusCode.ERROR]
|
||||
assert error_spans
|
||||
return {s.status.description for s in error_spans}
|
||||
|
||||
|
||||
@_FAILED_SPANS
|
||||
def test_every_error_span_carries_the_error_message_in_its_status(
|
||||
fail: Callable[[TracerProvider], None], expected_description: str
|
||||
) -> None:
|
||||
assert _error_status_descriptions(fail, TracerProvider()) == {expected_description}
|
||||
|
||||
|
||||
@_FAILED_SPANS
|
||||
def test_error_status_is_cut_to_the_span_attribute_length_limit(
|
||||
fail: Callable[[TracerProvider], None], expected_description: str
|
||||
) -> None:
|
||||
limit: Final = 12
|
||||
provider: Final = TracerProvider(span_limits=SpanLimits(max_attribute_length=limit))
|
||||
|
||||
assert _error_status_descriptions(fail, provider) == {expected_description[:limit]}
|
||||
|
||||
|
||||
class TestOpenTelemetrySetProxyRequestRouteAttributes(unittest.TestCase):
|
||||
"""http.route (template) + url.path (literal) must land on the SERVER
|
||||
span. The logging handlers write the litellm_request child span, so
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue