From 29a28ec0e07c44f57b8f6cb4b81a784033466270 Mon Sep 17 00:00:00 2001 From: Wiley Kestner Date: Fri, 30 Oct 2026 00:16:23 +0100 Subject: [PATCH 1/4] fix(proxy): trace model list requests with the caller's traceparent The proxy creates an OTEL SERVER span at the auth boundary whenever a request carries traceparent, but /v1/models never closed it. Unended spans are not exported, so callers saw nothing for the model list while inference requests showed up in the same trace Route the endpoint through the management span wrapper so the SERVER span is closed and parented to the caller's trace. The wrapper gains a variant that skips request and response payloads, since attaching the full model list would bloat every poll --- litellm/proxy/management_helpers/utils.py | 32 +++++--- litellm/proxy/proxy_server.py | 2 + .../test_otel_admin_endpoints.py | 77 +++++++++++++++++++ 3 files changed, 101 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/management_helpers/utils.py b/litellm/proxy/management_helpers/utils.py index 30aaaacad04..5fb26f89db0 100644 --- a/litellm/proxy/management_helpers/utils.py +++ b/litellm/proxy/management_helpers/utils.py @@ -1,11 +1,11 @@ # What is this? ## Helper utils for the management endpoints (keys/users/teams) -from collections.abc import Callable, Mapping, MutableMapping, Sequence +from collections.abc import Awaitable, Callable, Mapping, MutableMapping, Sequence from collections.abc import Set as AbstractSet from datetime import datetime from functools import wraps from types import MappingProxyType -from typing import Any, Final, Protocol +from typing import Any, Final, Protocol, TypeVar, cast from fastapi import HTTPException, Request from pydantic import BaseModel, TypeAdapter @@ -767,19 +767,31 @@ async def _emit_management_endpoint_otel_span( ) -def management_endpoint_wrapper(func): +_Endpoint: Final = TypeVar("_Endpoint", bound=Callable[..., Awaitable[Any]]) + + +def management_endpoint_wrapper(func: _Endpoint) -> _Endpoint: """ This wrapper does the following: 1. Log I/O, Exceptions to OTEL 2. Create an Audit log for success calls """ + return _wrap_management_endpoint(func, log_payload=True) - @wraps(func) + +def management_endpoint_span_wrapper(func: _Endpoint) -> _Endpoint: + """``management_endpoint_wrapper`` without request/response payloads on the span, for + endpoints that serialize large or high-frequency responses (e.g. the model list).""" + return _wrap_management_endpoint(func, log_payload=False) + + +def _wrap_management_endpoint(endpoint: _Endpoint, *, log_payload: bool) -> _Endpoint: + @wraps(endpoint) async def wrapper(*args, **kwargs): start_time: Final = datetime.now() try: - result: Final = await func(*args, **kwargs) + result: Final = await endpoint(*args, **kwargs) end_time = datetime.now() try: user_api_key_dict: UserAPIKeyAuth = kwargs.get("user_api_key_dict") or UserAPIKeyAuth() @@ -787,17 +799,17 @@ def management_endpoint_wrapper(func): await send_management_endpoint_alert( request_kwargs=kwargs, user_api_key_dict=user_api_key_dict, - function_name=func.__name__, + function_name=endpoint.__name__, ) parent_otel_span: Span | None = getattr(user_api_key_dict, "parent_otel_span", None) if parent_otel_span is not None: await _emit_management_endpoint_otel_span( - func=func, + func=endpoint, kwargs=kwargs, parent_otel_span=parent_otel_span, start_time=start_time, end_time=end_time, - result=result, + result=result if log_payload else None, ) # Delete updated/deleted info from cache @@ -818,7 +830,7 @@ def management_endpoint_wrapper(func): if parent_otel_span is not None: try: await _emit_management_endpoint_otel_span( - func=func, + func=endpoint, kwargs=kwargs, parent_otel_span=parent_otel_span, start_time=start_time, @@ -835,4 +847,4 @@ def management_endpoint_wrapper(func): raise e - return wrapper + return cast(_Endpoint, wrapper) # cast-ok: wraps() preserves the endpoint's signature diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index d4ec26614c0..eb490a1cf09 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -174,6 +174,7 @@ from litellm.proxy.common_utils.token_count_settings import ( ) from litellm.proxy.management_helpers.auto_router_availability import AutoRouterCatalogEntry, build_auto_router_catalog from litellm.proxy.management_helpers.model_group_custom_fields import model_group_custom_fields +from litellm.proxy.management_helpers.utils import management_endpoint_span_wrapper from litellm.router_strategy.auto_router.auto_router_model_naming import ( GATED_AUTO_ROUTER_CAPABILITIES, STRATEGY_ROUTER_PARAM_FIELDS, @@ -12335,6 +12336,7 @@ async def _deployment_hidden_by_listing_callbacks(deployment: Deployment, user_a @router.get( "/models", dependencies=[Depends(user_api_key_auth)], tags=["model management"] ) # if project requires model list +@management_endpoint_span_wrapper async def model_list( request: Request = None, # pyright: ignore[reportArgumentType] # FastAPI always injects the Request; the None default only serves direct in-process callers user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), diff --git a/tests/unit/integrations/open_telemetry/test_otel_admin_endpoints.py b/tests/unit/integrations/open_telemetry/test_otel_admin_endpoints.py index ebeb514ffff..5ed512492c5 100644 --- a/tests/unit/integrations/open_telemetry/test_otel_admin_endpoints.py +++ b/tests/unit/integrations/open_telemetry/test_otel_admin_endpoints.py @@ -340,3 +340,80 @@ def test_management_wrapper_swallows_post_success_errors(server_span_factory, ot result = asyncio.run(fake_fn(data={}, user_api_key_dict=_real_user_api_key_dict(server_span))) assert result == {"ok": True} + + +def test_span_wrapper_omits_payload(server_span_factory, otel_with_exporter, monkeypatch): + """The span-only wrapper keeps the endpoint span but drops the response + attributes — the models listing would otherwise serialize every model onto + the span.""" + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.management_helpers import utils as mgmt_utils + + otel, exporter = otel_with_exporter + monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False) + monkeypatch.setattr(mgmt_utils, "send_management_endpoint_alert", _noop_alert) + + server_span = server_span_factory(KEY_GENERATE_PATH) + + @mgmt_utils.management_endpoint_span_wrapper + async def fake_fn(data=None, user_api_key_dict=None): + return {"data": [{"id": "gpt-4o"}]} + + asyncio.run(fake_fn(data={}, user_api_key_dict=_real_user_api_key_dict(server_span))) + + spans = [s for s in exporter.get_finished_spans() if s.name == "fake_fn"] + assert len(spans) == 1 + assert not any(key.startswith("response.") for key in spans[0].attributes) + + +MODEL_LIST_PATH = "/v1/models" +INBOUND_TRACEPARENT = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01" + + +def test_model_list_ends_server_span_in_inbound_trace(otel_with_exporter, monkeypatch): + """Regression: ``/v1/models`` created the OTEL SERVER span at the auth + boundary but never ended it, so the span was never exported and a caller + passing ``traceparent`` saw nothing for the models list. + + Drives the real endpoint and asserts the SERVER span reaches the exporter + under the caller's trace id.""" + import litellm.proxy.proxy_server as proxy_server + import litellm.proxy.utils as proxy_utils + from litellm.proxy.management_helpers import utils as mgmt_utils + + otel, exporter = otel_with_exporter + monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False) + monkeypatch.setattr(mgmt_utils, "send_management_endpoint_alert", _noop_alert) + + async def _no_models_for_user(**kwargs): + return [] + + monkeypatch.setattr(proxy_utils, "get_available_models_for_user", _no_models_for_user) + + server_span = otel.create_litellm_proxy_request_started_span( + start_time=datetime.now(), headers={"traceparent": INBOUND_TRACEPARENT} + ) + otel.set_proxy_request_route_attributes(server_span, url_path=MODEL_LIST_PATH, http_route=MODEL_LIST_PATH) + + result = asyncio.run( + proxy_server.model_list( + user_api_key_dict=_real_user_api_key_dict(server_span), + ) + ) + + assert result == {"data": [], "object": "list"} + assert_server_span_attrs( + exporter, + expected_status=200, + expected_url_path=MODEL_LIST_PATH, + where="model list success", + ) + exported = get_server_span(exporter) + assert exported is not None + assert exported.context.trace_id == int(INBOUND_TRACEPARENT.split("-")[1], 16) + + endpoint_spans = [s for s in exporter.get_finished_spans() if s.name == "model_list"] + assert len(endpoint_spans) == 1 + assert endpoint_spans[0].parent is not None + assert endpoint_spans[0].parent.span_id == exported.context.span_id + assert not any(key.startswith("response.") for key in endpoint_spans[0].attributes) From 0d974356a925e15b3de05d2ccba32a725a369624 Mon Sep 17 00:00:00 2001 From: Wiley Kestner Date: Fri, 30 Oct 2026 04:07:27 +0100 Subject: [PATCH 2/4] refactor(proxy): type the span wrapper with ParamSpec The strict ruff rules ban new typing.cast imports, so the wrapper keeps the wrapped endpoint's exact signature via ParamSpec instead of casting the wrapped callable back to its type. No behavior change --- litellm/proxy/management_helpers/utils.py | 36 +++++++++++++---------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/management_helpers/utils.py b/litellm/proxy/management_helpers/utils.py index 5fb26f89db0..924bcbe496e 100644 --- a/litellm/proxy/management_helpers/utils.py +++ b/litellm/proxy/management_helpers/utils.py @@ -5,7 +5,7 @@ from collections.abc import Set as AbstractSet from datetime import datetime from functools import wraps from types import MappingProxyType -from typing import Any, Final, Protocol, TypeVar, cast +from typing import Any, Final, ParamSpec, Protocol, TypeVar from fastapi import HTTPException, Request from pydantic import BaseModel, TypeAdapter @@ -767,10 +767,11 @@ async def _emit_management_endpoint_otel_span( ) -_Endpoint: Final = TypeVar("_Endpoint", bound=Callable[..., Awaitable[Any]]) +_P: Final = ParamSpec("_P") +_R: Final = TypeVar("_R") -def management_endpoint_wrapper(func: _Endpoint) -> _Endpoint: +def management_endpoint_wrapper(func: Callable[_P, Awaitable[_R]]) -> Callable[_P, Awaitable[_R]]: """ This wrapper does the following: @@ -780,24 +781,27 @@ def management_endpoint_wrapper(func: _Endpoint) -> _Endpoint: return _wrap_management_endpoint(func, log_payload=True) -def management_endpoint_span_wrapper(func: _Endpoint) -> _Endpoint: +def management_endpoint_span_wrapper(func: Callable[_P, Awaitable[_R]]) -> Callable[_P, Awaitable[_R]]: """``management_endpoint_wrapper`` without request/response payloads on the span, for endpoints that serialize large or high-frequency responses (e.g. the model list).""" return _wrap_management_endpoint(func, log_payload=False) -def _wrap_management_endpoint(endpoint: _Endpoint, *, log_payload: bool) -> _Endpoint: +def _wrap_management_endpoint( + endpoint: Callable[_P, Awaitable[_R]], *, log_payload: bool +) -> Callable[_P, Awaitable[_R]]: @wraps(endpoint) - async def wrapper(*args, **kwargs): + async def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R: + request_kwargs: Final[dict[str, Any]] = dict(kwargs) # mutable-ok: passed to dict-typed helpers start_time: Final = datetime.now() try: result: Final = await endpoint(*args, **kwargs) end_time = datetime.now() try: - user_api_key_dict: UserAPIKeyAuth = kwargs.get("user_api_key_dict") or UserAPIKeyAuth() + user_api_key_dict: UserAPIKeyAuth = request_kwargs.get("user_api_key_dict") or UserAPIKeyAuth() await send_management_endpoint_alert( - request_kwargs=kwargs, + request_kwargs=request_kwargs, user_api_key_dict=user_api_key_dict, function_name=endpoint.__name__, ) @@ -805,7 +809,7 @@ def _wrap_management_endpoint(endpoint: _Endpoint, *, log_payload: bool) -> _End if parent_otel_span is not None: await _emit_management_endpoint_otel_span( func=endpoint, - kwargs=kwargs, + kwargs=request_kwargs, parent_otel_span=parent_otel_span, start_time=start_time, end_time=end_time, @@ -813,10 +817,10 @@ def _wrap_management_endpoint(endpoint: _Endpoint, *, log_payload: bool) -> _End ) # Delete updated/deleted info from cache - _delete_api_key_from_cache(kwargs=kwargs) - _delete_user_id_from_cache(kwargs=kwargs) - _delete_team_id_from_cache(kwargs=kwargs) - _delete_customer_id_from_cache(kwargs=kwargs) + _delete_api_key_from_cache(kwargs=request_kwargs) + _delete_user_id_from_cache(kwargs=request_kwargs) + _delete_team_id_from_cache(kwargs=request_kwargs) + _delete_customer_id_from_cache(kwargs=request_kwargs) except Exception as e: # Non-Blocking Exception verbose_logger.debug("Error in management endpoint wrapper: %s", str(e)) @@ -825,13 +829,13 @@ def _wrap_management_endpoint(endpoint: _Endpoint, *, log_payload: bool) -> _End except Exception as e: end_time = datetime.now() - user_api_key_dict: UserAPIKeyAuth = kwargs.get("user_api_key_dict") or UserAPIKeyAuth() + user_api_key_dict: UserAPIKeyAuth = request_kwargs.get("user_api_key_dict") or UserAPIKeyAuth() parent_otel_span = getattr(user_api_key_dict, "parent_otel_span", None) if parent_otel_span is not None: try: await _emit_management_endpoint_otel_span( func=endpoint, - kwargs=kwargs, + kwargs=request_kwargs, parent_otel_span=parent_otel_span, start_time=start_time, end_time=end_time, @@ -847,4 +851,4 @@ def _wrap_management_endpoint(endpoint: _Endpoint, *, log_payload: bool) -> _End raise e - return cast(_Endpoint, wrapper) # cast-ok: wraps() preserves the endpoint's signature + return wrapper From b851a92a08ab3f888d07209a102c58eed6411712 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 07:46:50 -0700 Subject: [PATCH 3/4] fix(proxy): keep request bodies off the model list span and tidy its tests The quiet span wrapper now skips the request body as well as the response, so no payload reaches the span on success or failure. The wrapper passes the endpoint's kwargs through instead of copying them, and reads the caller's key with an isinstance check. The model list test drives the real endpoint without swapping helpers, and the new payload test checks both wrappers against a real request body --- litellm/proxy/management_helpers/utils.py | 37 +++++---- .../test_otel_admin_endpoints.py | 78 +++++++++++-------- 2 files changed, 68 insertions(+), 47 deletions(-) diff --git a/litellm/proxy/management_helpers/utils.py b/litellm/proxy/management_helpers/utils.py index 924bcbe496e..f514f233b0a 100644 --- a/litellm/proxy/management_helpers/utils.py +++ b/litellm/proxy/management_helpers/utils.py @@ -687,6 +687,7 @@ async def _emit_management_endpoint_otel_span( end_time: datetime, result: Any = None, exception: Exception | None = None, + log_payload: bool = True, ) -> None: """Stamp + end the parent OTEL SERVER span for a management endpoint. @@ -716,8 +717,8 @@ async def _emit_management_endpoint_otel_span( ) route = get_request_route(http_request) - request_body: dict = await read_request_body( # rebind-ok: pre-existing rebinding on a rename-only line - request=http_request + request_body: dict = ( # rebind-ok: pre-existing rebinding on a rename-only line + await read_request_body(request=http_request) if log_payload else {} ) else: route = func.__name__ @@ -738,7 +739,7 @@ async def _emit_management_endpoint_otel_span( ) _response: dict[str, object] | None = None - if exception is None and result is not None: + if log_payload and exception is None and result is not None: try: raw: Final[Mapping[str, object]] = dict(result) _response = {k: v for k, v in raw.items() if k not in _CREDENTIAL_FIELDS} @@ -782,26 +783,28 @@ def management_endpoint_wrapper(func: Callable[_P, Awaitable[_R]]) -> Callable[_ def management_endpoint_span_wrapper(func: Callable[_P, Awaitable[_R]]) -> Callable[_P, Awaitable[_R]]: - """``management_endpoint_wrapper`` without request/response payloads on the span, for - endpoints that serialize large or high-frequency responses (e.g. the model list).""" return _wrap_management_endpoint(func, log_payload=False) +def _caller_auth(endpoint_kwargs: Mapping[str, object]) -> UserAPIKeyAuth: + caller: Final = endpoint_kwargs.get("user_api_key_dict") + return caller if isinstance(caller, UserAPIKeyAuth) else UserAPIKeyAuth() + + def _wrap_management_endpoint( endpoint: Callable[_P, Awaitable[_R]], *, log_payload: bool ) -> Callable[_P, Awaitable[_R]]: @wraps(endpoint) async def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R: - request_kwargs: Final[dict[str, Any]] = dict(kwargs) # mutable-ok: passed to dict-typed helpers start_time: Final = datetime.now() try: result: Final = await endpoint(*args, **kwargs) end_time = datetime.now() try: - user_api_key_dict: UserAPIKeyAuth = request_kwargs.get("user_api_key_dict") or UserAPIKeyAuth() + user_api_key_dict: UserAPIKeyAuth = _caller_auth(kwargs) await send_management_endpoint_alert( - request_kwargs=request_kwargs, + request_kwargs=kwargs, user_api_key_dict=user_api_key_dict, function_name=endpoint.__name__, ) @@ -809,18 +812,19 @@ def _wrap_management_endpoint( if parent_otel_span is not None: await _emit_management_endpoint_otel_span( func=endpoint, - kwargs=request_kwargs, + kwargs=kwargs, parent_otel_span=parent_otel_span, start_time=start_time, end_time=end_time, - result=result if log_payload else None, + result=result, + log_payload=log_payload, ) # Delete updated/deleted info from cache - _delete_api_key_from_cache(kwargs=request_kwargs) - _delete_user_id_from_cache(kwargs=request_kwargs) - _delete_team_id_from_cache(kwargs=request_kwargs) - _delete_customer_id_from_cache(kwargs=request_kwargs) + _delete_api_key_from_cache(kwargs=kwargs) + _delete_user_id_from_cache(kwargs=kwargs) + _delete_team_id_from_cache(kwargs=kwargs) + _delete_customer_id_from_cache(kwargs=kwargs) except Exception as e: # Non-Blocking Exception verbose_logger.debug("Error in management endpoint wrapper: %s", str(e)) @@ -829,17 +833,18 @@ def _wrap_management_endpoint( except Exception as e: end_time = datetime.now() - user_api_key_dict: UserAPIKeyAuth = request_kwargs.get("user_api_key_dict") or UserAPIKeyAuth() + user_api_key_dict: UserAPIKeyAuth = _caller_auth(kwargs) parent_otel_span = getattr(user_api_key_dict, "parent_otel_span", None) if parent_otel_span is not None: try: await _emit_management_endpoint_otel_span( func=endpoint, - kwargs=request_kwargs, + kwargs=kwargs, parent_otel_span=parent_otel_span, start_time=start_time, end_time=end_time, exception=e, + log_payload=log_payload, ) except Exception as otel_exc: # Non-Blocking Exception - never let OTEL failures swallow diff --git a/tests/unit/integrations/open_telemetry/test_otel_admin_endpoints.py b/tests/unit/integrations/open_telemetry/test_otel_admin_endpoints.py index 5ed512492c5..778845f6b62 100644 --- a/tests/unit/integrations/open_telemetry/test_otel_admin_endpoints.py +++ b/tests/unit/integrations/open_telemetry/test_otel_admin_endpoints.py @@ -3,9 +3,11 @@ async_management_endpoint_{success,failure}_hook integration points.""" import asyncio from datetime import datetime +from typing import Final from unittest.mock import MagicMock import pytest +from fastapi import Request from litellm.proxy._types import ( ManagementEndpointLoggingPayload, @@ -342,60 +344,74 @@ def test_management_wrapper_swallows_post_success_errors(server_span_factory, ot assert result == {"ok": True} -def test_span_wrapper_omits_payload(server_span_factory, otel_with_exporter, monkeypatch): - """The span-only wrapper keeps the endpoint span but drops the response - attributes — the models listing would otherwise serialize every model onto - the span.""" +TEAM_NEW_PATH: Final = "/team/new" + + +def _json_post(path: str, body: bytes) -> Request: + async def receive() -> dict[str, object]: + return {"type": "http.request", "body": body, "more_body": False} + + return Request( + { + "type": "http", + "method": "POST", + "path": path, + "root_path": "", + "query_string": b"", + "headers": [(b"content-type", b"application/json")], + }, + receive, + ) + + +@pytest.mark.parametrize( + ("wrapper_name", "exports_payload"), + [("management_endpoint_wrapper", True), ("management_endpoint_span_wrapper", False)], +) +def test_span_wrapper_omits_payload(server_span_factory, otel_with_exporter, monkeypatch, wrapper_name, exports_payload): import litellm.proxy.proxy_server as proxy_server from litellm.proxy.management_helpers import utils as mgmt_utils otel, exporter = otel_with_exporter monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False) - monkeypatch.setattr(mgmt_utils, "send_management_endpoint_alert", _noop_alert) - server_span = server_span_factory(KEY_GENERATE_PATH) + server_span: Final = server_span_factory(TEAM_NEW_PATH) + wrap: Final = getattr(mgmt_utils, wrapper_name) - @mgmt_utils.management_endpoint_span_wrapper - async def fake_fn(data=None, user_api_key_dict=None): + @wrap + async def fake_fn(http_request: Request, user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]: return {"data": [{"id": "gpt-4o"}]} - asyncio.run(fake_fn(data={}, user_api_key_dict=_real_user_api_key_dict(server_span))) + asyncio.run( + fake_fn( + http_request=_json_post(TEAM_NEW_PATH, b'{"team_alias": "t"}'), + user_api_key_dict=_real_user_api_key_dict(server_span), + ) + ) - spans = [s for s in exporter.get_finished_spans() if s.name == "fake_fn"] + spans: Final = [s for s in exporter.get_finished_spans() if s.name == TEAM_NEW_PATH] assert len(spans) == 1 - assert not any(key.startswith("response.") for key in spans[0].attributes) + attribute_keys: Final = tuple(spans[0].attributes) + assert any(key.startswith("request.") for key in attribute_keys) is exports_payload + assert any(key.startswith("response.") for key in attribute_keys) is exports_payload -MODEL_LIST_PATH = "/v1/models" -INBOUND_TRACEPARENT = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01" +MODEL_LIST_PATH: Final = "/v1/models" +INBOUND_TRACEPARENT: Final = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01" def test_model_list_ends_server_span_in_inbound_trace(otel_with_exporter, monkeypatch): - """Regression: ``/v1/models`` created the OTEL SERVER span at the auth - boundary but never ended it, so the span was never exported and a caller - passing ``traceparent`` saw nothing for the models list. - - Drives the real endpoint and asserts the SERVER span reaches the exporter - under the caller's trace id.""" import litellm.proxy.proxy_server as proxy_server - import litellm.proxy.utils as proxy_utils - from litellm.proxy.management_helpers import utils as mgmt_utils otel, exporter = otel_with_exporter monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False) - monkeypatch.setattr(mgmt_utils, "send_management_endpoint_alert", _noop_alert) - async def _no_models_for_user(**kwargs): - return [] - - monkeypatch.setattr(proxy_utils, "get_available_models_for_user", _no_models_for_user) - - server_span = otel.create_litellm_proxy_request_started_span( + server_span: Final = otel.create_litellm_proxy_request_started_span( start_time=datetime.now(), headers={"traceparent": INBOUND_TRACEPARENT} ) otel.set_proxy_request_route_attributes(server_span, url_path=MODEL_LIST_PATH, http_route=MODEL_LIST_PATH) - result = asyncio.run( + result: Final = asyncio.run( proxy_server.model_list( user_api_key_dict=_real_user_api_key_dict(server_span), ) @@ -408,11 +424,11 @@ def test_model_list_ends_server_span_in_inbound_trace(otel_with_exporter, monkey expected_url_path=MODEL_LIST_PATH, where="model list success", ) - exported = get_server_span(exporter) + exported: Final = get_server_span(exporter) assert exported is not None assert exported.context.trace_id == int(INBOUND_TRACEPARENT.split("-")[1], 16) - endpoint_spans = [s for s in exporter.get_finished_spans() if s.name == "model_list"] + endpoint_spans: Final = [s for s in exporter.get_finished_spans() if s.name == "model_list"] assert len(endpoint_spans) == 1 assert endpoint_spans[0].parent is not None assert endpoint_spans[0].parent.span_id == exported.context.span_id From 023c6b0fedc741809f67459ae2f8335607944c9e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 10:46:50 -0700 Subject: [PATCH 4/4] fix(proxy): keep the 5xx status on the model list span and trace model retrieve The span wrapper ended the server span on every failure, so a 500 or 503 from /v1/models lost its status code. It now leaves that span to the proxy's exception handler, which knows the answered status, and only ends it itself for an HTTPException, which no handler closes on these routes. GET /v1/models/{model_id} (the OpenAI SDK's models.retrieve) gets the same wrapper, so it joins the caller's trace like the listing does. --- litellm/proxy/management_helpers/utils.py | 7 +- litellm/proxy/proxy_server.py | 1 + .../test_otel_admin_endpoints.py | 169 +++++++++++++++--- 3 files changed, 147 insertions(+), 30 deletions(-) diff --git a/litellm/proxy/management_helpers/utils.py b/litellm/proxy/management_helpers/utils.py index f514f233b0a..ea19befccc6 100644 --- a/litellm/proxy/management_helpers/utils.py +++ b/litellm/proxy/management_helpers/utils.py @@ -9,6 +9,7 @@ from typing import Any, Final, ParamSpec, Protocol, TypeVar from fastapi import HTTPException, Request from pydantic import BaseModel, TypeAdapter +from starlette.exceptions import HTTPException as StarletteHTTPException import litellm from litellm._internal_context import with_service_target @@ -791,6 +792,10 @@ def _caller_auth(endpoint_kwargs: Mapping[str, object]) -> UserAPIKeyAuth: return caller if isinstance(caller, UserAPIKeyAuth) else UserAPIKeyAuth() +def _proxy_exception_handler_ends_server_span(exception: Exception) -> bool: + return not isinstance(exception, StarletteHTTPException) + + def _wrap_management_endpoint( endpoint: Callable[_P, Awaitable[_R]], *, log_payload: bool ) -> Callable[_P, Awaitable[_R]]: @@ -835,7 +840,7 @@ def _wrap_management_endpoint( user_api_key_dict: UserAPIKeyAuth = _caller_auth(kwargs) parent_otel_span = getattr(user_api_key_dict, "parent_otel_span", None) - if parent_otel_span is not None: + if parent_otel_span is not None and (log_payload or not _proxy_exception_handler_ends_server_span(e)): try: await _emit_management_endpoint_otel_span( func=endpoint, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index eb490a1cf09..f5d62c1405e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -12629,6 +12629,7 @@ async def model_list( dependencies=[Depends(user_api_key_auth)], tags=["model management"], ) +@management_endpoint_span_wrapper async def model_info( model_id: str, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), diff --git a/tests/unit/integrations/open_telemetry/test_otel_admin_endpoints.py b/tests/unit/integrations/open_telemetry/test_otel_admin_endpoints.py index 778845f6b62..1965fba94c6 100644 --- a/tests/unit/integrations/open_telemetry/test_otel_admin_endpoints.py +++ b/tests/unit/integrations/open_telemetry/test_otel_admin_endpoints.py @@ -6,9 +6,11 @@ from datetime import datetime from typing import Final from unittest.mock import MagicMock +import httpx import pytest from fastapi import Request +from litellm import Router from litellm.proxy._types import ( ManagementEndpointLoggingPayload, UserAPIKeyAuth, @@ -345,16 +347,19 @@ def test_management_wrapper_swallows_post_success_errors(server_span_factory, ot TEAM_NEW_PATH: Final = "/team/new" +MODEL_LIST_PATH: Final = "/v1/models" +MODEL_RETRIEVE_PATH: Final = "/v1/models/gpt-4o" +INBOUND_TRACEPARENT: Final = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01" -def _json_post(path: str, body: bytes) -> Request: +def _request(method: str, path: str, body: bytes = b"") -> Request: async def receive() -> dict[str, object]: return {"type": "http.request", "body": body, "more_body": False} return Request( { "type": "http", - "method": "POST", + "method": method, "path": path, "root_path": "", "query_string": b"", @@ -364,6 +369,15 @@ def _json_post(path: str, body: bytes) -> Request: ) +async def _answer_like_the_proxy(request: Request, exception: Exception) -> None: + from litellm.proxy.proxy_server import app + + handler: Final = next( + app.exception_handlers[cls] for cls in type(exception).__mro__ if cls in app.exception_handlers + ) + await handler(request, exception) + + @pytest.mark.parametrize( ("wrapper_name", "exports_payload"), [("management_endpoint_wrapper", True), ("management_endpoint_span_wrapper", False)], @@ -384,7 +398,7 @@ def test_span_wrapper_omits_payload(server_span_factory, otel_with_exporter, mon asyncio.run( fake_fn( - http_request=_json_post(TEAM_NEW_PATH, b'{"team_alias": "t"}'), + http_request=_request("POST", TEAM_NEW_PATH, b'{"team_alias": "t"}'), user_api_key_dict=_real_user_api_key_dict(server_span), ) ) @@ -396,8 +410,107 @@ def test_span_wrapper_omits_payload(server_span_factory, otel_with_exporter, mon assert any(key.startswith("response.") for key in attribute_keys) is exports_payload -MODEL_LIST_PATH: Final = "/v1/models" -INBOUND_TRACEPARENT: Final = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01" +@pytest.mark.parametrize( + ("wrapper_name", "exports_request_body"), + [("management_endpoint_wrapper", True), ("management_endpoint_span_wrapper", False)], +) +def test_span_wrapper_omits_request_body_on_failure( + server_span_factory, otel_with_exporter, monkeypatch, wrapper_name, exports_request_body +): + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.management_helpers import utils as mgmt_utils + + otel, exporter = otel_with_exporter + monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False) + + server_span: Final = server_span_factory(TEAM_NEW_PATH) + wrap: Final = getattr(mgmt_utils, wrapper_name) + + @wrap + async def fake_fn(http_request: Request, user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]: + raise make_fastapi_http_exception(400, "team_alias taken") + + with pytest.raises(Exception, match="team_alias taken"): + asyncio.run( + fake_fn( + http_request=_request("POST", TEAM_NEW_PATH, b'{"team_alias": "t"}'), + user_api_key_dict=_real_user_api_key_dict(server_span), + ) + ) + + spans: Final = [s for s in exporter.get_finished_spans() if s.name == TEAM_NEW_PATH] + assert len(spans) == 1 + assert any(key.startswith("request.") for key in spans[0].attributes) is exports_request_body + + +@pytest.mark.parametrize( + ("raised", "status_code"), + [ + (RuntimeError("router settings lookup failed"), 500), + (httpx.ConnectError("All connection attempts failed"), 503), + (make_fastapi_http_exception(400, "Invalid scope parameter"), 400), + ], +) +def test_span_wrapper_failure_keeps_the_answered_status(otel_with_exporter, monkeypatch, raised, status_code): + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.management_helpers import utils as mgmt_utils + + otel, exporter = otel_with_exporter + monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False) + + server_span: Final = _server_span(otel, MODEL_LIST_PATH, headers={}) + request: Final = _request("GET", MODEL_LIST_PATH) + request.state.parent_otel_span = server_span + + @mgmt_utils.management_endpoint_span_wrapper + async def failing_model_list(request: Request, user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]: + raise raised + + async def call_and_answer() -> None: + try: + await failing_model_list(request=request, user_api_key_dict=_real_user_api_key_dict(server_span)) + except Exception as exc: + await _answer_like_the_proxy(request, exc) + + asyncio.run(call_and_answer()) + + assert_server_span_attrs( + exporter, + expected_status=status_code, + expected_url_path=MODEL_LIST_PATH, + where=f"model list {type(raised).__name__}", + ) + exported: Final = get_server_span(exporter) + assert exported is not None + assert exported.attributes.get("error.code") == str(status_code) + + +def _server_span(otel, path: str, headers: dict[str, str]): + server_span: Final = otel.create_litellm_proxy_request_started_span(start_time=datetime.now(), headers=headers) + otel.set_proxy_request_route_attributes(server_span, url_path=path, http_route=path) + return server_span + + +def _assert_ended_in_inbound_trace(exporter, path: str, handler_name: str) -> None: + assert_server_span_attrs(exporter, expected_status=200, expected_url_path=path, where=f"{path} success") + exported: Final = get_server_span(exporter) + assert exported is not None + assert exported.context.trace_id == int(INBOUND_TRACEPARENT.split("-")[1], 16) + + endpoint_spans: Final = [s for s in exporter.get_finished_spans() if s.name == handler_name] + assert len(endpoint_spans) == 1 + assert endpoint_spans[0].parent is not None + assert endpoint_spans[0].parent.span_id == exported.context.span_id + assert not any(key.startswith("response.") for key in endpoint_spans[0].attributes) + + +def _install_proxy_state(monkeypatch, router: Router | None) -> None: + import litellm.proxy.proxy_server as proxy_server + + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "user_model", None) def test_model_list_ends_server_span_in_inbound_trace(otel_with_exporter, monkeypatch): @@ -405,31 +518,29 @@ def test_model_list_ends_server_span_in_inbound_trace(otel_with_exporter, monkey otel, exporter = otel_with_exporter monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False) + _install_proxy_state(monkeypatch, None) + server_span: Final = _server_span(otel, MODEL_LIST_PATH, headers={"traceparent": INBOUND_TRACEPARENT}) - server_span: Final = otel.create_litellm_proxy_request_started_span( - start_time=datetime.now(), headers={"traceparent": INBOUND_TRACEPARENT} - ) - otel.set_proxy_request_route_attributes(server_span, url_path=MODEL_LIST_PATH, http_route=MODEL_LIST_PATH) - - result: Final = asyncio.run( - proxy_server.model_list( - user_api_key_dict=_real_user_api_key_dict(server_span), - ) - ) + result: Final = asyncio.run(proxy_server.model_list(user_api_key_dict=_real_user_api_key_dict(server_span))) assert result == {"data": [], "object": "list"} - assert_server_span_attrs( - exporter, - expected_status=200, - expected_url_path=MODEL_LIST_PATH, - where="model list success", - ) - exported: Final = get_server_span(exporter) - assert exported is not None - assert exported.context.trace_id == int(INBOUND_TRACEPARENT.split("-")[1], 16) + _assert_ended_in_inbound_trace(exporter, MODEL_LIST_PATH, "model_list") - endpoint_spans: Final = [s for s in exporter.get_finished_spans() if s.name == "model_list"] - assert len(endpoint_spans) == 1 - assert endpoint_spans[0].parent is not None - assert endpoint_spans[0].parent.span_id == exported.context.span_id - assert not any(key.startswith("response.") for key in endpoint_spans[0].attributes) + +def test_model_retrieve_ends_server_span_in_inbound_trace(otel_with_exporter, monkeypatch): + import litellm.proxy.proxy_server as proxy_server + + otel, exporter = otel_with_exporter + monkeypatch.setattr(proxy_server, "open_telemetry_logger", otel, raising=False) + _install_proxy_state( + monkeypatch, + Router(model_list=[{"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-x"}}]), + ) + server_span: Final = _server_span(otel, MODEL_RETRIEVE_PATH, headers={"traceparent": INBOUND_TRACEPARENT}) + + result: Final = asyncio.run( + proxy_server.model_info(model_id="gpt-4o", user_api_key_dict=_real_user_api_key_dict(server_span)) + ) + + assert result["id"] == "gpt-4o" + _assert_ended_in_inbound_trace(exporter, MODEL_RETRIEVE_PATH, "model_info")