From 8b33da7bb36e66c0168de4388ccbae75e86dd752 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 19:17:18 -0700 Subject: [PATCH] feat(proxy): opt-in litellm_call_id in JSON error bodies (#42391) * feat(proxy): opt-in litellm_call_id in JSON error bodies Add general_settings.include_call_id_in_error_body. When true, the value already on the x-litellm-call-id response header is copied into JSON error bodies: as error.litellm_call_id on the OpenAI-shaped routes, /v1/messages, and streaming first-chunk errors, and as a top-level litellm_call_id on pass-through routes. Off by default, so error bodies stay byte-identical unless an admin opts in * chore(proxy): drop helper docstring and restore lazy OpenAPI snapshot --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../exceptions/exceptions.py | 1 + litellm/proxy/_types.py | 4 + .../proxy/anthropic_endpoints/endpoints.py | 33 ++++++- litellm/proxy/common_request_processing.py | 15 +++- .../proxy/common_utils/error_body_call_id.py | 20 +++++ .../pass_through_endpoints.py | 20 ++++- litellm/proxy/proxy_server.py | 14 ++- .../anthropic_endpoints/test_endpoints.py | 56 ++++++++++++ .../common_utils/test_error_body_call_id.py | 35 ++++++++ .../test_pass_through_endpoints.py | 84 +++++++++++++++++ .../proxy_server/test_exception_handlers.py | 65 ++++++++++++++ .../proxy/test_common_request_processing.py | 89 +++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 5 ++ 13 files changed, 433 insertions(+), 8 deletions(-) create mode 100644 litellm/proxy/common_utils/error_body_call_id.py create mode 100644 tests/test_litellm/proxy/common_utils/test_error_body_call_id.py diff --git a/litellm/anthropic_interface/exceptions/exceptions.py b/litellm/anthropic_interface/exceptions/exceptions.py index 91bcf82f455..b48cd2fee6f 100644 --- a/litellm/anthropic_interface/exceptions/exceptions.py +++ b/litellm/anthropic_interface/exceptions/exceptions.py @@ -25,6 +25,7 @@ class AnthropicErrorDetail(TypedDict): type: AnthropicErrorType message: str provider_specific_fields: NotRequired[ReadOnly[Mapping[str, object]]] + litellm_call_id: NotRequired[ReadOnly[str]] class AnthropicErrorResponse(TypedDict, total=False): diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 70e49bb18b9..a76dd983930 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2648,6 +2648,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="opt-in to RFC 8628 verification_uri_complete for the CLI SSO device flow, pre-filling the user_code in the browser. Off by default; intended for same-host clients where the device that starts the flow and the browser run on the same machine", ) + include_call_id_in_error_body: bool | None = Field( + None, + description="opt-in to copy the x-litellm-call-id response header's value into JSON error bodies, as error.litellm_call_id on the OpenAI-shaped and /v1/messages routes and as a top-level litellm_call_id on pass-through routes, so an error a client prints names the request to look up. Off by default", + ) enable_claude_code_gateway: bool | None = Field( None, description="serve the Claude Code gateway protocol (https://code.claude.com/docs/en/claude-apps-gateway) under /claude_code_gateway: OAuth device-flow sign-in reusing proxy SSO, plus managed settings and OTLP telemetry ingestion. Off by default", diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 644778bcb9f..d9558b86e95 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -8,7 +8,11 @@ from fastapi import APIRouter, Depends, HTTPException, Request, Response from fastapi.responses import JSONResponse import litellm -from litellm.anthropic_interface.exceptions import AnthropicErrorResponse, AnthropicExceptionMapping +from litellm.anthropic_interface.exceptions import ( + AnthropicErrorDetail, + AnthropicErrorResponse, + AnthropicExceptionMapping, +) from litellm.integrations.custom_guardrail import ModifyResponseException from litellm.llms.anthropic.experimental_pass_through.context_management import ( AnthropicContextManagementError, @@ -25,8 +29,10 @@ from litellm.proxy.common_request_processing import ( proxy_exception_from_http_exception, resolve_litellm_call_id, ) +from litellm.proxy.common_utils.error_body_call_id import error_body_call_id from litellm.proxy.common_utils.http_parsing_utils import _read_request_body from litellm.proxy.common_utils.openai_error_payload import ( + LITELLM_CALL_ID_HEADER, error_status_code, openai_error_param, openai_error_type, @@ -37,9 +43,29 @@ from litellm.types.utils import TokenCountResponse router: Final = APIRouter() +def _with_provider_specific_fields(exc: ProxyException, detail: AnthropicErrorDetail) -> AnthropicErrorDetail: + if not exc.provider_specific_fields: + return detail + with_fields: Final[AnthropicErrorDetail] = {**detail, "provider_specific_fields": exc.provider_specific_fields} + return with_fields + + +def _anthropic_error_detail( + exc: ProxyException, detail: AnthropicErrorDetail, call_id: str | None +) -> AnthropicErrorDetail: + if call_id is None: + return _with_provider_specific_fields(exc, detail) + with_call_id: Final[AnthropicErrorDetail] = { + **_with_provider_specific_fields(exc, detail), + "litellm_call_id": call_id, + } + return with_call_id + + def _anthropic_error_json_response(exc: ProxyException, request: Request) -> JSONResponse: from litellm.proxy.proxy_server import ( _close_dangling_otel_server_span, # pyright: ignore[reportPrivateUsage] # proxy_server keeps the span-close helper private; error JSONResponses returned by the route must stamp the OTel server span like the global ProxyException handler does + general_settings_view, ) status_code: Final = int(exc.code) if exc.code is not None and exc.code.isdigit() else 500 @@ -49,11 +75,10 @@ def _anthropic_error_json_response(exc: ProxyException, request: Request) -> JSO raw_message=exc.message, request_id=request.headers.get("x-request-id"), ) - if not exc.provider_specific_fields: - return JSONResponse(status_code=status_code, content=envelope, headers=exc.headers) + body_call_id: Final = error_body_call_id(general_settings_view(), exc.headers.get(LITELLM_CALL_ID_HEADER)) content: Final[AnthropicErrorResponse] = { **envelope, - "error": {**envelope["error"], "provider_specific_fields": exc.provider_specific_fields}, + "error": _anthropic_error_detail(exc, envelope["error"], body_call_id), } return JSONResponse(status_code=status_code, content=content, headers=exc.headers) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 9484fd7c723..f6e0d56127f 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -75,11 +75,13 @@ from litellm.proxy.common_utils.callback_utils import ( get_logging_caching_headers, get_remaining_tokens_and_requests_from_request_data, ) +from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id from litellm.proxy.common_utils.http_parsing_utils import ( get_client_requested_model, get_tags_from_request_body, ) from litellm.proxy.common_utils.openai_error_payload import ( + LITELLM_CALL_ID_HEADER, attribute_of, error_status_code, openai_error_param, @@ -946,6 +948,9 @@ async def _resolve_stream_headers( return headers +_NO_GENERAL_SETTINGS: Final[Mapping[str, object]] = MappingProxyType({}) + + async def create_response( generator: AsyncGenerator[str, None], media_type: str, @@ -953,6 +958,7 @@ async def create_response( default_status_code: int = status.HTTP_200_OK, request: Request | None = None, refresh_headers: Callable[[], Awaitable[Mapping[str, str]]] | None = None, + general_settings: Mapping[str, object] = _NO_GENERAL_SETTINGS, ) -> StreamingResponse | JSONResponse: """ Create streaming response, checking if the first chunk is an error. @@ -960,7 +966,8 @@ async def create_response( Otherwise, return StreamingResponse and stream all content. ``refresh_headers`` is consulted once the first chunk has been buffered, for - callers whose headers can only be known then. + callers whose headers can only be known then. ``general_settings`` decides whether + the first-chunk error body also carries the ``x-litellm-call-id`` header's value. """ first_chunk_value: str | None = None final_status_code = default_status_code @@ -987,7 +994,10 @@ async def create_response( ) # Parse error content - error_dict: Final = _extract_error_from_sse_chunk(first_chunk_value) + error_dict: Final = with_call_id( + JSON_OBJECT.validate_python(_extract_error_from_sse_chunk(first_chunk_value)), + error_body_call_id(general_settings, resolved_headers.get(LITELLM_CALL_ID_HEADER)), + ) # Consume and close generator (avoid resource leak) try: @@ -2738,6 +2748,7 @@ class ProxyBaseLLMRequestProcessing: headers=custom_headers, request=request, refresh_headers=refresh_stream_headers, + general_settings=general_settings, ) ### CALL HOOKS ### - modify outgoing data diff --git a/litellm/proxy/common_utils/error_body_call_id.py b/litellm/proxy/common_utils/error_body_call_id.py new file mode 100644 index 00000000000..f50be5df509 --- /dev/null +++ b/litellm/proxy/common_utils/error_body_call_id.py @@ -0,0 +1,20 @@ +from collections.abc import Mapping +from typing import Final + +from pydantic import TypeAdapter + +INCLUDE_CALL_ID_IN_ERROR_BODY_SETTING: Final = "include_call_id_in_error_body" +LITELLM_CALL_ID_BODY_KEY: Final = "litellm_call_id" +JSON_OBJECT: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) # mutable-ok: JSONResponse input + + +def error_body_call_id(general_settings: Mapping[str, object], call_id: str | None) -> str | None: + if general_settings.get(INCLUDE_CALL_ID_IN_ERROR_BODY_SETTING) is not True: + return None + return call_id if call_id else None + + +def with_call_id(error: dict[str, object], call_id: str | None) -> dict[str, object]: # mutable-ok: JSONResponse input + if call_id is None: + return error + return {**error, LITELLM_CALL_ID_BODY_KEY: call_id} # mutable-ok: JSONResponse input diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 79a328f5199..bbbf5e4b94f 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -78,11 +78,13 @@ from litellm.proxy.common_request_processing import ( open_sse_before_first_byte, resolve_litellm_call_id, ) +from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id from litellm.proxy.common_utils.http_parsing_utils import ( _read_request_body, _safe_get_request_headers, ) from litellm.proxy.common_utils.openai_error_payload import ( + LITELLM_CALL_ID_HEADER, error_status_code, litellm_call_id_headers, openai_error_param, @@ -1127,6 +1129,9 @@ async def pass_through_request( from litellm.proxy.proxy_server import ( general_settings as proxy_general_settings, ) + from litellm.proxy.proxy_server import ( + general_settings_view, + ) _managed_id_provider: Final = resolve_passthrough_managed_id_provider(custom_llm_provider) @@ -1656,11 +1661,24 @@ async def pass_through_request( headers=response.headers, custom_headers=custom_headers, ) + emitted_call_id: Final = ( + JSON_OBJECT.validate_python(response_headers).get(LITELLM_CALL_ID_HEADER) + if response.status_code >= 400 + else None + ) + error_call_id: Final = ( + error_body_call_id(general_settings_view(), emitted_call_id) if isinstance(emitted_call_id, str) else None + ) + relayed_content: Final = ( + json.dumps(with_call_id(JSON_OBJECT.validate_python(response_body), error_call_id)).encode("utf-8") + if error_call_id is not None and isinstance(response_body, dict) + else content + ) if _content_modified: response_headers.pop("content-length", None) return Response( - content=content, + content=relayed_content, status_code=response.status_code, headers=response_headers, ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 534dcf418cc..87da8a6e44e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -393,6 +393,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, ) +from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id from litellm.proxy.common_utils.healthy_model_filter import ( get_hidden_unhealthy_model_names, is_healthy_only_listing_default, @@ -418,6 +419,7 @@ from litellm.proxy.common_utils.openai_endpoint_utils import ( remove_sensitive_info_from_deployment, ) from litellm.proxy.common_utils.openai_error_payload import ( + LITELLM_CALL_ID_HEADER, headers_with_litellm_call_id, litellm_call_id_headers, with_litellm_call_id, @@ -1813,7 +1815,10 @@ async def openai_exception_handler(request: Request, exc: ProxyException): # NOTE: DO NOT MODIFY THIS, its crucial to map to Openai exceptions _log_model_access_denial(exc) headers: Final = exc.headers - error_dict: Final = exc.to_dict() + error_dict: Final = with_call_id( + JSON_OBJECT.validate_python(exc.to_dict()), + error_body_call_id(general_settings_view(), headers.get(LITELLM_CALL_ID_HEADER)), + ) status_code: Final = int(exc.code) if exc.code else status.HTTP_500_INTERNAL_SERVER_ERROR _close_dangling_otel_server_span(request, status_code, exc=exc) return JSONResponse( @@ -2477,6 +2482,13 @@ heuristic_v1_tuning_baselines: Mapping[str, str] | None = None # second ProxyConfig instance must not get its own independent lock over it. MODEL_RECONCILE_LOCK: Final = asyncio.Lock() general_settings: dict = {} +_GENERAL_SETTINGS_VIEW: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) + + +def general_settings_view() -> Mapping[str, object]: + return _GENERAL_SETTINGS_VIEW.validate_python(general_settings) + + config_passthrough_endpoints: list[dict[str, Any]] | None = None log_file: Final = "api_log.json" worker_config: Final = None diff --git a/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py b/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py index 9a9ccd9a213..801f61aa498 100644 --- a/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/anthropic_endpoints/test_endpoints.py @@ -198,6 +198,62 @@ class TestProxyExceptionAnthropicEnvelope: assert fallback.status_code == 500 assert json.loads(fallback.body)["error"]["type"] == "api_error" + @staticmethod + def _call_id_error_response(general_settings, provider_specific_fields=None): + import litellm.proxy.anthropic_endpoints.endpoints as ep + from litellm.proxy._types import ProxyException + + request = MagicMock() + request.headers = {} + exc = ProxyException( + message="Rate limit exceeded", + type="rate_limit_error", + param=None, + code=429, + headers={"x-litellm-call-id": "call-8302"}, + provider_specific_fields=provider_specific_fields, + ) + with patch("litellm.proxy.proxy_server.general_settings", general_settings): + return ep._anthropic_error_json_response(exc, request) + + def test_anthropic_error_copies_the_call_id_into_the_error_when_opted_in(self): + """With include_call_id_in_error_body on, error.litellm_call_id is byte-identical to + the x-litellm-call-id header and lives inside the error object, which is what the + Anthropic SDK keeps as e.body.""" + response = self._call_id_error_response({"include_call_id_in_error_body": True}) + + assert response.headers["x-litellm-call-id"] == "call-8302" + assert json.loads(response.body) == { + "type": "error", + "error": { + "type": "rate_limit_error", + "message": "Rate limit exceeded", + "litellm_call_id": "call-8302", + }, + } + + def test_anthropic_error_keeps_provider_specific_fields_next_to_the_call_id(self): + response = self._call_id_error_response( + {"include_call_id_in_error_body": True}, + provider_specific_fields={"guardrail": "keyword-block"}, + ) + + assert json.loads(response.body)["error"] == { + "type": "rate_limit_error", + "message": "Rate limit exceeded", + "provider_specific_fields": {"guardrail": "keyword-block"}, + "litellm_call_id": "call-8302", + } + + def test_anthropic_error_leaves_the_envelope_alone_when_opted_out(self): + response = self._call_id_error_response({}) + + assert response.headers["x-litellm-call-id"] == "call-8302" + assert json.loads(response.body) == { + "type": "error", + "error": {"type": "rate_limit_error", "message": "Rate limit exceeded"}, + } + class TestHttpExceptionDictDetail: @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/common_utils/test_error_body_call_id.py b/tests/test_litellm/proxy/common_utils/test_error_body_call_id.py new file mode 100644 index 00000000000..8872b5de397 --- /dev/null +++ b/tests/test_litellm/proxy/common_utils/test_error_body_call_id.py @@ -0,0 +1,35 @@ +import pytest + +from litellm.proxy._types import ConfigGeneralSettings +from litellm.proxy.common_utils.error_body_call_id import error_body_call_id, with_call_id + + +@pytest.mark.parametrize( + "general_settings, call_id, expected", + [ + ({"include_call_id_in_error_body": True}, "call-1", "call-1"), + ({"include_call_id_in_error_body": True}, None, None), + ({"include_call_id_in_error_body": True}, "", None), + ({"include_call_id_in_error_body": False}, "call-1", None), + ({"include_call_id_in_error_body": "true"}, "call-1", None), + ({}, "call-1", None), + ], +) +def test_only_the_boolean_opt_in_with_a_real_id_yields_a_body_call_id(general_settings, call_id, expected): + """The setting is off by default and only a literal true turns it on; without an id + there is nothing to copy, so the body must never get a fabricated one.""" + assert error_body_call_id(general_settings, call_id) == expected + + +def test_with_call_id_appends_the_key_without_touching_the_input(): + error = {"message": "bad input", "type": "invalid_request_error", "param": None, "code": "400"} + + assert with_call_id(error, "call-1") == {**error, "litellm_call_id": "call-1"} + assert with_call_id(error, None) == error + assert "litellm_call_id" not in error + + +def test_the_setting_name_is_a_config_general_settings_field(): + """The yaml key the docs name and the key the runtime reads must be the same field.""" + assert ConfigGeneralSettings.model_validate({"include_call_id_in_error_body": True}).include_call_id_in_error_body + assert ConfigGeneralSettings.model_validate({}).include_call_id_in_error_body is None diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index fb89e3a6973..6cd537489ac 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -4651,6 +4651,90 @@ async def test_pass_through_request_upstream_error_body_stays_buffered(): await fake_client.aclose() +_UPSTREAM_JSON_ERROR: Final = b'{"error": {"message": "bad request", "type": "invalid_request_error"}}' + + +async def _relay_upstream_through_pass_through_request( + general_settings, status_code, content_type, body, callback_headers=None +): + from litellm.proxy._types import UserAPIKeyAuth + + fake_client, cleanup = _inject_fake_passthrough_client( + _FakeUpstreamTransport( + status_code=status_code, + headers={"content-type": content_type}, + stream=_RecordingUpstreamByteStream((body,)), + ), + timeout=313.0, + ) + try: + with ExitStack() as stack: + mock_proxy_logging, _ = _enter_relay_logging_mocks(stack, {}) + mock_proxy_logging.post_call_response_headers_hook = AsyncMock(return_value=callback_headers) + stack.enter_context(patch("litellm.proxy.proxy_server.general_settings", general_settings)) + return await pass_through_request( + request=_relay_client_request(), + target="http://upstream.test/v1/messages", + custom_headers={}, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-relay-test"), + timeout=313.0, + ) + finally: + cleanup() + await fake_client.aclose() + + +@pytest.mark.asyncio +async def test_pass_through_error_body_carries_the_call_id_when_opted_in(): + """With include_call_id_in_error_body on, a buffered upstream JSON error gets a top-level + litellm_call_id byte-identical to the x-litellm-call-id header, and content-length still + matches the rewritten body.""" + response = await _relay_upstream_through_pass_through_request( + {"include_call_id_in_error_body": True}, 400, "application/json", _UPSTREAM_JSON_ERROR + ) + + call_id = response.headers["x-litellm-call-id"] + assert response.status_code == 400 + assert json.loads(response.body) == {**json.loads(_UPSTREAM_JSON_ERROR), "litellm_call_id": call_id} + assert int(response.headers["content-length"]) == len(response.body) + + +@pytest.mark.asyncio +async def test_pass_through_error_body_call_id_follows_a_restamped_header(): + """A post_call_response_headers_hook that rewrites x-litellm-call-id wins in the header, so the + body copies the emitted header value rather than the id the proxy generated.""" + response = await _relay_upstream_through_pass_through_request( + {"include_call_id_in_error_body": True}, + 400, + "application/json", + _UPSTREAM_JSON_ERROR, + callback_headers={"x-litellm-call-id": "restamped-by-hook"}, + ) + + assert response.headers["x-litellm-call-id"] == "restamped-by-hook" + assert json.loads(response.body)["litellm_call_id"] == "restamped-by-hook" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "general_settings, status_code, content_type, body", + [ + ({}, 400, "application/json", _UPSTREAM_JSON_ERROR), + ({"include_call_id_in_error_body": True}, 502, "text/plain", b"upstream exploded"), + ({"include_call_id_in_error_body": True}, 200, "application/json", b'{"id": "msg_1", "type": "message"}'), + ], +) +async def test_pass_through_body_stays_byte_identical_outside_the_opt_in( + general_settings, status_code, content_type, body +): + """Opted out, a non-JSON error, or a success body: the upstream bytes are relayed as-is.""" + response = await _relay_upstream_through_pass_through_request(general_settings, status_code, content_type, body) + + assert response.status_code == status_code + assert response.body == body + assert "x-litellm-call-id" in response.headers + + _PARTIAL_RELAY_WARNING_MARKER = "ended before upstream body was fully relayed" diff --git a/tests/test_litellm/proxy/proxy_server/test_exception_handlers.py b/tests/test_litellm/proxy/proxy_server/test_exception_handlers.py index 53ea761daa7..b3028ae71dd 100644 --- a/tests/test_litellm/proxy/proxy_server/test_exception_handlers.py +++ b/tests/test_litellm/proxy/proxy_server/test_exception_handlers.py @@ -95,6 +95,71 @@ async def test_openai_exception_handler_invalid_empty_code_defaults_to_500(): } +def _call_id_exception(headers): + return ProxyException( + message="bad input", + type="invalid_request_error", + param="model", + code=400, + headers=headers, + ) + + +@pytest.mark.asyncio +async def test_openai_exception_handler_copies_the_call_id_into_the_error_when_opted_in(monkeypatch): + """With include_call_id_in_error_body on, error.litellm_call_id is byte-identical to the + x-litellm-call-id header, so a pasted str(e) names the request to look up.""" + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"include_call_id_in_error_body": True}) + exc = _call_id_exception({"x-litellm-call-id": "call-8302"}) + + response = await openai_exception_handler(request=_make_request(), exc=exc) + body = json.loads(response.body) + + assert response.headers["x-litellm-call-id"] == "call-8302" + assert body == { + "error": { + "message": "bad input", + "type": "invalid_request_error", + "param": "model", + "code": "400", + "litellm_call_id": "call-8302", + } + } + + +@pytest.mark.asyncio +async def test_openai_exception_handler_leaves_the_error_alone_when_opted_out(monkeypatch): + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + exc = _call_id_exception({"x-litellm-call-id": "call-8302"}) + + response = await openai_exception_handler(request=_make_request(), exc=exc) + body = json.loads(response.body) + + assert response.headers["x-litellm-call-id"] == "call-8302" + assert body == { + "error": { + "message": "bad input", + "type": "invalid_request_error", + "param": "model", + "code": "400", + } + } + + +@pytest.mark.asyncio +async def test_openai_exception_handler_never_fabricates_a_call_id(monkeypatch): + """An error raised before a call id exists (auth failures, say) carries no header, + and the body must not invent one.""" + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"include_call_id_in_error_body": True}) + exc = _call_id_exception({}) + + response = await openai_exception_handler(request=_make_request(), exc=exc) + body = json.loads(response.body) + + assert "x-litellm-call-id" not in response.headers + assert "litellm_call_id" not in body["error"] + + # --------------------------------------------------------------------------- # _close_dangling_otel_server_span # --------------------------------------------------------------------------- diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 0b872400be0..218d8246715 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -2343,6 +2343,39 @@ class TestCommonRequestProcessingHelpers: assert isinstance(response, JSONResponse) assert response.headers["x-litellm-model-id"] == "fallback-deployment" + @staticmethod + async def _first_chunk_error_response(**create_response_kwargs): + async def mock_generator(): + yield 'data: {"error": {"code": 403, "message": "forbidden"}}\n\n' + yield "data: [DONE]\n\n" + + return await create_response( + mock_generator(), + "text/event-stream", + {"x-litellm-call-id": "call-8302"}, + **create_response_kwargs, + ) + + async def test_create_response_first_chunk_error_carries_the_call_id_when_opted_in(self): + """A stream that fails on its first chunk answers as JSON, and with + include_call_id_in_error_body on that JSON names the request like the + non-streaming error path does, byte-identical to the header.""" + response = await self._first_chunk_error_response(general_settings={"include_call_id_in_error_body": True}) + + assert isinstance(response, JSONResponse) + assert response.status_code == 403 + assert response.headers["x-litellm-call-id"] == "call-8302" + assert json.loads(response.body) == { + "error": {"code": 403, "message": "forbidden", "litellm_call_id": "call-8302"} + } + + async def test_create_response_first_chunk_error_body_is_unchanged_by_default(self): + response = await self._first_chunk_error_response() + + assert isinstance(response, JSONResponse) + assert response.headers["x-litellm-call-id"] == "call-8302" + assert json.loads(response.body) == {"error": {"code": 403, "message": "forbidden"}} + async def test_create_streaming_response_disables_proxy_buffering(self): """Regression for #28384: every StreamingResponse create_response returns must carry the headers that stop nginx/ingress/Envoy from buffering the @@ -9122,6 +9155,62 @@ class TestStreamingResponseHeadersFollowFallback: assert result.status_code == 400 assert result.headers["x-litellm-applied-guardrails"] == "stream-blocker" + @pytest.mark.asyncio + async def test_streaming_first_chunk_error_carries_the_call_id_when_opted_in(self, monkeypatch): + """The opt-in reaches the streaming path through base_process_llm_request, so a stream + that fails on its first chunk answers with the call id inside its JSON error body, + byte-identical to the x-litellm-call-id header.""" + + def select_data_generator(**kwargs): + async def generator(): + yield 'data: {"error": {"code": 403, "message": "forbidden"}}\n\n' + yield "data: [DONE]\n\n" + + return generator() + + logging_obj = MagicMock() + logging_obj.litellm_call_id = "lit-8302-call" + logging_obj._defer_async_logging = False + logging_obj._on_deferred_stream_complete = None + logging_obj.cost_breakdown = None + processor = ProxyBaseLLMRequestProcessing( + data={"model": "oa", "stream": True, "litellm_logging_obj": logging_obj} + ) + + proxy_logging_obj = MagicMock(spec=ProxyLogging) + proxy_logging_obj.during_call_hook = AsyncMock(return_value=None) + proxy_logging_obj.update_request_status = AsyncMock(return_value=None) + proxy_logging_obj.post_call_success_hook = AsyncMock( + side_effect=lambda data, user_api_key_dict, response: response + ) + proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={}) + + async def fake_route_request(**kwargs): + async def call(): + return SimpleNamespace(_hidden_params={}, fallback_headers_adopted=False) + + return call() + + monkeypatch.setattr(litellm.proxy.common_request_processing, "route_request", fake_route_request) + + result = await processor.base_process_llm_request( + request=Request(scope={"type": "http", "headers": []}), + fastapi_response=Response(), + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + route_type="acompletion", + proxy_logging_obj=proxy_logging_obj, + general_settings={"include_call_id_in_error_body": True}, + proxy_config=MagicMock(spec=ProxyConfig), + select_data_generator=select_data_generator, + is_streaming_request=True, + skip_pre_call_logic=True, + ) + + assert isinstance(result, JSONResponse) + assert result.status_code == 403 + assert result.headers["x-litellm-call-id"] == "lit-8302-call" + assert json.loads(result.body)["error"]["litellm_call_id"] == "lit-8302-call" + class _MessagesFallbackStream: def __init__(self) -> None: diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 08b7ad5706a..b2b16038f27 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -27119,6 +27119,11 @@ export interface components { * @default false */ health_check_skip_disabled_background_models: boolean; + /** + * Include Call Id In Error Body + * @description opt-in to copy the x-litellm-call-id response header's value into JSON error bodies, as error.litellm_call_id on the OpenAI-shaped and /v1/messages routes and as a top-level litellm_call_id on pass-through routes, so an error a client prints names the request to look up. Off by default + */ + include_call_id_in_error_body?: boolean | null; /** * Infer Model From Keys * @description for `/models` endpoint, infers available model based on environment keys (e.g. OPENAI_API_KEY)