From ea51e77dc5bd718e16de370ae32badf955d3ce7f Mon Sep 17 00:00:00 2001 From: mika-lt <349045450+mika-lt@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:39:56 -0700 Subject: [PATCH] fix(responses)!: keep the proxy's error body on litellm.responses errors from litellm_proxy models (#49073) * fix(responses): keep the proxy's error body on litellm.responses errors from litellm_proxy models litellm.responses() with a litellm_proxy/ model raised BadRequestError with body None when the proxy answered a 4xx, while litellm.completion() on the same proxy carries the decoded error object. The litellm_proxy Responses config now decodes the error JSON into the exception body the way the OpenAI SDK does, so body, code, type and the message match chat completions on sync, async and streaming calls. * fix(proxy): relay a downstream proxy's provider_specific_fields in error responses A LiteLLM proxy routing to another LiteLLM proxy now copies the downstream's provider_specific_fields (which guardrail fired) from the exception body into its own error answer, on chat and Responses alike. Adds tests for the two _openai_sdk_error_body branches the existing tests left open. * fix(proxy): redact internal details from relayed provider_specific_fields * fix(proxy): redact relayed provider_specific_fields per key so credential-named values stay hidden * fix(proxy): redact numeric credentials in relayed provider_specific_fields --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/_logging.py | 28 +++++ .../litellm_proxy/responses/transformation.py | 32 ++++++ litellm/proxy/common_request_processing.py | 4 +- .../common_utils/openai_error_payload.py | 16 +++ .../code_coverage_tests/recursive_detector.py | 1 + .../responses/test_transformation.py | 84 +++++++++++++++ .../proxy/test_common_request_processing.py | 101 ++++++++++++++++++ 7 files changed, 265 insertions(+), 1 deletion(-) diff --git a/litellm/_logging.py b/litellm/_logging.py index 5279cb233b4..6a38eea2b12 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -28,9 +28,11 @@ from litellm.litellm_core_utils.env_utils import get_env_int from litellm.litellm_core_utils.safe_json_dumps import UNSERIALIZABLE_OBJECT, safe_dumps, safe_json_structure from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.secret_redaction import ( + REDACTED, python_redact_string, python_redact_structured_value, redact_internal_details, + redact_structured_value, ) from litellm.litellm_core_utils.secret_redaction import ( redact_string as redact_secret_string, @@ -140,6 +142,32 @@ def redact_internal_details_from_client_message(value: str) -> str: return redact_internal_details(value) +_CLIENT_FIELDS: Final = TypeAdapter(dict[str, object]) +_CLIENT_ITEMS: Final = TypeAdapter(list[object]) + + +def _redacted_client_field(key: str | None, value: object) -> object: + if isinstance(value, str): + return redact_internal_details(redact_structured_value(key, value)) + if isinstance(value, dict): + return { + child_key: _redacted_client_field(child_key, child) + for child_key, child in _CLIENT_FIELDS.validate_python(value).items() + } + if isinstance(value, (list, tuple)): + return [_redacted_client_field(key, child) for child in _CLIENT_ITEMS.validate_python(value)] + if not isinstance(value, (int, float)): + return value + text: Final = str(value) + return value if redact_structured_value(key, text) == text else REDACTED + + +def redact_internal_details_from_client_fields(fields: Mapping[str, object]) -> object: + if not _ENABLE_SECRET_REDACTION: + return fields + return _redacted_client_field(None, dict(fields)) + + def _substituted_color_message(record: logging.LogRecord) -> str | None: """Render a record's ``color_message`` against its args, or None if absent. diff --git a/litellm/llms/litellm_proxy/responses/transformation.py b/litellm/llms/litellm_proxy/responses/transformation.py index 3092b49b743..fc94eb30504 100644 --- a/litellm/llms/litellm_proxy/responses/transformation.py +++ b/litellm/llms/litellm_proxy/responses/transformation.py @@ -5,11 +5,30 @@ LiteLLM Proxy supports the OpenAI Responses API natively when the underlying mod This config enables pass-through behavior to the proxy's /v1/responses endpoint. """ +from typing import Final + +import httpx +from pydantic import TypeAdapter, ValidationError + from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str from litellm.types.utils import LlmProviders +_ERROR_BODY: Final = TypeAdapter(dict[str, object]) + + +def _openai_sdk_error_body( + error_message: str, +) -> dict[str, object] | None: # mutable-ok: BaseLLMException.body is a dict + try: + decoded: Final = _ERROR_BODY.validate_json(error_message) + except ValidationError: + return None + nested: Final = decoded.get("error") + return _ERROR_BODY.validate_python(nested) if isinstance(nested, dict) else decoded + class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig): """ @@ -43,6 +62,19 @@ class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig): return join_api_base(api_base, "responses") + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict[str, object] | httpx.Headers, # mutable-ok: inherited contract + ) -> BaseLLMException: + return BaseLLMException( + status_code=status_code, + message=error_message, + headers=headers, + body=_openai_sdk_error_body(error_message), + ) + def supports_native_websocket(self) -> bool: """LiteLLM Proxy does not support native WebSocket for Responses API""" return False diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 666269be2eb..7cd752c2297 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -114,6 +114,7 @@ from litellm.proxy.common_utils.openai_error_payload import ( error_status_code, openai_error_param, openai_error_type, + relayed_provider_specific_fields, ) from litellm.proxy.common_utils.sse_keepalive import ( SSE_COMMENT_PING_BYTES, @@ -4462,7 +4463,8 @@ class ProxyBaseLLMRequestProcessing: param=openai_error_param(e), openai_code=getattr(e, "code", None), code=_code, - provider_specific_fields=getattr(e, "provider_specific_fields", None), + provider_specific_fields=getattr(e, "provider_specific_fields", None) + or relayed_provider_specific_fields(e), headers=safe_headers, ) diff --git a/litellm/proxy/common_utils/openai_error_payload.py b/litellm/proxy/common_utils/openai_error_payload.py index 3c91e87ee51..f92fbeef9a1 100644 --- a/litellm/proxy/common_utils/openai_error_payload.py +++ b/litellm/proxy/common_utils/openai_error_payload.py @@ -8,7 +8,9 @@ from types import MappingProxyType from typing import Final from fastapi import HTTPException, status +from pydantic import TypeAdapter +from litellm._logging import redact_internal_details_from_client_fields from litellm.constants import STRINGIFIED_NONE from litellm.proxy._types import ProxyErrorTypes, ProxyException, proxy_exception_status_code from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded @@ -26,6 +28,8 @@ _OPENAI_ERROR_TYPE_BY_STATUS: Final[Mapping[int, str]] = MappingProxyType( _AUTH_FAILURE_STATUS_CODES: Final = frozenset({status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN}) +_ERROR_BODY: Final = TypeAdapter(dict[str, object]) + _FAILOVER_SQLSTATES: Final = frozenset({"25006", "57P01", "57P02", "57P03"}) _FAILOVER_REFUSALS: Final = ( @@ -77,6 +81,18 @@ def openai_error_param(exc: object) -> str | None: return carried if isinstance(carried, str) and carried != STRINGIFIED_NONE else None +def relayed_provider_specific_fields( + exc: object, +) -> dict[str, object] | None: # mutable-ok: ProxyException.provider_specific_fields is a dict + body: Final = attribute_of(exc, "body") + if not isinstance(body, dict): + return None + fields: Final = _ERROR_BODY.validate_python(body).get("provider_specific_fields") + if not isinstance(fields, dict): + return None + return _ERROR_BODY.validate_python(redact_internal_details_from_client_fields(_ERROR_BODY.validate_python(fields))) + + def proxy_exception_for(exc: Exception, default_status_code: int) -> ProxyException: match exc: case ProxyException(): diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index a5bb5e0b340..c943ff2cbd0 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -81,6 +81,7 @@ IGNORE_FUNCTIONS = [ "completion_cost", # max depth 1: recursion only fires for mixed-tier Responses WS logging objects, and each split part carries a single service_tier so _split_responses_ws_logging_object_by_service_tier returns None. "_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible). "_replace_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible). + "_redacted_client_field", # bounded by the nesting depth of a JSON-decoded error body (a finite JSON tree, no cycles possible). "_sort_processed_sets", # bounded by the nesting depth of the log-record extra it walks (a finite JSON tree, no cycles possible). "scrub_json_strings", # max depth set (MAX_SCRUB_DEPTH); fails closed by returning "[Filtered]" for anything nested past the cap. "_without_requester_ip_address", # walks only the fresh tree _sanitize_request_body_for_spend_logs_payload returns (mapping cycles already cut), so it never goes deeper than that sanitizer did. diff --git a/tests/unit/llms/litellm_proxy/responses/test_transformation.py b/tests/unit/llms/litellm_proxy/responses/test_transformation.py index 3d774b79eed..5537d9d1916 100644 --- a/tests/unit/llms/litellm_proxy/responses/test_transformation.py +++ b/tests/unit/llms/litellm_proxy/responses/test_transformation.py @@ -1,4 +1,5 @@ import json +from collections.abc import Callable, Iterable from typing import Final import httpx @@ -94,3 +95,86 @@ def test_compact_request_forwards_metadata_and_user_to_the_downstream_proxy(drop "user": "end-user-1", "temperature": 0.2, } + + +_GUARDRAIL_BLOCK: Final = { + "error": { + "message": "Content blocked: secret_project_codename pattern detected", + "type": "invalid_request_error", + "param": None, + "code": "400", + "provider_specific_fields": {"guardrail_name": "block-secret-project", "guardrail_mode": "pre_call"}, + } +} + + +def _proxy_rejection(route: str, response: httpx.Response, call: Callable[[], object]) -> litellm.BadRequestError: + with respx.mock(assert_all_called=True) as mock: + _ = mock.post(f"https://my-proxy.example.com/{route}").mock(return_value=response) + with pytest.raises(litellm.BadRequestError) as caught: + _ = call() + return caught.value + + +def _consume(stream: Iterable[object]) -> None: + for _ in stream: + pass + + +_PROXY_PARAMS: Final = {"api_base": "https://my-proxy.example.com/", "api_key": "sk-proxy-key"} + + +@pytest.mark.usefixtures(httpx_transport.__name__) +@pytest.mark.parametrize("stream", [False, True]) +def test_proxy_400_reaches_responses_caller_with_the_same_body_as_chat_completions(stream: bool) -> None: + chat_error: Final = _proxy_rejection( + "chat/completions", + httpx.Response(400, json=_GUARDRAIL_BLOCK), + lambda: litellm.completion( + model="litellm_proxy/gpt-5.5", messages=[{"role": "user", "content": "project bluebird"}], **_PROXY_PARAMS + ), + ) + responses_error: Final = _proxy_rejection( + "responses", + httpx.Response(400, json=_GUARDRAIL_BLOCK), + lambda: ( + _consume(litellm.responses(model="litellm_proxy/gpt-5.5", input="project bluebird", stream=True, **_PROXY_PARAMS)) + if stream + else litellm.responses(model="litellm_proxy/gpt-5.5", input="project bluebird", **_PROXY_PARAMS) + ), + ) + assert responses_error.body == chat_error.body == _GUARDRAIL_BLOCK["error"] + assert (responses_error.code, responses_error.type) == (chat_error.code, chat_error.type) + assert str(responses_error) == str(chat_error) + + +@pytest.mark.usefixtures(httpx_transport.__name__) +def test_proxy_error_without_a_json_object_body_still_raises_with_its_text() -> None: + error: Final = _proxy_rejection( + "responses", + httpx.Response(400, text="upstream proxy rejected the request"), + lambda: litellm.responses(model="litellm_proxy/gpt-5.5", input="hi", **_PROXY_PARAMS), + ) + assert error.body is None + assert "upstream proxy rejected the request" in str(error) + + +@pytest.mark.usefixtures(httpx_transport.__name__) +def test_proxy_error_object_without_an_error_member_is_the_body_as_sent() -> None: + sent: Final = {"message": "upstream proxy rejected the request", "type": "invalid_request_error"} + error: Final = _proxy_rejection( + "responses", + httpx.Response(400, json=sent), + lambda: litellm.responses(model="litellm_proxy/gpt-5.5", input="hi", **_PROXY_PARAMS), + ) + assert error.body == sent + + +@pytest.mark.usefixtures(httpx_transport.__name__) +def test_proxy_error_whose_error_member_is_text_still_raises_with_that_text() -> None: + error: Final = _proxy_rejection( + "responses", + httpx.Response(400, json={"error": "upstream proxy rejected the request"}), + lambda: litellm.responses(model="litellm_proxy/gpt-5.5", input="hi", **_PROXY_PARAMS), + ) + assert "upstream proxy rejected the request" in str(error) diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index cc1fc5cd6c3..aeab7a3d348 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -14,6 +14,7 @@ from tests.unit.proxy.router_entry_point_stubs import route_entry_points_to_stub import anthropic import httpx import pytest +import respx from fastapi import HTTPException, Request, Response, status from fastapi.responses import JSONResponse, StreamingResponse @@ -12516,3 +12517,103 @@ def test_maybe_get_model_id_prefers_litellm_metadata_over_caller_metadata_withou def test_maybe_get_model_id_returns_none_before_the_router_picked_a_deployment(): assert ProxyBaseLLMRequestProcessing(data={"metadata": {}}).maybe_get_model_id(None) is None + + +_DOWNSTREAM_GUARDRAIL_FIELDS: Final = MappingProxyType( + {"guardrail_name": "block-secret-project", "guardrail_mode": "pre_call", "pattern": "secret_project_codename"} +) + + +def _downstream_proxy_rejection( + route: str, call: Callable[[], object], fields: Mapping[str, object] = _DOWNSTREAM_GUARDRAIL_FIELDS +) -> Exception: + body: Final = { + "error": { + "message": "Content blocked: secret_project_codename pattern detected", + "type": "invalid_request_error", + "param": None, + "code": "400", + "provider_specific_fields": dict(fields), + } + } + with respx.mock(assert_all_called=True) as mock: + _ = mock.post(f"https://downstream-proxy.example.com/{route}").mock(return_value=httpx.Response(400, json=body)) + with pytest.raises(litellm.BadRequestError) as caught: + _ = call() + return caught.value + + +_DOWNSTREAM: Final = MappingProxyType( + {"api_base": "https://downstream-proxy.example.com/", "api_key": "sk-downstream-key"} +) + + +@pytest.mark.usefixtures("httpx_transport") +@pytest.mark.parametrize( + ("route", "call"), + [ + ( + "responses", + lambda: litellm.responses(model="litellm_proxy/gpt-5.5", input="project bluebird", **_DOWNSTREAM), + ), + ( + "chat/completions", + lambda: litellm.completion( + model="litellm_proxy/gpt-5.5", messages=[{"role": "user", "content": "project bluebird"}], **_DOWNSTREAM + ), + ), + ], +) +async def test_a_chained_proxy_relays_which_downstream_guardrail_blocked_the_request( + route: str, call: Callable[[], object] +) -> None: + rejection: Final = _downstream_proxy_rejection(route, call) + with pytest.raises(ProxyException) as relayed: + await ProxyBaseLLMRequestProcessing(data={"model": "chained"})._handle_llm_api_exception( + e=rejection, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-front"), + proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()), + ) + assert relayed.value.to_dict()["provider_specific_fields"] == dict(_DOWNSTREAM_GUARDRAIL_FIELDS) + + +_DOWNSTREAM_LEAKABLE_VALUES: Final = ( + "guard-scan.internal.corp", + "/srv/secret/app/guard.py", + "10.0.4.17", + "abcd1234efgh5678", + "cs-0123456789", + "tok-0123456789abcdef", + "4815162342108", +) + + +@pytest.mark.usefixtures("httpx_transport") +async def test_a_chained_proxy_redacts_secrets_and_internal_details_from_relayed_guardrail_fields() -> None: + fields: Final = { + **_DOWNSTREAM_GUARDRAIL_FIELDS, + "exception": "connect to http://guard-scan.internal.corp:9000/v1/check failed at /srv/secret/app/guard.py", + "api_key": "abcd1234efgh5678", + "access_token": ["tok-0123456789abcdef"], + "scanner": { + "hosts": ["10.0.4.17:8080"], + "client_secret": "cs-0123456789", + "api_secret": 4815162342108, + "attempts": 2, + }, + } + rejection: Final = _downstream_proxy_rejection( + "responses", + lambda: litellm.responses(model="litellm_proxy/gpt-5.5", input="project bluebird", **_DOWNSTREAM), + fields, + ) + with pytest.raises(ProxyException) as relayed: + await ProxyBaseLLMRequestProcessing(data={"model": "chained"})._handle_llm_api_exception( + e=rejection, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-front"), + proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()), + ) + answered: Final = json.loads(json.dumps(relayed.value.to_dict()["provider_specific_fields"])) + assert {key: answered[key] for key in _DOWNSTREAM_GUARDRAIL_FIELDS} == dict(_DOWNSTREAM_GUARDRAIL_FIELDS) + assert answered["scanner"]["attempts"] == 2 + assert [value for value in _DOWNSTREAM_LEAKABLE_VALUES if value in json.dumps(answered)] == []