diff --git a/litellm/_logging.py b/litellm/_logging.py index 5279cb233b4..6a38eea2b12 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -28,9 +28,11 @@ from litellm.litellm_core_utils.env_utils import get_env_int from litellm.litellm_core_utils.safe_json_dumps import UNSERIALIZABLE_OBJECT, safe_dumps, safe_json_structure from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.secret_redaction import ( + REDACTED, python_redact_string, python_redact_structured_value, redact_internal_details, + redact_structured_value, ) from litellm.litellm_core_utils.secret_redaction import ( redact_string as redact_secret_string, @@ -140,6 +142,32 @@ def redact_internal_details_from_client_message(value: str) -> str: return redact_internal_details(value) +_CLIENT_FIELDS: Final = TypeAdapter(dict[str, object]) +_CLIENT_ITEMS: Final = TypeAdapter(list[object]) + + +def _redacted_client_field(key: str | None, value: object) -> object: + if isinstance(value, str): + return redact_internal_details(redact_structured_value(key, value)) + if isinstance(value, dict): + return { + child_key: _redacted_client_field(child_key, child) + for child_key, child in _CLIENT_FIELDS.validate_python(value).items() + } + if isinstance(value, (list, tuple)): + return [_redacted_client_field(key, child) for child in _CLIENT_ITEMS.validate_python(value)] + if not isinstance(value, (int, float)): + return value + text: Final = str(value) + return value if redact_structured_value(key, text) == text else REDACTED + + +def redact_internal_details_from_client_fields(fields: Mapping[str, object]) -> object: + if not _ENABLE_SECRET_REDACTION: + return fields + return _redacted_client_field(None, dict(fields)) + + def _substituted_color_message(record: logging.LogRecord) -> str | None: """Render a record's ``color_message`` against its args, or None if absent. diff --git a/litellm/llms/litellm_proxy/responses/transformation.py b/litellm/llms/litellm_proxy/responses/transformation.py index 3092b49b743..fc94eb30504 100644 --- a/litellm/llms/litellm_proxy/responses/transformation.py +++ b/litellm/llms/litellm_proxy/responses/transformation.py @@ -5,11 +5,30 @@ LiteLLM Proxy supports the OpenAI Responses API natively when the underlying mod This config enables pass-through behavior to the proxy's /v1/responses endpoint. """ +from typing import Final + +import httpx +from pydantic import TypeAdapter, ValidationError + from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str from litellm.types.utils import LlmProviders +_ERROR_BODY: Final = TypeAdapter(dict[str, object]) + + +def _openai_sdk_error_body( + error_message: str, +) -> dict[str, object] | None: # mutable-ok: BaseLLMException.body is a dict + try: + decoded: Final = _ERROR_BODY.validate_json(error_message) + except ValidationError: + return None + nested: Final = decoded.get("error") + return _ERROR_BODY.validate_python(nested) if isinstance(nested, dict) else decoded + class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig): """ @@ -43,6 +62,19 @@ class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig): return join_api_base(api_base, "responses") + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict[str, object] | httpx.Headers, # mutable-ok: inherited contract + ) -> BaseLLMException: + return BaseLLMException( + status_code=status_code, + message=error_message, + headers=headers, + body=_openai_sdk_error_body(error_message), + ) + def supports_native_websocket(self) -> bool: """LiteLLM Proxy does not support native WebSocket for Responses API""" return False diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 666269be2eb..7cd752c2297 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -114,6 +114,7 @@ from litellm.proxy.common_utils.openai_error_payload import ( error_status_code, openai_error_param, openai_error_type, + relayed_provider_specific_fields, ) from litellm.proxy.common_utils.sse_keepalive import ( SSE_COMMENT_PING_BYTES, @@ -4462,7 +4463,8 @@ class ProxyBaseLLMRequestProcessing: param=openai_error_param(e), openai_code=getattr(e, "code", None), code=_code, - provider_specific_fields=getattr(e, "provider_specific_fields", None), + provider_specific_fields=getattr(e, "provider_specific_fields", None) + or relayed_provider_specific_fields(e), headers=safe_headers, ) diff --git a/litellm/proxy/common_utils/openai_error_payload.py b/litellm/proxy/common_utils/openai_error_payload.py index 3c91e87ee51..f92fbeef9a1 100644 --- a/litellm/proxy/common_utils/openai_error_payload.py +++ b/litellm/proxy/common_utils/openai_error_payload.py @@ -8,7 +8,9 @@ from types import MappingProxyType from typing import Final from fastapi import HTTPException, status +from pydantic import TypeAdapter +from litellm._logging import redact_internal_details_from_client_fields from litellm.constants import STRINGIFIED_NONE from litellm.proxy._types import ProxyErrorTypes, ProxyException, proxy_exception_status_code from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded @@ -26,6 +28,8 @@ _OPENAI_ERROR_TYPE_BY_STATUS: Final[Mapping[int, str]] = MappingProxyType( _AUTH_FAILURE_STATUS_CODES: Final = frozenset({status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN}) +_ERROR_BODY: Final = TypeAdapter(dict[str, object]) + _FAILOVER_SQLSTATES: Final = frozenset({"25006", "57P01", "57P02", "57P03"}) _FAILOVER_REFUSALS: Final = ( @@ -77,6 +81,18 @@ def openai_error_param(exc: object) -> str | None: return carried if isinstance(carried, str) and carried != STRINGIFIED_NONE else None +def relayed_provider_specific_fields( + exc: object, +) -> dict[str, object] | None: # mutable-ok: ProxyException.provider_specific_fields is a dict + body: Final = attribute_of(exc, "body") + if not isinstance(body, dict): + return None + fields: Final = _ERROR_BODY.validate_python(body).get("provider_specific_fields") + if not isinstance(fields, dict): + return None + return _ERROR_BODY.validate_python(redact_internal_details_from_client_fields(_ERROR_BODY.validate_python(fields))) + + def proxy_exception_for(exc: Exception, default_status_code: int) -> ProxyException: match exc: case ProxyException(): diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index a5bb5e0b340..c943ff2cbd0 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -81,6 +81,7 @@ IGNORE_FUNCTIONS = [ "completion_cost", # max depth 1: recursion only fires for mixed-tier Responses WS logging objects, and each split part carries a single service_tier so _split_responses_ws_logging_object_by_service_tier returns None. "_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible). "_replace_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible). + "_redacted_client_field", # bounded by the nesting depth of a JSON-decoded error body (a finite JSON tree, no cycles possible). "_sort_processed_sets", # bounded by the nesting depth of the log-record extra it walks (a finite JSON tree, no cycles possible). "scrub_json_strings", # max depth set (MAX_SCRUB_DEPTH); fails closed by returning "[Filtered]" for anything nested past the cap. "_without_requester_ip_address", # walks only the fresh tree _sanitize_request_body_for_spend_logs_payload returns (mapping cycles already cut), so it never goes deeper than that sanitizer did. diff --git a/tests/unit/llms/litellm_proxy/responses/test_transformation.py b/tests/unit/llms/litellm_proxy/responses/test_transformation.py index 3d774b79eed..5537d9d1916 100644 --- a/tests/unit/llms/litellm_proxy/responses/test_transformation.py +++ b/tests/unit/llms/litellm_proxy/responses/test_transformation.py @@ -1,4 +1,5 @@ import json +from collections.abc import Callable, Iterable from typing import Final import httpx @@ -94,3 +95,86 @@ def test_compact_request_forwards_metadata_and_user_to_the_downstream_proxy(drop "user": "end-user-1", "temperature": 0.2, } + + +_GUARDRAIL_BLOCK: Final = { + "error": { + "message": "Content blocked: secret_project_codename pattern detected", + "type": "invalid_request_error", + "param": None, + "code": "400", + "provider_specific_fields": {"guardrail_name": "block-secret-project", "guardrail_mode": "pre_call"}, + } +} + + +def _proxy_rejection(route: str, response: httpx.Response, call: Callable[[], object]) -> litellm.BadRequestError: + with respx.mock(assert_all_called=True) as mock: + _ = mock.post(f"https://my-proxy.example.com/{route}").mock(return_value=response) + with pytest.raises(litellm.BadRequestError) as caught: + _ = call() + return caught.value + + +def _consume(stream: Iterable[object]) -> None: + for _ in stream: + pass + + +_PROXY_PARAMS: Final = {"api_base": "https://my-proxy.example.com/", "api_key": "sk-proxy-key"} + + +@pytest.mark.usefixtures(httpx_transport.__name__) +@pytest.mark.parametrize("stream", [False, True]) +def test_proxy_400_reaches_responses_caller_with_the_same_body_as_chat_completions(stream: bool) -> None: + chat_error: Final = _proxy_rejection( + "chat/completions", + httpx.Response(400, json=_GUARDRAIL_BLOCK), + lambda: litellm.completion( + model="litellm_proxy/gpt-5.5", messages=[{"role": "user", "content": "project bluebird"}], **_PROXY_PARAMS + ), + ) + responses_error: Final = _proxy_rejection( + "responses", + httpx.Response(400, json=_GUARDRAIL_BLOCK), + lambda: ( + _consume(litellm.responses(model="litellm_proxy/gpt-5.5", input="project bluebird", stream=True, **_PROXY_PARAMS)) + if stream + else litellm.responses(model="litellm_proxy/gpt-5.5", input="project bluebird", **_PROXY_PARAMS) + ), + ) + assert responses_error.body == chat_error.body == _GUARDRAIL_BLOCK["error"] + assert (responses_error.code, responses_error.type) == (chat_error.code, chat_error.type) + assert str(responses_error) == str(chat_error) + + +@pytest.mark.usefixtures(httpx_transport.__name__) +def test_proxy_error_without_a_json_object_body_still_raises_with_its_text() -> None: + error: Final = _proxy_rejection( + "responses", + httpx.Response(400, text="upstream proxy rejected the request"), + lambda: litellm.responses(model="litellm_proxy/gpt-5.5", input="hi", **_PROXY_PARAMS), + ) + assert error.body is None + assert "upstream proxy rejected the request" in str(error) + + +@pytest.mark.usefixtures(httpx_transport.__name__) +def test_proxy_error_object_without_an_error_member_is_the_body_as_sent() -> None: + sent: Final = {"message": "upstream proxy rejected the request", "type": "invalid_request_error"} + error: Final = _proxy_rejection( + "responses", + httpx.Response(400, json=sent), + lambda: litellm.responses(model="litellm_proxy/gpt-5.5", input="hi", **_PROXY_PARAMS), + ) + assert error.body == sent + + +@pytest.mark.usefixtures(httpx_transport.__name__) +def test_proxy_error_whose_error_member_is_text_still_raises_with_that_text() -> None: + error: Final = _proxy_rejection( + "responses", + httpx.Response(400, json={"error": "upstream proxy rejected the request"}), + lambda: litellm.responses(model="litellm_proxy/gpt-5.5", input="hi", **_PROXY_PARAMS), + ) + assert "upstream proxy rejected the request" in str(error) diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index cc1fc5cd6c3..aeab7a3d348 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -14,6 +14,7 @@ from tests.unit.proxy.router_entry_point_stubs import route_entry_points_to_stub import anthropic import httpx import pytest +import respx from fastapi import HTTPException, Request, Response, status from fastapi.responses import JSONResponse, StreamingResponse @@ -12516,3 +12517,103 @@ def test_maybe_get_model_id_prefers_litellm_metadata_over_caller_metadata_withou def test_maybe_get_model_id_returns_none_before_the_router_picked_a_deployment(): assert ProxyBaseLLMRequestProcessing(data={"metadata": {}}).maybe_get_model_id(None) is None + + +_DOWNSTREAM_GUARDRAIL_FIELDS: Final = MappingProxyType( + {"guardrail_name": "block-secret-project", "guardrail_mode": "pre_call", "pattern": "secret_project_codename"} +) + + +def _downstream_proxy_rejection( + route: str, call: Callable[[], object], fields: Mapping[str, object] = _DOWNSTREAM_GUARDRAIL_FIELDS +) -> Exception: + body: Final = { + "error": { + "message": "Content blocked: secret_project_codename pattern detected", + "type": "invalid_request_error", + "param": None, + "code": "400", + "provider_specific_fields": dict(fields), + } + } + with respx.mock(assert_all_called=True) as mock: + _ = mock.post(f"https://downstream-proxy.example.com/{route}").mock(return_value=httpx.Response(400, json=body)) + with pytest.raises(litellm.BadRequestError) as caught: + _ = call() + return caught.value + + +_DOWNSTREAM: Final = MappingProxyType( + {"api_base": "https://downstream-proxy.example.com/", "api_key": "sk-downstream-key"} +) + + +@pytest.mark.usefixtures("httpx_transport") +@pytest.mark.parametrize( + ("route", "call"), + [ + ( + "responses", + lambda: litellm.responses(model="litellm_proxy/gpt-5.5", input="project bluebird", **_DOWNSTREAM), + ), + ( + "chat/completions", + lambda: litellm.completion( + model="litellm_proxy/gpt-5.5", messages=[{"role": "user", "content": "project bluebird"}], **_DOWNSTREAM + ), + ), + ], +) +async def test_a_chained_proxy_relays_which_downstream_guardrail_blocked_the_request( + route: str, call: Callable[[], object] +) -> None: + rejection: Final = _downstream_proxy_rejection(route, call) + with pytest.raises(ProxyException) as relayed: + await ProxyBaseLLMRequestProcessing(data={"model": "chained"})._handle_llm_api_exception( + e=rejection, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-front"), + proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()), + ) + assert relayed.value.to_dict()["provider_specific_fields"] == dict(_DOWNSTREAM_GUARDRAIL_FIELDS) + + +_DOWNSTREAM_LEAKABLE_VALUES: Final = ( + "guard-scan.internal.corp", + "/srv/secret/app/guard.py", + "10.0.4.17", + "abcd1234efgh5678", + "cs-0123456789", + "tok-0123456789abcdef", + "4815162342108", +) + + +@pytest.mark.usefixtures("httpx_transport") +async def test_a_chained_proxy_redacts_secrets_and_internal_details_from_relayed_guardrail_fields() -> None: + fields: Final = { + **_DOWNSTREAM_GUARDRAIL_FIELDS, + "exception": "connect to http://guard-scan.internal.corp:9000/v1/check failed at /srv/secret/app/guard.py", + "api_key": "abcd1234efgh5678", + "access_token": ["tok-0123456789abcdef"], + "scanner": { + "hosts": ["10.0.4.17:8080"], + "client_secret": "cs-0123456789", + "api_secret": 4815162342108, + "attempts": 2, + }, + } + rejection: Final = _downstream_proxy_rejection( + "responses", + lambda: litellm.responses(model="litellm_proxy/gpt-5.5", input="project bluebird", **_DOWNSTREAM), + fields, + ) + with pytest.raises(ProxyException) as relayed: + await ProxyBaseLLMRequestProcessing(data={"model": "chained"})._handle_llm_api_exception( + e=rejection, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-front"), + proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()), + ) + answered: Final = json.loads(json.dumps(relayed.value.to_dict()["provider_specific_fields"])) + assert {key: answered[key] for key in _DOWNSTREAM_GUARDRAIL_FIELDS} == dict(_DOWNSTREAM_GUARDRAIL_FIELDS) + assert answered["scanner"]["attempts"] == 2 + assert [value for value in _DOWNSTREAM_LEAKABLE_VALUES if value in json.dumps(answered)] == []