scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

fix(proxy): return upstream error bodies unchanged in passthrough

Browse source 
Generic pass-through endpoints called raise_for_status() on upstream 4xx/5xx
responses and re-raised as HTTPException, which the outer handler reshaped
into a ProxyException with the upstream body stringified into error.message.
Success responses were already forwarded as-is, so failures were the only
case where passthrough wasn't actually transparent. Removes the
raise_for_status() calls for both streaming and non-streaming passthrough so
upstream status, body, and headers reach the client unchanged, while keeping
guardrails/managed-id rewriting scoped to successful responses and leaving
internal proxy failures (auth, config, network errors before any upstream
response) on the existing ProxyException path.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Shivam Rawat 2026-07-04 11:15:40 -07:00
parent 36cbc3a24c
commit 8c9878025e
2 changed files with 221 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
View file

@ -1053,11 +1053,6 @@ async def pass_through_request(
response = await async_client.send(req, stream=stream)
try:
response.raise_for_status()
except httpx.HTTPStatusError as e:
raise HTTPException(status_code=e.response.status_code, detail=await e.response.aread())
# Call response headers hook for streaming pass-through
_response_headers = HttpPassThroughEndpointHelpers.get_response_headers(
headers=response.headers,
@ -1112,11 +1107,6 @@ async def pass_through_request(
logging_obj.stream = True
logging_obj.model_call_details["stream"] = True
try:
response.raise_for_status()
except httpx.HTTPStatusError as e:
raise HTTPException(status_code=e.response.status_code, detail=await e.response.aread())
# Call response headers hook for detected streaming pass-through
_response_headers = HttpPassThroughEndpointHelpers.get_response_headers(
headers=response.headers,
@ -1145,19 +1135,13 @@ async def pass_through_request(
status_code=response.status_code,
)
try:
response.raise_for_status()
except httpx.HTTPStatusError as e:
raise HTTPException(status_code=e.response.status_code, detail=e.response.text)
if response.status_code >= 300:
raise HTTPException(status_code=response.status_code, detail=response.text)
content = await response.aread()
## POST-CALL GUARDRAILS ##
# Upstream errors (4xx/5xx) must reach the client unchanged; guardrails
# and managed-id rewriting only apply to successful upstream responses.
_content_modified = False
response_body: Optional[dict] = get_response_body(response)
response_body: Optional[dict] = get_response_body(response) if response.status_code < 400 else None
if response_body is not None and guardrails_to_run:
# Build an enriched data dict: _parsed_body has been stripped of
# `metadata` by both pre_call_hook and _init_kwargs_for_pass_through_endpoint,
@ -1187,7 +1171,7 @@ async def pass_through_request(
)
elif response_body is None:
verbose_proxy_logger.debug(
"pass_through_endpoint: response body not JSON-parseable, skipping post-call guardrails"
"pass_through_endpoint: response not JSON-parseable or upstream error, skipping post-call guardrails"
)
## PASSTHROUGH MANAGED ID MINTING (OUTPUT) ##

217
tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py
View file

@ -1,3 +1,4 @@
import asyncio
import json
import os
import sys
@ -3619,3 +3620,219 @@ async def test_non_guardrail_exception_still_logs_with_traceback():
assert (
logger.warning.call_count == 0
), "a genuine failure must not be downgraded to WARNING"
# Regression: generic config-based passthrough (`pass_through_request`) used to
# call `response.raise_for_status()` on upstream errors and re-raise as an
# `HTTPException`, which the outer `except` block then reshaped into a
# `ProxyException` (`{"error": {"message": "<stringified upstream body>", ...}}`).
# Upstream error responses must reach the client byte-for-byte, with the
# original status code, exactly like success responses already do.
_UPSTREAM_ERROR_BODY = {
"error": "Permission denied",
"error_code": "ACCESS_DENIED",
"request_id": "req_mock_403",
"trace_id": "trace_mock_403",
}
@pytest.mark.asyncio
async def test_pass_through_request_non_streaming_upstream_error_returned_unchanged():
upstream_content = json.dumps(_UPSTREAM_ERROR_BODY).encode("utf-8")
upstream_response = httpx.Response(
status_code=403,
headers={"content-type": "application/json"},
content=upstream_content,
request=httpx.Request("POST", "http://target-api.com/api/denied"),
)
with patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging:
with patch(
"litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_async_httpx_client"
) as mock_get_client:
with patch(
"litellm.proxy.pass_through_endpoints.pass_through_endpoints.ProxyBaseLLMRequestProcessing"
) as mock_processing:
with patch(
"litellm.proxy.pass_through_endpoints.pass_through_endpoints.pass_through_endpoint_logging.pass_through_async_success_handler"
) as mock_success_handler:
mock_proxy_logging.pre_call_hook = AsyncMock(return_value={})
mock_proxy_logging.post_call_failure_hook = AsyncMock()
mock_proxy_logging.post_call_response_headers_hook = AsyncMock(
return_value=None
)
mock_processing.get_custom_headers.return_value = {}
mock_success_handler.return_value = None
async_client = MagicMock()
async_client.request = AsyncMock(return_value=upstream_response)
mock_get_client.return_value = MagicMock(client=async_client)
mock_request = MagicMock(spec=Request)
mock_request.method = "POST"
mock_request.url = "http://test-proxy.com/mock-upstream/api/denied"
mock_request.body = AsyncMock(return_value=b'{"action": "read"}')
mock_request.headers = Headers({"content-type": "application/json"})
mock_request.query_params = QueryParams({})
response = await pass_through_request(
request=mock_request,
target="http://target-api.com/api/denied",
custom_headers={},
user_api_key_dict=MagicMock(),
)
await asyncio.sleep(0)
assert response.status_code == 403
body = json.loads(response.body)
# Exact dict equality proves the upstream body was forwarded verbatim,
# not stringified into a ProxyException's `error.message` field.
assert body == _UPSTREAM_ERROR_BODY
assert set(body.keys()) != {"error"} or not isinstance(body["error"], dict)
mock_success_handler.assert_called_once()
@pytest.mark.asyncio
async def test_pass_through_request_streaming_upstream_error_returned_unchanged():
from fastapi.responses import StreamingResponse
upstream_content = json.dumps(_UPSTREAM_ERROR_BODY).encode("utf-8")
upstream_response = httpx.Response(
status_code=403,
headers={"content-type": "application/json"},
content=upstream_content,
request=httpx.Request("GET", "http://target-api.com/api/stream-denied"),
)
with patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging:
with patch(
"litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_async_httpx_client"
) as mock_get_client:
with patch(
"litellm.proxy.pass_through_endpoints.pass_through_endpoints.pass_through_endpoint_logging.pass_through_async_success_handler"
) as mock_success_handler:
mock_proxy_logging.pre_call_hook = AsyncMock(return_value={})
mock_proxy_logging.post_call_failure_hook = AsyncMock()
mock_proxy_logging.post_call_response_headers_hook = AsyncMock(
return_value=None
)
mock_success_handler.return_value = None
async_client = MagicMock()
async_client.build_request = MagicMock(return_value=MagicMock())
async_client.send = AsyncMock(return_value=upstream_response)
mock_get_client.return_value = MagicMock(client=async_client)
mock_request = MagicMock(spec=Request)
mock_request.method = "GET"
mock_request.url = "http://test-proxy.com/mock-upstream/api/stream-denied"
mock_request.body = AsyncMock(return_value=b"")
mock_request.headers = Headers({})
mock_request.query_params = QueryParams({})
response = await pass_through_request(
request=mock_request,
target="http://target-api.com/api/stream-denied",
custom_headers={},
user_api_key_dict=MagicMock(),
stream=True,
)
assert isinstance(response, StreamingResponse)
assert response.status_code == 403
streamed_chunks = [chunk async for chunk in response.body_iterator]
await asyncio.sleep(0)
streamed_bytes = b"".join(
chunk if isinstance(chunk, bytes) else chunk.encode("utf-8")
for chunk in streamed_chunks
)
assert streamed_bytes == upstream_content
assert json.loads(streamed_bytes) == _UPSTREAM_ERROR_BODY
@pytest.mark.asyncio
async def test_pass_through_request_non_streaming_success_unchanged():
"""Success (2xx) passthrough behavior must remain unchanged by the error fix."""
upstream_success_body = {"status": "ok", "message": "mock upstream success"}
upstream_content = json.dumps(upstream_success_body).encode("utf-8")
upstream_response = httpx.Response(
status_code=200,
headers={"content-type": "application/json"},
content=upstream_content,
request=httpx.Request("GET", "http://target-api.com/api/success"),
)
with patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging:
with patch(
"litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_async_httpx_client"
) as mock_get_client:
with patch(
"litellm.proxy.pass_through_endpoints.pass_through_endpoints.ProxyBaseLLMRequestProcessing"
) as mock_processing:
with patch(
"litellm.proxy.pass_through_endpoints.pass_through_endpoints.pass_through_endpoint_logging.pass_through_async_success_handler"
) as mock_success_handler:
mock_proxy_logging.pre_call_hook = AsyncMock(return_value={})
mock_proxy_logging.post_call_failure_hook = AsyncMock()
mock_proxy_logging.post_call_response_headers_hook = AsyncMock(
return_value=None
)
mock_processing.get_custom_headers.return_value = {}
mock_success_handler.return_value = None
async_client = MagicMock()
async_client.request = AsyncMock(return_value=upstream_response)
mock_get_client.return_value = MagicMock(client=async_client)
mock_request = MagicMock(spec=Request)
mock_request.method = "GET"
mock_request.url = "http://test-proxy.com/mock-upstream/api/success"
mock_request.body = AsyncMock(return_value=b"")
mock_request.headers = Headers({})
mock_request.query_params = QueryParams({})
response = await pass_through_request(
request=mock_request,
target="http://target-api.com/api/success",
custom_headers={},
user_api_key_dict=MagicMock(),
)
await asyncio.sleep(0)
assert response.status_code == 200
assert json.loads(response.body) == upstream_success_body
@pytest.mark.asyncio
async def test_pass_through_request_internal_failure_still_raises_proxy_exception():
"""
Internal proxy failures (e.g. a hook raising before any upstream request is
made) must still surface as ProxyException, distinct from upstream
passthrough errors which are now returned unchanged.
"""
from litellm.proxy._types import ProxyException
with patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging:
mock_proxy_logging.pre_call_hook = AsyncMock(
side_effect=RuntimeError("auth backend unavailable")
)
mock_proxy_logging.post_call_failure_hook = AsyncMock()
mock_request = MagicMock(spec=Request)
mock_request.method = "GET"
mock_request.url = "http://test-proxy.com/mock-upstream/api/success"
mock_request.body = AsyncMock(return_value=b"")
mock_request.headers = Headers({})
mock_request.query_params = QueryParams({})
with pytest.raises(ProxyException) as exc_info:
await pass_through_request(
request=mock_request,
target="http://target-api.com/api/success",
custom_headers={},
user_api_key_dict=MagicMock(),
)
assert int(exc_info.value.code) == 500
assert "auth backend unavailable" in exc_info.value.message