fix(passthrough): log upstream 4xx/5xx error bodies and carry them into failure hooks

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Devin AI 2026-09-04 00:54:43 +00:00
parent a06d63f99e
commit 1a112a315f
3 changed files with 192 additions and 1 deletions

View file

@ -1446,6 +1446,7 @@ PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES: Final = int(
CLOUDZERO_EXPORT_INTERVAL_MINUTES: Final = int(os.getenv("CLOUDZERO_EXPORT_INTERVAL_MINUTES", 60))
MCP_TOOL_NAME_PREFIX: Final = "mcp_tool"
MAXIMUM_TRACEBACK_LINES_TO_LOG: Final = int(os.getenv("MAXIMUM_TRACEBACK_LINES_TO_LOG", 100))
PASSTHROUGH_UPSTREAM_ERROR_BODY_MAX_LOG_CHARS: Final = 4096
# Headers to control callbacks
X_LITELLM_DISABLE_CALLBACKS: Final = "x-litellm-disable-callbacks"

View file

@ -40,6 +40,7 @@ from litellm._logging import verbose_proxy_logger
from litellm._uuid import uuid
from litellm.constants import (
MAXIMUM_TRACEBACK_LINES_TO_LOG,
PASSTHROUGH_UPSTREAM_ERROR_BODY_MAX_LOG_CHARS,
SESSION_ID_OMITTED_METADATA_KEY,
WEBSOCKET_CLOSE_REASON_MAX_BYTES,
)
@ -825,10 +826,23 @@ def _resolve_team_callback_wiring(
)
async def _read_upstream_error_body_for_logging(response: httpx.Response) -> str:
"""Buffer the upstream error body for logging; httpx replays it from ``aiter_bytes()`` so the client still gets it."""
await response.aread()
body: Final = response.text
if len(body) <= PASSTHROUGH_UPSTREAM_ERROR_BODY_MAX_LOG_CHARS:
return body
return (
f"{body[:PASSTHROUGH_UPSTREAM_ERROR_BODY_MAX_LOG_CHARS]}"
f"... (truncated, {len(body) - PASSTHROUGH_UPSTREAM_ERROR_BODY_MAX_LOG_CHARS} more chars)"
)
async def _log_passthrough_upstream_failure(
response: httpx.Response,
user_api_key_dict: UserAPIKeyAuth,
request_payload: dict,
url_route: str,
) -> None:
"""Fire LiteLLM-side failure hooks (spend tracking, alerting callbacks) for
an upstream 4xx/5xx passthrough response.
@ -842,6 +856,13 @@ async def _log_passthrough_upstream_failure(
return
from litellm.proxy.proxy_server import proxy_logging_obj
upstream_error_body: Final = await _read_upstream_error_body_for_logging(response)
verbose_proxy_logger.warning(
"Passthrough upstream request to %s failed with status %s: %s",
url_route,
response.status_code,
upstream_error_body,
)
try:
response.raise_for_status()
except httpx.HTTPStatusError:
@ -854,7 +875,7 @@ async def _log_passthrough_upstream_failure(
# rate-limit errors already are.
synthetic_exception: Final = HTTPException(
status_code=response.status_code,
detail=f"Upstream passthrough request failed with status {response.status_code}",
detail=f"Upstream passthrough request failed with status {response.status_code}: {upstream_error_body}",
)
try:
await proxy_logging_obj.post_call_failure_hook(
@ -1274,6 +1295,7 @@ async def pass_through_request(
custom_llm_provider=custom_llm_provider,
upstream_usage=upstream_usage,
),
url_route=str(url),
)
# Call response headers hook for streaming pass-through
@ -1355,6 +1377,7 @@ async def pass_through_request(
custom_llm_provider=custom_llm_provider,
upstream_usage=upstream_usage,
),
url_route=str(url),
)
# Call response headers hook for detected streaming pass-through
@ -1449,6 +1472,7 @@ async def pass_through_request(
response=response,
user_api_key_dict=user_api_key_dict,
request_payload=failure_request_payload,
url_route=str(url),
)
if response.status_code < 400 and response_body is not None and guardrails_to_run:

View file

@ -3988,6 +3988,172 @@ async def test_pass_through_request_streaming_upstream_error_returned_unchanged(
assert failure_call_kwargs["original_exception"].status_code == 403
_UPSTREAM_FAILURE_WARNING_MARKER = "Passthrough upstream request to"
def _upstream_failure_warnings(caplog):
return [
record.getMessage()
for record in caplog.records
if record.levelno == logging.WARNING and _UPSTREAM_FAILURE_WARNING_MARKER in record.getMessage()
]
@pytest.mark.asyncio
async def test_pass_through_request_upstream_error_body_is_logged_and_carried_into_failure_hook(caplog):
"""Regression (LIT-6644): the upstream error body must reach the WARNING log and the failure hook."""
from fastapi import HTTPException
from litellm.proxy._types import UserAPIKeyAuth
upstream_error = {
"error": {
"code": 404,
"message": "Publisher model `claude-nope-9` was not found or your project does not have access to it.",
"status": "NOT_FOUND",
}
}
upstream_content = json.dumps(upstream_error).encode("utf-8")
fake_client, cleanup = _inject_fake_passthrough_client(
_FakeUpstreamTransport(
status_code=404,
headers={"content-type": "application/json"},
stream=_RecordingUpstreamByteStream((upstream_content,)),
),
timeout=314.0,
)
try:
with ExitStack() as stack:
mock_proxy_logging, _ = _enter_relay_logging_mocks(stack, {})
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
response = await pass_through_request(
request=_relay_client_request(method="POST"),
target="http://upstream.test/v1/publishers/anthropic/models/claude-nope-9:streamRawPredict",
custom_headers={},
user_api_key_dict=UserAPIKeyAuth(api_key="sk-relay-test"),
timeout=314.0,
)
finally:
cleanup()
await fake_client.aclose()
assert response.status_code == 404
assert json.loads(response.body) == upstream_error
warnings = _upstream_failure_warnings(caplog)
assert len(warnings) == 1
assert "http://upstream.test/v1/publishers/anthropic/models/claude-nope-9:streamRawPredict" in warnings[0]
assert "status 404" in warnings[0]
assert "was not found or your project does not have access to it" in warnings[0]
original_exception = mock_proxy_logging.post_call_failure_hook.call_args.kwargs["original_exception"]
assert isinstance(original_exception, HTTPException)
assert original_exception.status_code == 404
assert "was not found or your project does not have access to it" in original_exception.detail
assert "NOT_FOUND" in str(original_exception)
@pytest.mark.asyncio
async def test_pass_through_request_streaming_upstream_error_body_is_logged_and_still_relayed(caplog):
"""Regression (LIT-6644): logging a streamed error body must not consume the bytes owed to the client."""
from fastapi.responses import StreamingResponse
from litellm.proxy._types import UserAPIKeyAuth
upstream_error = {
"type": "error",
"error": {
"type": "invalid_request_error",
"message": "Unexpected value(s) `context-1m-2025-08-07` for the `anthropic-beta` header.",
},
}
upstream_content = json.dumps(upstream_error).encode("utf-8")
upstream_stream = _RecordingUpstreamByteStream((upstream_content[:20], upstream_content[20:]))
fake_client, cleanup = _inject_fake_passthrough_client(
_FakeUpstreamTransport(
status_code=400,
headers={"content-type": "application/json"},
stream=upstream_stream,
),
timeout=314.0,
)
try:
with ExitStack() as stack:
mock_proxy_logging, mock_success_handler = _enter_relay_logging_mocks(stack, {})
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
response = await pass_through_request(
request=_relay_client_request(method="POST"),
target="http://upstream.test/v1/messages",
custom_headers={},
user_api_key_dict=UserAPIKeyAuth(api_key="sk-relay-test"),
stream=True,
timeout=314.0,
)
assert isinstance(response, StreamingResponse)
assert response.status_code == 400
streamed_chunks = [chunk async for chunk in response.body_iterator]
finally:
cleanup()
await fake_client.aclose()
streamed_bytes = b"".join(chunk if isinstance(chunk, bytes) else chunk.encode("utf-8") for chunk in streamed_chunks)
assert streamed_bytes == upstream_content
assert upstream_stream.closed is True
mock_success_handler.assert_not_called()
warnings = _upstream_failure_warnings(caplog)
assert len(warnings) == 1
assert "http://upstream.test/v1/messages" in warnings[0]
assert "status 400" in warnings[0]
assert "for the `anthropic-beta` header" in warnings[0]
original_exception = mock_proxy_logging.post_call_failure_hook.call_args.kwargs["original_exception"]
assert original_exception.status_code == 400
assert "invalid_request_error" in original_exception.detail
@pytest.mark.asyncio
async def test_pass_through_request_upstream_error_body_is_truncated_in_logs(caplog):
from litellm.constants import PASSTHROUGH_UPSTREAM_ERROR_BODY_MAX_LOG_CHARS
from litellm.proxy._types import UserAPIKeyAuth
oversized_message = "x" * (PASSTHROUGH_UPSTREAM_ERROR_BODY_MAX_LOG_CHARS * 3)
upstream_content = json.dumps({"error": oversized_message}).encode("utf-8")
fake_client, cleanup = _inject_fake_passthrough_client(
_FakeUpstreamTransport(
status_code=500,
headers={"content-type": "application/json"},
stream=_RecordingUpstreamByteStream((upstream_content,)),
),
timeout=314.0,
)
try:
with ExitStack() as stack:
mock_proxy_logging, _ = _enter_relay_logging_mocks(stack, {})
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
response = await pass_through_request(
request=_relay_client_request(method="POST"),
target="http://upstream.test/v1/huge-error",
custom_headers={},
user_api_key_dict=UserAPIKeyAuth(api_key="sk-relay-test"),
timeout=314.0,
)
finally:
cleanup()
await fake_client.aclose()
assert response.body == upstream_content
warnings = _upstream_failure_warnings(caplog)
assert len(warnings) == 1
assert len(warnings[0]) < len(upstream_content)
assert "truncated" in warnings[0]
detail = mock_proxy_logging.post_call_failure_hook.call_args.kwargs["original_exception"].detail
assert len(detail) < len(upstream_content)
assert "truncated" in detail
@pytest.mark.asyncio
async def test_pass_through_request_non_streaming_success_unchanged():
"""Success (2xx) passthrough behavior must remain unchanged by the error fix."""