mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(passthrough): log upstream 4xx/5xx error bodies and carry them into failure hooks
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
a06d63f99e
commit
1a112a315f
3 changed files with 192 additions and 1 deletions
|
|
@ -1446,6 +1446,7 @@ PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES: Final = int(
|
|||
CLOUDZERO_EXPORT_INTERVAL_MINUTES: Final = int(os.getenv("CLOUDZERO_EXPORT_INTERVAL_MINUTES", 60))
|
||||
MCP_TOOL_NAME_PREFIX: Final = "mcp_tool"
|
||||
MAXIMUM_TRACEBACK_LINES_TO_LOG: Final = int(os.getenv("MAXIMUM_TRACEBACK_LINES_TO_LOG", 100))
|
||||
PASSTHROUGH_UPSTREAM_ERROR_BODY_MAX_LOG_CHARS: Final = 4096
|
||||
|
||||
# Headers to control callbacks
|
||||
X_LITELLM_DISABLE_CALLBACKS: Final = "x-litellm-disable-callbacks"
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ from litellm._logging import verbose_proxy_logger
|
|||
from litellm._uuid import uuid
|
||||
from litellm.constants import (
|
||||
MAXIMUM_TRACEBACK_LINES_TO_LOG,
|
||||
PASSTHROUGH_UPSTREAM_ERROR_BODY_MAX_LOG_CHARS,
|
||||
SESSION_ID_OMITTED_METADATA_KEY,
|
||||
WEBSOCKET_CLOSE_REASON_MAX_BYTES,
|
||||
)
|
||||
|
|
@ -825,10 +826,23 @@ def _resolve_team_callback_wiring(
|
|||
)
|
||||
|
||||
|
||||
async def _read_upstream_error_body_for_logging(response: httpx.Response) -> str:
|
||||
"""Buffer the upstream error body for logging; httpx replays it from ``aiter_bytes()`` so the client still gets it."""
|
||||
await response.aread()
|
||||
body: Final = response.text
|
||||
if len(body) <= PASSTHROUGH_UPSTREAM_ERROR_BODY_MAX_LOG_CHARS:
|
||||
return body
|
||||
return (
|
||||
f"{body[:PASSTHROUGH_UPSTREAM_ERROR_BODY_MAX_LOG_CHARS]}"
|
||||
f"... (truncated, {len(body) - PASSTHROUGH_UPSTREAM_ERROR_BODY_MAX_LOG_CHARS} more chars)"
|
||||
)
|
||||
|
||||
|
||||
async def _log_passthrough_upstream_failure(
|
||||
response: httpx.Response,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
request_payload: dict,
|
||||
url_route: str,
|
||||
) -> None:
|
||||
"""Fire LiteLLM-side failure hooks (spend tracking, alerting callbacks) for
|
||||
an upstream 4xx/5xx passthrough response.
|
||||
|
|
@ -842,6 +856,13 @@ async def _log_passthrough_upstream_failure(
|
|||
return
|
||||
from litellm.proxy.proxy_server import proxy_logging_obj
|
||||
|
||||
upstream_error_body: Final = await _read_upstream_error_body_for_logging(response)
|
||||
verbose_proxy_logger.warning(
|
||||
"Passthrough upstream request to %s failed with status %s: %s",
|
||||
url_route,
|
||||
response.status_code,
|
||||
upstream_error_body,
|
||||
)
|
||||
try:
|
||||
response.raise_for_status()
|
||||
except httpx.HTTPStatusError:
|
||||
|
|
@ -854,7 +875,7 @@ async def _log_passthrough_upstream_failure(
|
|||
# rate-limit errors already are.
|
||||
synthetic_exception: Final = HTTPException(
|
||||
status_code=response.status_code,
|
||||
detail=f"Upstream passthrough request failed with status {response.status_code}",
|
||||
detail=f"Upstream passthrough request failed with status {response.status_code}: {upstream_error_body}",
|
||||
)
|
||||
try:
|
||||
await proxy_logging_obj.post_call_failure_hook(
|
||||
|
|
@ -1274,6 +1295,7 @@ async def pass_through_request(
|
|||
custom_llm_provider=custom_llm_provider,
|
||||
upstream_usage=upstream_usage,
|
||||
),
|
||||
url_route=str(url),
|
||||
)
|
||||
|
||||
# Call response headers hook for streaming pass-through
|
||||
|
|
@ -1355,6 +1377,7 @@ async def pass_through_request(
|
|||
custom_llm_provider=custom_llm_provider,
|
||||
upstream_usage=upstream_usage,
|
||||
),
|
||||
url_route=str(url),
|
||||
)
|
||||
|
||||
# Call response headers hook for detected streaming pass-through
|
||||
|
|
@ -1449,6 +1472,7 @@ async def pass_through_request(
|
|||
response=response,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
request_payload=failure_request_payload,
|
||||
url_route=str(url),
|
||||
)
|
||||
|
||||
if response.status_code < 400 and response_body is not None and guardrails_to_run:
|
||||
|
|
|
|||
|
|
@ -3988,6 +3988,172 @@ async def test_pass_through_request_streaming_upstream_error_returned_unchanged(
|
|||
assert failure_call_kwargs["original_exception"].status_code == 403
|
||||
|
||||
|
||||
_UPSTREAM_FAILURE_WARNING_MARKER = "Passthrough upstream request to"
|
||||
|
||||
|
||||
def _upstream_failure_warnings(caplog):
|
||||
return [
|
||||
record.getMessage()
|
||||
for record in caplog.records
|
||||
if record.levelno == logging.WARNING and _UPSTREAM_FAILURE_WARNING_MARKER in record.getMessage()
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pass_through_request_upstream_error_body_is_logged_and_carried_into_failure_hook(caplog):
|
||||
"""Regression (LIT-6644): the upstream error body must reach the WARNING log and the failure hook."""
|
||||
from fastapi import HTTPException
|
||||
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
upstream_error = {
|
||||
"error": {
|
||||
"code": 404,
|
||||
"message": "Publisher model `claude-nope-9` was not found or your project does not have access to it.",
|
||||
"status": "NOT_FOUND",
|
||||
}
|
||||
}
|
||||
upstream_content = json.dumps(upstream_error).encode("utf-8")
|
||||
fake_client, cleanup = _inject_fake_passthrough_client(
|
||||
_FakeUpstreamTransport(
|
||||
status_code=404,
|
||||
headers={"content-type": "application/json"},
|
||||
stream=_RecordingUpstreamByteStream((upstream_content,)),
|
||||
),
|
||||
timeout=314.0,
|
||||
)
|
||||
try:
|
||||
with ExitStack() as stack:
|
||||
mock_proxy_logging, _ = _enter_relay_logging_mocks(stack, {})
|
||||
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
|
||||
response = await pass_through_request(
|
||||
request=_relay_client_request(method="POST"),
|
||||
target="http://upstream.test/v1/publishers/anthropic/models/claude-nope-9:streamRawPredict",
|
||||
custom_headers={},
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-relay-test"),
|
||||
timeout=314.0,
|
||||
)
|
||||
finally:
|
||||
cleanup()
|
||||
await fake_client.aclose()
|
||||
|
||||
assert response.status_code == 404
|
||||
assert json.loads(response.body) == upstream_error
|
||||
|
||||
warnings = _upstream_failure_warnings(caplog)
|
||||
assert len(warnings) == 1
|
||||
assert "http://upstream.test/v1/publishers/anthropic/models/claude-nope-9:streamRawPredict" in warnings[0]
|
||||
assert "status 404" in warnings[0]
|
||||
assert "was not found or your project does not have access to it" in warnings[0]
|
||||
|
||||
original_exception = mock_proxy_logging.post_call_failure_hook.call_args.kwargs["original_exception"]
|
||||
assert isinstance(original_exception, HTTPException)
|
||||
assert original_exception.status_code == 404
|
||||
assert "was not found or your project does not have access to it" in original_exception.detail
|
||||
assert "NOT_FOUND" in str(original_exception)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pass_through_request_streaming_upstream_error_body_is_logged_and_still_relayed(caplog):
|
||||
"""Regression (LIT-6644): logging a streamed error body must not consume the bytes owed to the client."""
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
upstream_error = {
|
||||
"type": "error",
|
||||
"error": {
|
||||
"type": "invalid_request_error",
|
||||
"message": "Unexpected value(s) `context-1m-2025-08-07` for the `anthropic-beta` header.",
|
||||
},
|
||||
}
|
||||
upstream_content = json.dumps(upstream_error).encode("utf-8")
|
||||
upstream_stream = _RecordingUpstreamByteStream((upstream_content[:20], upstream_content[20:]))
|
||||
fake_client, cleanup = _inject_fake_passthrough_client(
|
||||
_FakeUpstreamTransport(
|
||||
status_code=400,
|
||||
headers={"content-type": "application/json"},
|
||||
stream=upstream_stream,
|
||||
),
|
||||
timeout=314.0,
|
||||
)
|
||||
try:
|
||||
with ExitStack() as stack:
|
||||
mock_proxy_logging, mock_success_handler = _enter_relay_logging_mocks(stack, {})
|
||||
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
|
||||
response = await pass_through_request(
|
||||
request=_relay_client_request(method="POST"),
|
||||
target="http://upstream.test/v1/messages",
|
||||
custom_headers={},
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-relay-test"),
|
||||
stream=True,
|
||||
timeout=314.0,
|
||||
)
|
||||
assert isinstance(response, StreamingResponse)
|
||||
assert response.status_code == 400
|
||||
streamed_chunks = [chunk async for chunk in response.body_iterator]
|
||||
finally:
|
||||
cleanup()
|
||||
await fake_client.aclose()
|
||||
|
||||
streamed_bytes = b"".join(chunk if isinstance(chunk, bytes) else chunk.encode("utf-8") for chunk in streamed_chunks)
|
||||
assert streamed_bytes == upstream_content
|
||||
assert upstream_stream.closed is True
|
||||
mock_success_handler.assert_not_called()
|
||||
|
||||
warnings = _upstream_failure_warnings(caplog)
|
||||
assert len(warnings) == 1
|
||||
assert "http://upstream.test/v1/messages" in warnings[0]
|
||||
assert "status 400" in warnings[0]
|
||||
assert "for the `anthropic-beta` header" in warnings[0]
|
||||
|
||||
original_exception = mock_proxy_logging.post_call_failure_hook.call_args.kwargs["original_exception"]
|
||||
assert original_exception.status_code == 400
|
||||
assert "invalid_request_error" in original_exception.detail
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pass_through_request_upstream_error_body_is_truncated_in_logs(caplog):
|
||||
from litellm.constants import PASSTHROUGH_UPSTREAM_ERROR_BODY_MAX_LOG_CHARS
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
oversized_message = "x" * (PASSTHROUGH_UPSTREAM_ERROR_BODY_MAX_LOG_CHARS * 3)
|
||||
upstream_content = json.dumps({"error": oversized_message}).encode("utf-8")
|
||||
fake_client, cleanup = _inject_fake_passthrough_client(
|
||||
_FakeUpstreamTransport(
|
||||
status_code=500,
|
||||
headers={"content-type": "application/json"},
|
||||
stream=_RecordingUpstreamByteStream((upstream_content,)),
|
||||
),
|
||||
timeout=314.0,
|
||||
)
|
||||
try:
|
||||
with ExitStack() as stack:
|
||||
mock_proxy_logging, _ = _enter_relay_logging_mocks(stack, {})
|
||||
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
|
||||
response = await pass_through_request(
|
||||
request=_relay_client_request(method="POST"),
|
||||
target="http://upstream.test/v1/huge-error",
|
||||
custom_headers={},
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-relay-test"),
|
||||
timeout=314.0,
|
||||
)
|
||||
finally:
|
||||
cleanup()
|
||||
await fake_client.aclose()
|
||||
|
||||
assert response.body == upstream_content
|
||||
|
||||
warnings = _upstream_failure_warnings(caplog)
|
||||
assert len(warnings) == 1
|
||||
assert len(warnings[0]) < len(upstream_content)
|
||||
assert "truncated" in warnings[0]
|
||||
|
||||
detail = mock_proxy_logging.post_call_failure_hook.call_args.kwargs["original_exception"].detail
|
||||
assert len(detail) < len(upstream_content)
|
||||
assert "truncated" in detail
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pass_through_request_non_streaming_success_unchanged():
|
||||
"""Success (2xx) passthrough behavior must remain unchanged by the error fix."""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue