diff --git a/litellm/constants.py b/litellm/constants.py index da731cb5eb2..ece091561cc 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1446,6 +1446,7 @@ PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES: Final = int( CLOUDZERO_EXPORT_INTERVAL_MINUTES: Final = int(os.getenv("CLOUDZERO_EXPORT_INTERVAL_MINUTES", 60)) MCP_TOOL_NAME_PREFIX: Final = "mcp_tool" MAXIMUM_TRACEBACK_LINES_TO_LOG: Final = int(os.getenv("MAXIMUM_TRACEBACK_LINES_TO_LOG", 100)) +PASSTHROUGH_UPSTREAM_ERROR_BODY_MAX_LOG_CHARS: Final = 4096 # Headers to control callbacks X_LITELLM_DISABLE_CALLBACKS: Final = "x-litellm-disable-callbacks" diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 323756bf204..a1ef3a91d9a 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -40,6 +40,7 @@ from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid from litellm.constants import ( MAXIMUM_TRACEBACK_LINES_TO_LOG, + PASSTHROUGH_UPSTREAM_ERROR_BODY_MAX_LOG_CHARS, SESSION_ID_OMITTED_METADATA_KEY, WEBSOCKET_CLOSE_REASON_MAX_BYTES, ) @@ -825,10 +826,23 @@ def _resolve_team_callback_wiring( ) +async def _read_upstream_error_body_for_logging(response: httpx.Response) -> str: + """Buffer the upstream error body for logging; httpx replays it from ``aiter_bytes()`` so the client still gets it.""" + await response.aread() + body: Final = response.text + if len(body) <= PASSTHROUGH_UPSTREAM_ERROR_BODY_MAX_LOG_CHARS: + return body + return ( + f"{body[:PASSTHROUGH_UPSTREAM_ERROR_BODY_MAX_LOG_CHARS]}" + f"... (truncated, {len(body) - PASSTHROUGH_UPSTREAM_ERROR_BODY_MAX_LOG_CHARS} more chars)" + ) + + async def _log_passthrough_upstream_failure( response: httpx.Response, user_api_key_dict: UserAPIKeyAuth, request_payload: dict, + url_route: str, ) -> None: """Fire LiteLLM-side failure hooks (spend tracking, alerting callbacks) for an upstream 4xx/5xx passthrough response. @@ -842,6 +856,13 @@ async def _log_passthrough_upstream_failure( return from litellm.proxy.proxy_server import proxy_logging_obj + upstream_error_body: Final = await _read_upstream_error_body_for_logging(response) + verbose_proxy_logger.warning( + "Passthrough upstream request to %s failed with status %s: %s", + url_route, + response.status_code, + upstream_error_body, + ) try: response.raise_for_status() except httpx.HTTPStatusError: @@ -854,7 +875,7 @@ async def _log_passthrough_upstream_failure( # rate-limit errors already are. synthetic_exception: Final = HTTPException( status_code=response.status_code, - detail=f"Upstream passthrough request failed with status {response.status_code}", + detail=f"Upstream passthrough request failed with status {response.status_code}: {upstream_error_body}", ) try: await proxy_logging_obj.post_call_failure_hook( @@ -1274,6 +1295,7 @@ async def pass_through_request( custom_llm_provider=custom_llm_provider, upstream_usage=upstream_usage, ), + url_route=str(url), ) # Call response headers hook for streaming pass-through @@ -1355,6 +1377,7 @@ async def pass_through_request( custom_llm_provider=custom_llm_provider, upstream_usage=upstream_usage, ), + url_route=str(url), ) # Call response headers hook for detected streaming pass-through @@ -1449,6 +1472,7 @@ async def pass_through_request( response=response, user_api_key_dict=user_api_key_dict, request_payload=failure_request_payload, + url_route=str(url), ) if response.status_code < 400 and response_body is not None and guardrails_to_run: diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 91367507247..f5036858ddd 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -3988,6 +3988,172 @@ async def test_pass_through_request_streaming_upstream_error_returned_unchanged( assert failure_call_kwargs["original_exception"].status_code == 403 +_UPSTREAM_FAILURE_WARNING_MARKER = "Passthrough upstream request to" + + +def _upstream_failure_warnings(caplog): + return [ + record.getMessage() + for record in caplog.records + if record.levelno == logging.WARNING and _UPSTREAM_FAILURE_WARNING_MARKER in record.getMessage() + ] + + +@pytest.mark.asyncio +async def test_pass_through_request_upstream_error_body_is_logged_and_carried_into_failure_hook(caplog): + """Regression (LIT-6644): the upstream error body must reach the WARNING log and the failure hook.""" + from fastapi import HTTPException + + from litellm.proxy._types import UserAPIKeyAuth + + upstream_error = { + "error": { + "code": 404, + "message": "Publisher model `claude-nope-9` was not found or your project does not have access to it.", + "status": "NOT_FOUND", + } + } + upstream_content = json.dumps(upstream_error).encode("utf-8") + fake_client, cleanup = _inject_fake_passthrough_client( + _FakeUpstreamTransport( + status_code=404, + headers={"content-type": "application/json"}, + stream=_RecordingUpstreamByteStream((upstream_content,)), + ), + timeout=314.0, + ) + try: + with ExitStack() as stack: + mock_proxy_logging, _ = _enter_relay_logging_mocks(stack, {}) + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + response = await pass_through_request( + request=_relay_client_request(method="POST"), + target="http://upstream.test/v1/publishers/anthropic/models/claude-nope-9:streamRawPredict", + custom_headers={}, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-relay-test"), + timeout=314.0, + ) + finally: + cleanup() + await fake_client.aclose() + + assert response.status_code == 404 + assert json.loads(response.body) == upstream_error + + warnings = _upstream_failure_warnings(caplog) + assert len(warnings) == 1 + assert "http://upstream.test/v1/publishers/anthropic/models/claude-nope-9:streamRawPredict" in warnings[0] + assert "status 404" in warnings[0] + assert "was not found or your project does not have access to it" in warnings[0] + + original_exception = mock_proxy_logging.post_call_failure_hook.call_args.kwargs["original_exception"] + assert isinstance(original_exception, HTTPException) + assert original_exception.status_code == 404 + assert "was not found or your project does not have access to it" in original_exception.detail + assert "NOT_FOUND" in str(original_exception) + + +@pytest.mark.asyncio +async def test_pass_through_request_streaming_upstream_error_body_is_logged_and_still_relayed(caplog): + """Regression (LIT-6644): logging a streamed error body must not consume the bytes owed to the client.""" + from fastapi.responses import StreamingResponse + + from litellm.proxy._types import UserAPIKeyAuth + + upstream_error = { + "type": "error", + "error": { + "type": "invalid_request_error", + "message": "Unexpected value(s) `context-1m-2025-08-07` for the `anthropic-beta` header.", + }, + } + upstream_content = json.dumps(upstream_error).encode("utf-8") + upstream_stream = _RecordingUpstreamByteStream((upstream_content[:20], upstream_content[20:])) + fake_client, cleanup = _inject_fake_passthrough_client( + _FakeUpstreamTransport( + status_code=400, + headers={"content-type": "application/json"}, + stream=upstream_stream, + ), + timeout=314.0, + ) + try: + with ExitStack() as stack: + mock_proxy_logging, mock_success_handler = _enter_relay_logging_mocks(stack, {}) + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + response = await pass_through_request( + request=_relay_client_request(method="POST"), + target="http://upstream.test/v1/messages", + custom_headers={}, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-relay-test"), + stream=True, + timeout=314.0, + ) + assert isinstance(response, StreamingResponse) + assert response.status_code == 400 + streamed_chunks = [chunk async for chunk in response.body_iterator] + finally: + cleanup() + await fake_client.aclose() + + streamed_bytes = b"".join(chunk if isinstance(chunk, bytes) else chunk.encode("utf-8") for chunk in streamed_chunks) + assert streamed_bytes == upstream_content + assert upstream_stream.closed is True + mock_success_handler.assert_not_called() + + warnings = _upstream_failure_warnings(caplog) + assert len(warnings) == 1 + assert "http://upstream.test/v1/messages" in warnings[0] + assert "status 400" in warnings[0] + assert "for the `anthropic-beta` header" in warnings[0] + + original_exception = mock_proxy_logging.post_call_failure_hook.call_args.kwargs["original_exception"] + assert original_exception.status_code == 400 + assert "invalid_request_error" in original_exception.detail + + +@pytest.mark.asyncio +async def test_pass_through_request_upstream_error_body_is_truncated_in_logs(caplog): + from litellm.constants import PASSTHROUGH_UPSTREAM_ERROR_BODY_MAX_LOG_CHARS + from litellm.proxy._types import UserAPIKeyAuth + + oversized_message = "x" * (PASSTHROUGH_UPSTREAM_ERROR_BODY_MAX_LOG_CHARS * 3) + upstream_content = json.dumps({"error": oversized_message}).encode("utf-8") + fake_client, cleanup = _inject_fake_passthrough_client( + _FakeUpstreamTransport( + status_code=500, + headers={"content-type": "application/json"}, + stream=_RecordingUpstreamByteStream((upstream_content,)), + ), + timeout=314.0, + ) + try: + with ExitStack() as stack: + mock_proxy_logging, _ = _enter_relay_logging_mocks(stack, {}) + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + response = await pass_through_request( + request=_relay_client_request(method="POST"), + target="http://upstream.test/v1/huge-error", + custom_headers={}, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-relay-test"), + timeout=314.0, + ) + finally: + cleanup() + await fake_client.aclose() + + assert response.body == upstream_content + + warnings = _upstream_failure_warnings(caplog) + assert len(warnings) == 1 + assert len(warnings[0]) < len(upstream_content) + assert "truncated" in warnings[0] + + detail = mock_proxy_logging.post_call_failure_hook.call_args.kwargs["original_exception"].detail + assert len(detail) < len(upstream_content) + assert "truncated" in detail + + @pytest.mark.asyncio async def test_pass_through_request_non_streaming_success_unchanged(): """Success (2xx) passthrough behavior must remain unchanged by the error fix."""