From 54c6cc60eccaa3a26f4459b428420cb847a194b9 Mon Sep 17 00:00:00 2001 From: DewellNaa <210961864+DewellNaa@users.noreply.github.com> Date: Fri, 30 Oct 2026 11:06:03 -0700 Subject: [PATCH] fix(proxy): forward anthropic 429 rate-limit and retry-after headers (#37754) - On `/v1/messages` error path, extract and forward allowlisted Anthropic protocol rate limit headers (`retry-after`, `request-id`, `anthropic-*`) so clients like Claude Code recognize rejected quotas and fail fast instead of retrying endlessly. - Strip unsafe framing/browser security headers via `UNSAFE_PROXY_RESPONSE_HEADERS` and ensure proxy-owned `x-litellm-*` headers strictly override upstream keys. - Preserve `retry-after` and `anthropic-ratelimit-unified-status` in `get_response_headers` for provider exception handling. - Add unit tests in `test_endpoints.py` and `test_exception_header_preservation.py` verifying header forwarding, header spoof isolation, and non-mapping error robustness. --- .../llm_response_utils/get_headers.py | 11 +- .../proxy/anthropic_endpoints/endpoints.py | 48 +++++- .../anthropic_endpoints/test_endpoints.py | 147 ++++++++++++++++-- .../test_exception_header_preservation.py | 20 +++ 4 files changed, 208 insertions(+), 18 deletions(-) diff --git a/litellm/litellm_core_utils/llm_response_utils/get_headers.py b/litellm/litellm_core_utils/llm_response_utils/get_headers.py index 3c08f16fa80..cdf66114bc4 100644 --- a/litellm/litellm_core_utils/llm_response_utils/get_headers.py +++ b/litellm/litellm_core_utils/llm_response_utils/get_headers.py @@ -8,6 +8,12 @@ from litellm.constants import PROVIDER_REQUEST_ID_HEADERS _HEADERS: Final = TypeAdapter(Mapping[str, str]) _REQUEST_ID_HEADERS: Final = ("request-id", "x-request-id", "llm_provider-request-id", "llm_provider-x-request-id") +_PROVIDER_HEADERS_TO_PASSTHROUGH: Final[frozenset[str]] = frozenset( + { + "anthropic-ratelimit-unified-status", + "retry-after", + } +) def get_provider_request_id(headers: object) -> str | None: @@ -58,7 +64,10 @@ def get_response_headers(_response_headers: Mapping[str, str] | None = None) -> llm_provider_headers: Final = _get_llm_provider_headers(_response_headers) request_id: Final = get_provider_request_id(_response_headers) native_headers: Final = {"request-id": request_id} if request_id is not None else {} - return {**llm_provider_headers, **openai_headers, **native_headers} + passthrough_headers: Final = { + key: value for key, value in _response_headers.items() if key.lower() in _PROVIDER_HEADERS_TO_PASSTHROUGH + } + return {**llm_provider_headers, **openai_headers, **native_headers, **passthrough_headers} def _get_llm_provider_headers(response_headers: Mapping[str, str]) -> dict: diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 0e204cf9211..77ff7ec834d 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -17,6 +17,7 @@ from litellm.anthropic_interface.exceptions import ( AnthropicErrorResponse, AnthropicExceptionMapping, ) +from litellm.constants import UNSAFE_PROXY_RESPONSE_HEADERS from litellm.integrations.custom_guardrail import ModifyResponseException from litellm.litellm_core_utils.tool_fields import first_validation_error from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicMessagesHandler @@ -58,6 +59,45 @@ from litellm.types.utils import TokenCountResponse router: Final = APIRouter() +_FORWARDABLE_UPSTREAM_HEADERS: Final[frozenset[str]] = frozenset({"retry-after", "request-id"}) +_FORWARDABLE_UPSTREAM_PREFIX: Final[str] = "anthropic-" + + +def _extract_upstream_anthropic_headers(e: Exception) -> dict[str, str]: + """Return the upstream provider's Anthropic-native rate-limit and protocol headers from a + mapped LiteLLM exception, so they survive the proxy's error wrapping (#37754). + + The ``/v1/messages`` endpoint speaks the Anthropic protocol, so safe headers + (``retry-after``, ``request-id``, ``anthropic-*``) are forwarded unprefixed. + Keys are lowercased and filtered to avoid exposing unsafe browser framing headers or + letting an upstream forge proxy-owned ``x-litellm-*`` metadata. + """ + raw = getattr(e, "litellm_response_headers", None) or getattr(e, "headers", None) + if not raw: + _response = getattr(e, "response", None) + raw = getattr(_response, "headers", None) if _response is not None else None + if not raw and hasattr(e, "original_exception"): + orig = getattr(e, "original_exception") + if orig is not None: + raw = ( + getattr(orig, "litellm_response_headers", None) + or getattr(orig, "headers", None) + or getattr(getattr(orig, "response", None), "headers", None) + ) + if not raw: + return {} + try: + items = raw.items() + except AttributeError: + return {} + return { + key: str(v) + for k, v in items + if (key := str(k).lower()) not in UNSAFE_PROXY_RESPONSE_HEADERS + and not key.startswith("x-litellm-") + and (key in _FORWARDABLE_UPSTREAM_HEADERS or key.startswith(_FORWARDABLE_UPSTREAM_PREFIX)) + } + _NO_HEADERS: Final[Mapping[str, str]] = MappingProxyType({}) @@ -268,7 +308,12 @@ async def anthropic_response( ) log_llm_api_exception(e, base_llm_response_processor.litellm_call_id) + # Preserve upstream rate-limit headers so clients can fail fast on 429 (#37754). + upstream_headers: Final = _extract_upstream_anthropic_headers(e) + if isinstance(e, ProxyException): + if upstream_headers: + e.headers = {**upstream_headers, **(e.headers or {})} return _anthropic_error_json_response( with_litellm_call_id(e, base_llm_response_processor.litellm_call_id), request ) @@ -279,7 +324,7 @@ async def anthropic_response( model_id: Final = model_info.get("id", "") or "" # Get headers - headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers( + custom_headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers( user_api_key_dict=user_api_key_dict, call_id=base_llm_response_processor.litellm_call_id, model_id=model_id, @@ -290,6 +335,7 @@ async def anthropic_response( timeout=getattr(e, "timeout", None), litellm_logging_obj=None, ) + headers: Final = {**upstream_headers, **custom_headers} if isinstance(e, HTTPException): return _anthropic_error_json_response(proxy_exception_from_http_exception(e, headers), request) diff --git a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py index 604fc84a096..d9a03aa16c4 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py +++ b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py @@ -118,7 +118,7 @@ async def _blocked_messages_reply( ) -> tuple[object, str]: import litellm import litellm.proxy.anthropic_endpoints.endpoints as ep - import litellm.proxy.proxy_server as proxy_server + from litellm.proxy import proxy_server from litellm.proxy.utils import ProxyLogging blocker: Final = _BlockEveryRequest( @@ -173,8 +173,8 @@ class TestBlockedResponseUsage: from unittest.mock import AsyncMock, MagicMock import litellm.proxy.anthropic_endpoints.endpoints as ep - import litellm.proxy.proxy_server as proxy_server from litellm.integrations.custom_guardrail import ModifyResponseException + from litellm.proxy import proxy_server exc = ModifyResponseException( message="blocked by guardrail", @@ -255,7 +255,7 @@ class TestProxyExceptionAnthropicEnvelope: Anthropic's documented {"type": "error", "error": {...}} envelope with the original status and message, not the OpenAI {"error": {...}} envelope.""" import litellm.proxy.anthropic_endpoints.endpoints as ep - import litellm.proxy.proxy_server as proxy_server + from litellm.proxy import proxy_server from litellm.proxy._types import ProxyErrorTypes, ProxyException exc = ProxyException( @@ -386,7 +386,7 @@ class TestHttpExceptionDictDetail: from fastapi import HTTPException import litellm.proxy.anthropic_endpoints.endpoints as ep - import litellm.proxy.proxy_server as proxy_server + from litellm.proxy import proxy_server from litellm.proxy._types import UserAPIKeyAuth detail = { @@ -435,7 +435,7 @@ class TestFailureHookRequestData: object the failure hook needs to lift token usage from); the exception handler must pass that replaced dict, not the raw request body dict.""" import litellm.proxy.anthropic_endpoints.endpoints as ep - import litellm.proxy.proxy_server as proxy_server + from litellm.proxy import proxy_server from litellm.proxy._types import UserAPIKeyAuth captured = {} @@ -487,7 +487,7 @@ class TestErrorLogCarriesCallId: @pytest.mark.asyncio async def test_messages_failure_log_carries_call_id(self, caplog: pytest.LogCaptureFixture): import litellm.proxy.anthropic_endpoints.endpoints as ep - import litellm.proxy.proxy_server as proxy_server + from litellm.proxy import proxy_server from litellm.proxy._types import UserAPIKeyAuth call_id = "messages-call-7836" @@ -526,7 +526,7 @@ class TestErrorLogCarriesCallId: @pytest.mark.asyncio async def test_messages_already_shaped_failure_answers_with_the_call_id(self): import litellm.proxy.anthropic_endpoints.endpoints as ep - import litellm.proxy.proxy_server as proxy_server + from litellm.proxy import proxy_server from litellm.proxy._types import ProxyErrorTypes, ProxyException, UserAPIKeyAuth call_id = "messages-call-7836-shaped" @@ -562,7 +562,7 @@ class TestErrorLogCarriesCallId: @pytest.mark.asyncio async def test_count_tokens_failure_log_carries_callers_call_id(self, caplog: pytest.LogCaptureFixture): import litellm.proxy.anthropic_endpoints.endpoints as ep - import litellm.proxy.proxy_server as proxy_server + from litellm.proxy import proxy_server from litellm.proxy._types import UserAPIKeyAuth call_id = "count-tokens-call-7836" @@ -596,7 +596,7 @@ class TestCountTokensCompactedHistory: @pytest.mark.asyncio async def test_count_tokens_counts_a_compaction_block_like_its_summary_text(self): import litellm.proxy.anthropic_endpoints.endpoints as ep - import litellm.proxy.proxy_server as proxy_server + from litellm.proxy import proxy_server from litellm.proxy._types import UserAPIKeyAuth summary = "The user is building a recipe app and asked for one-sentence class descriptions." @@ -661,7 +661,7 @@ _SEVEN_TOKENS: Final = TokenCountResponse( ) async def test_count_tokens_answers_400_for_malformed_tools_without_counting(model: str, tools: object) -> None: import litellm.proxy.anthropic_endpoints.endpoints as ep - import litellm.proxy.proxy_server as proxy_server + from litellm.proxy import proxy_server from litellm.proxy._types import UserAPIKeyAuth request: Final = MagicMock() @@ -684,7 +684,7 @@ async def test_count_tokens_answers_400_for_malformed_tools_without_counting(mod @pytest.mark.asyncio async def test_count_tokens_counts_malformed_tool_entries_for_a_model_the_router_cannot_name() -> None: import litellm.proxy.anthropic_endpoints.endpoints as ep - import litellm.proxy.proxy_server as proxy_server + from litellm.proxy import proxy_server from litellm.proxy._types import UserAPIKeyAuth request: Final = MagicMock() @@ -717,7 +717,7 @@ async def test_count_tokens_drops_malformed_tools_when_drop_params_is_set( dropping: dict[str, object], headers: dict[str, str] ) -> None: import litellm.proxy.anthropic_endpoints.endpoints as ep - import litellm.proxy.proxy_server as proxy_server + from litellm.proxy import proxy_server from litellm.proxy._types import UserAPIKeyAuth request: Final = MagicMock() @@ -741,7 +741,7 @@ async def test_count_tokens_drops_malformed_tools_when_drop_params_is_set( @pytest.mark.asyncio async def test_count_tokens_keeps_claude_code_malformed_tools_when_operator_sets_drop_params_false() -> None: import litellm.proxy.anthropic_endpoints.endpoints as ep - import litellm.proxy.proxy_server as proxy_server + from litellm.proxy import proxy_server from litellm.proxy._types import UserAPIKeyAuth operator_config: Final = proxy_server.ProxyConfig() @@ -767,8 +767,8 @@ async def test_count_tokens_keeps_claude_code_malformed_tools_when_operator_sets @pytest.mark.asyncio async def test_count_tokens_drops_malformed_tools_the_counted_deployment_drops_even_with_fallbacks() -> None: import litellm.proxy.anthropic_endpoints.endpoints as ep - import litellm.proxy.proxy_server as proxy_server from litellm import Router + from litellm.proxy import proxy_server from litellm.proxy._types import UserAPIKeyAuth router: Final = Router( @@ -809,9 +809,9 @@ def _count_on_claude( ) -> httpx.Response: from fastapi import FastAPI - import litellm.proxy.proxy_server as proxy_server from litellm.llms.anthropic.count_tokens import handler as anthropic_count_tokens_handler from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.proxy import proxy_server from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.anthropic_endpoints.endpoints import router from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -923,7 +923,7 @@ def test_count_tokens_provider_rejection_logs_its_status_and_answers_the_same_er def _count_on_local_model(monkeypatch: pytest.MonkeyPatch, body: Mapping[str, object]) -> httpx.Response: from fastapi import FastAPI - import litellm.proxy.proxy_server as proxy_server + from litellm.proxy import proxy_server from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.anthropic_endpoints.endpoints import router from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -1108,3 +1108,118 @@ class TestStripTotalTokensFeatureFlag(unittest.TestCase): import litellm assert litellm.strip_anthropic_total_tokens is False + + + +class TestUpstreamRateLimitHeaderPassthrough: + """Issue #37754: on a 429 from an Anthropic-compatible upstream (e.g. GLM), + the `/v1/messages` endpoint must forward the upstream's Anthropic-native + rate-limit headers (`retry-after`, `anthropic-ratelimit-unified-status`) + to the client. Without them, Claude Code treats the 429 as a transient + throttle and retries forever. + """ + + def _build_upstream_ratelimit_exception(self): + """Produce the exception the proxy sees when an Anthropic-compatible + upstream returns 429, through the real litellm exception mapping so + `litellm_response_headers` is populated exactly as in production.""" + import httpx + + from litellm.llms.anthropic.common_utils import AnthropicError + + upstream_headers = httpx.Headers( + { + "anthropic-ratelimit-unified-status": "rejected", + "retry-after": "287441", + "request-id": "20260821091810b67f7ae6443d450c", + "anthropic-ratelimit-requests-limit": "1000", + "set-cookie": "secret=1; HttpOnly", + "access-control-allow-origin": "*", + "content-type": "application/json", + } + ) + return AnthropicError( + status_code=429, + message="Rate limit exceeded", + headers=upstream_headers, + ) + + @pytest.mark.asyncio + async def test_anthropic_response_forwards_upstream_rate_limit_headers(self): + import litellm.proxy.anthropic_endpoints.endpoints as ep + from litellm.proxy._types import UserAPIKeyAuth + + mapped = self._build_upstream_ratelimit_exception() + + req = _messages_request({"model": "glm", "messages": [{"role": "user", "content": "hi"}]}) + with patch.object( + ep.ProxyBaseLLMRequestProcessing, + "base_process_llm_request", + new=AsyncMock(side_effect=mapped), + ): + response = await ep.anthropic_response( + fastapi_response=MagicMock(), + request=req, + user_api_key_dict=UserAPIKeyAuth(), + ) + + headers = {k.lower(): v for k, v in response.headers.items()} + assert response.status_code == 429 + assert headers.get("anthropic-ratelimit-unified-status") == "rejected" + assert headers.get("retry-after") == "287441" + assert headers.get("anthropic-ratelimit-requests-limit") == "1000" + assert headers.get("request-id") == "20260821091810b67f7ae6443d450c" + assert "set-cookie" not in headers + assert "access-control-allow-origin" not in headers + assert "x-litellm-call-id" in headers or "x-litellm-version" in headers + + @pytest.mark.asyncio + async def test_upstream_cannot_spoof_litellm_headers(self): + import litellm.proxy.anthropic_endpoints.endpoints as ep + from litellm.proxy._types import UserAPIKeyAuth + + mapped = self._build_upstream_ratelimit_exception() + mapped.litellm_response_headers = { + "retry-after": "287441", + "x-litellm-version": "spoofed", + "X-LiteLLM-Model-ID": "forged-model", + "x-litellm-call-id": "fake-call-id", + } + + req = _messages_request({"model": "glm", "messages": [{"role": "user", "content": "hi"}]}) + with patch.object( + ep.ProxyBaseLLMRequestProcessing, + "base_process_llm_request", + new=AsyncMock(side_effect=mapped), + ): + response = await ep.anthropic_response( + fastapi_response=MagicMock(), + request=req, + user_api_key_dict=UserAPIKeyAuth(), + ) + + headers = {k.lower(): v for k, v in response.headers.items()} + assert headers.get("retry-after") == "287441" + assert headers.get("x-litellm-version") != "spoofed" + assert headers.get("x-litellm-model-id") != "forged-model" + + def test_extract_upstream_headers_edge_cases(self): + from litellm.proxy.anthropic_endpoints.endpoints import _extract_upstream_anthropic_headers + + e = Exception("plain error") + assert _extract_upstream_anthropic_headers(e) == {} + + e.litellm_response_headers = "not-a-dict" + assert _extract_upstream_anthropic_headers(e) == {} + + e.litellm_response_headers = { + "retry-after": "60", + "Anthropic-RateLimit-Unified-Status": "rejected", + "set-cookie": "secret", + "cross-origin-opener-policy": "same-origin", + } + res = _extract_upstream_anthropic_headers(e) + assert res == { + "retry-after": "60", + "anthropic-ratelimit-unified-status": "rejected", + } diff --git a/tests/unit/test_exception_header_preservation.py b/tests/unit/test_exception_header_preservation.py index c5a0868f36d..65ed06832d6 100644 --- a/tests/unit/test_exception_header_preservation.py +++ b/tests/unit/test_exception_header_preservation.py @@ -338,6 +338,26 @@ class TestProxyHeaderExtraction: assert headers.get("llm_provider-x-request-id") == "req-abc123" assert headers.get("llm_provider-x-ms-region") == "eastus" + def test_get_response_headers_preserves_anthropic_rate_limit_headers(self): + """Anthropic-compatible and rate-limited clients need these headers on downstream 429.""" + from litellm.litellm_core_utils.llm_response_utils.get_headers import ( + get_response_headers, + ) + + result = get_response_headers( + { + "Anthropic-RateLimit-Unified-Status": "rejected", + "Retry-After": "287441", + "x-request-id": "req-123", + } + ) + + assert result["Anthropic-RateLimit-Unified-Status"] == "rejected" + assert result["Retry-After"] == "287441" + assert result["llm_provider-Anthropic-RateLimit-Unified-Status"] == "rejected" + assert result["llm_provider-Retry-After"] == "287441" + assert result["llm_provider-x-request-id"] == "req-123" + class TestBedrockErrorHeaders: """A BedrockError built with headers but no response still exposes them (LIT-5428)."""