mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
Merge 54c6cc60ec into 10b3deae0d
This commit is contained in:
commit
69f086207c
4 changed files with 208 additions and 18 deletions
|
|
@ -8,6 +8,12 @@ from litellm.constants import PROVIDER_REQUEST_ID_HEADERS
|
|||
|
||||
_HEADERS: Final = TypeAdapter(Mapping[str, str])
|
||||
_REQUEST_ID_HEADERS: Final = ("request-id", "x-request-id", "llm_provider-request-id", "llm_provider-x-request-id")
|
||||
_PROVIDER_HEADERS_TO_PASSTHROUGH: Final[frozenset[str]] = frozenset(
|
||||
{
|
||||
"anthropic-ratelimit-unified-status",
|
||||
"retry-after",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def get_provider_request_id(headers: object) -> str | None:
|
||||
|
|
@ -58,7 +64,10 @@ def get_response_headers(_response_headers: Mapping[str, str] | None = None) ->
|
|||
llm_provider_headers: Final = _get_llm_provider_headers(_response_headers)
|
||||
request_id: Final = get_provider_request_id(_response_headers)
|
||||
native_headers: Final = {"request-id": request_id} if request_id is not None else {}
|
||||
return {**llm_provider_headers, **openai_headers, **native_headers}
|
||||
passthrough_headers: Final = {
|
||||
key: value for key, value in _response_headers.items() if key.lower() in _PROVIDER_HEADERS_TO_PASSTHROUGH
|
||||
}
|
||||
return {**llm_provider_headers, **openai_headers, **native_headers, **passthrough_headers}
|
||||
|
||||
|
||||
def _get_llm_provider_headers(response_headers: Mapping[str, str]) -> dict:
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from litellm.anthropic_interface.exceptions import (
|
|||
AnthropicErrorResponse,
|
||||
AnthropicExceptionMapping,
|
||||
)
|
||||
from litellm.constants import UNSAFE_PROXY_RESPONSE_HEADERS
|
||||
from litellm.integrations.custom_guardrail import ModifyResponseException
|
||||
from litellm.litellm_core_utils.tool_fields import first_validation_error
|
||||
from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicMessagesHandler
|
||||
|
|
@ -58,6 +59,45 @@ from litellm.types.utils import TokenCountResponse
|
|||
|
||||
router: Final = APIRouter()
|
||||
|
||||
_FORWARDABLE_UPSTREAM_HEADERS: Final[frozenset[str]] = frozenset({"retry-after", "request-id"})
|
||||
_FORWARDABLE_UPSTREAM_PREFIX: Final[str] = "anthropic-"
|
||||
|
||||
|
||||
def _extract_upstream_anthropic_headers(e: Exception) -> dict[str, str]:
|
||||
"""Return the upstream provider's Anthropic-native rate-limit and protocol headers from a
|
||||
mapped LiteLLM exception, so they survive the proxy's error wrapping (#37754).
|
||||
|
||||
The ``/v1/messages`` endpoint speaks the Anthropic protocol, so safe headers
|
||||
(``retry-after``, ``request-id``, ``anthropic-*``) are forwarded unprefixed.
|
||||
Keys are lowercased and filtered to avoid exposing unsafe browser framing headers or
|
||||
letting an upstream forge proxy-owned ``x-litellm-*`` metadata.
|
||||
"""
|
||||
raw = getattr(e, "litellm_response_headers", None) or getattr(e, "headers", None)
|
||||
if not raw:
|
||||
_response = getattr(e, "response", None)
|
||||
raw = getattr(_response, "headers", None) if _response is not None else None
|
||||
if not raw and hasattr(e, "original_exception"):
|
||||
orig = getattr(e, "original_exception")
|
||||
if orig is not None:
|
||||
raw = (
|
||||
getattr(orig, "litellm_response_headers", None)
|
||||
or getattr(orig, "headers", None)
|
||||
or getattr(getattr(orig, "response", None), "headers", None)
|
||||
)
|
||||
if not raw:
|
||||
return {}
|
||||
try:
|
||||
items = raw.items()
|
||||
except AttributeError:
|
||||
return {}
|
||||
return {
|
||||
key: str(v)
|
||||
for k, v in items
|
||||
if (key := str(k).lower()) not in UNSAFE_PROXY_RESPONSE_HEADERS
|
||||
and not key.startswith("x-litellm-")
|
||||
and (key in _FORWARDABLE_UPSTREAM_HEADERS or key.startswith(_FORWARDABLE_UPSTREAM_PREFIX))
|
||||
}
|
||||
|
||||
_NO_HEADERS: Final[Mapping[str, str]] = MappingProxyType({})
|
||||
|
||||
|
||||
|
|
@ -268,7 +308,12 @@ async def anthropic_response(
|
|||
)
|
||||
log_llm_api_exception(e, base_llm_response_processor.litellm_call_id)
|
||||
|
||||
# Preserve upstream rate-limit headers so clients can fail fast on 429 (#37754).
|
||||
upstream_headers: Final = _extract_upstream_anthropic_headers(e)
|
||||
|
||||
if isinstance(e, ProxyException):
|
||||
if upstream_headers:
|
||||
e.headers = {**upstream_headers, **(e.headers or {})}
|
||||
return _anthropic_error_json_response(
|
||||
with_litellm_call_id(e, base_llm_response_processor.litellm_call_id), request
|
||||
)
|
||||
|
|
@ -279,7 +324,7 @@ async def anthropic_response(
|
|||
model_id: Final = model_info.get("id", "") or ""
|
||||
|
||||
# Get headers
|
||||
headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers(
|
||||
custom_headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
call_id=base_llm_response_processor.litellm_call_id,
|
||||
model_id=model_id,
|
||||
|
|
@ -290,6 +335,7 @@ async def anthropic_response(
|
|||
timeout=getattr(e, "timeout", None),
|
||||
litellm_logging_obj=None,
|
||||
)
|
||||
headers: Final = {**upstream_headers, **custom_headers}
|
||||
|
||||
if isinstance(e, HTTPException):
|
||||
return _anthropic_error_json_response(proxy_exception_from_http_exception(e, headers), request)
|
||||
|
|
|
|||
|
|
@ -118,7 +118,7 @@ async def _blocked_messages_reply(
|
|||
) -> tuple[object, str]:
|
||||
import litellm
|
||||
import litellm.proxy.anthropic_endpoints.endpoints as ep
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
|
||||
blocker: Final = _BlockEveryRequest(
|
||||
|
|
@ -173,8 +173,8 @@ class TestBlockedResponseUsage:
|
|||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import litellm.proxy.anthropic_endpoints.endpoints as ep
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.integrations.custom_guardrail import ModifyResponseException
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
exc = ModifyResponseException(
|
||||
message="blocked by guardrail",
|
||||
|
|
@ -255,7 +255,7 @@ class TestProxyExceptionAnthropicEnvelope:
|
|||
Anthropic's documented {"type": "error", "error": {...}} envelope with the
|
||||
original status and message, not the OpenAI {"error": {...}} envelope."""
|
||||
import litellm.proxy.anthropic_endpoints.endpoints as ep
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import ProxyErrorTypes, ProxyException
|
||||
|
||||
exc = ProxyException(
|
||||
|
|
@ -386,7 +386,7 @@ class TestHttpExceptionDictDetail:
|
|||
from fastapi import HTTPException
|
||||
|
||||
import litellm.proxy.anthropic_endpoints.endpoints as ep
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
detail = {
|
||||
|
|
@ -435,7 +435,7 @@ class TestFailureHookRequestData:
|
|||
object the failure hook needs to lift token usage from); the exception
|
||||
handler must pass that replaced dict, not the raw request body dict."""
|
||||
import litellm.proxy.anthropic_endpoints.endpoints as ep
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
captured = {}
|
||||
|
|
@ -487,7 +487,7 @@ class TestErrorLogCarriesCallId:
|
|||
@pytest.mark.asyncio
|
||||
async def test_messages_failure_log_carries_call_id(self, caplog: pytest.LogCaptureFixture):
|
||||
import litellm.proxy.anthropic_endpoints.endpoints as ep
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
call_id = "messages-call-7836"
|
||||
|
|
@ -526,7 +526,7 @@ class TestErrorLogCarriesCallId:
|
|||
@pytest.mark.asyncio
|
||||
async def test_messages_already_shaped_failure_answers_with_the_call_id(self):
|
||||
import litellm.proxy.anthropic_endpoints.endpoints as ep
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import ProxyErrorTypes, ProxyException, UserAPIKeyAuth
|
||||
|
||||
call_id = "messages-call-7836-shaped"
|
||||
|
|
@ -562,7 +562,7 @@ class TestErrorLogCarriesCallId:
|
|||
@pytest.mark.asyncio
|
||||
async def test_count_tokens_failure_log_carries_callers_call_id(self, caplog: pytest.LogCaptureFixture):
|
||||
import litellm.proxy.anthropic_endpoints.endpoints as ep
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
call_id = "count-tokens-call-7836"
|
||||
|
|
@ -596,7 +596,7 @@ class TestCountTokensCompactedHistory:
|
|||
@pytest.mark.asyncio
|
||||
async def test_count_tokens_counts_a_compaction_block_like_its_summary_text(self):
|
||||
import litellm.proxy.anthropic_endpoints.endpoints as ep
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
summary = "The user is building a recipe app and asked for one-sentence class descriptions."
|
||||
|
|
@ -661,7 +661,7 @@ _SEVEN_TOKENS: Final = TokenCountResponse(
|
|||
)
|
||||
async def test_count_tokens_answers_400_for_malformed_tools_without_counting(model: str, tools: object) -> None:
|
||||
import litellm.proxy.anthropic_endpoints.endpoints as ep
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
request: Final = MagicMock()
|
||||
|
|
@ -684,7 +684,7 @@ async def test_count_tokens_answers_400_for_malformed_tools_without_counting(mod
|
|||
@pytest.mark.asyncio
|
||||
async def test_count_tokens_counts_malformed_tool_entries_for_a_model_the_router_cannot_name() -> None:
|
||||
import litellm.proxy.anthropic_endpoints.endpoints as ep
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
request: Final = MagicMock()
|
||||
|
|
@ -717,7 +717,7 @@ async def test_count_tokens_drops_malformed_tools_when_drop_params_is_set(
|
|||
dropping: dict[str, object], headers: dict[str, str]
|
||||
) -> None:
|
||||
import litellm.proxy.anthropic_endpoints.endpoints as ep
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
request: Final = MagicMock()
|
||||
|
|
@ -741,7 +741,7 @@ async def test_count_tokens_drops_malformed_tools_when_drop_params_is_set(
|
|||
@pytest.mark.asyncio
|
||||
async def test_count_tokens_keeps_claude_code_malformed_tools_when_operator_sets_drop_params_false() -> None:
|
||||
import litellm.proxy.anthropic_endpoints.endpoints as ep
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
operator_config: Final = proxy_server.ProxyConfig()
|
||||
|
|
@ -767,8 +767,8 @@ async def test_count_tokens_keeps_claude_code_malformed_tools_when_operator_sets
|
|||
@pytest.mark.asyncio
|
||||
async def test_count_tokens_drops_malformed_tools_the_counted_deployment_drops_even_with_fallbacks() -> None:
|
||||
import litellm.proxy.anthropic_endpoints.endpoints as ep
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm import Router
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
router: Final = Router(
|
||||
|
|
@ -809,9 +809,9 @@ def _count_on_claude(
|
|||
) -> httpx.Response:
|
||||
from fastapi import FastAPI
|
||||
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.llms.anthropic.count_tokens import handler as anthropic_count_tokens_handler
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.anthropic_endpoints.endpoints import router
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
|
|
@ -923,7 +923,7 @@ def test_count_tokens_provider_rejection_logs_its_status_and_answers_the_same_er
|
|||
def _count_on_local_model(monkeypatch: pytest.MonkeyPatch, body: Mapping[str, object]) -> httpx.Response:
|
||||
from fastapi import FastAPI
|
||||
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.anthropic_endpoints.endpoints import router
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
|
|
@ -1108,3 +1108,118 @@ class TestStripTotalTokensFeatureFlag(unittest.TestCase):
|
|||
import litellm
|
||||
|
||||
assert litellm.strip_anthropic_total_tokens is False
|
||||
|
||||
|
||||
|
||||
class TestUpstreamRateLimitHeaderPassthrough:
|
||||
"""Issue #37754: on a 429 from an Anthropic-compatible upstream (e.g. GLM),
|
||||
the `/v1/messages` endpoint must forward the upstream's Anthropic-native
|
||||
rate-limit headers (`retry-after`, `anthropic-ratelimit-unified-status`)
|
||||
to the client. Without them, Claude Code treats the 429 as a transient
|
||||
throttle and retries forever.
|
||||
"""
|
||||
|
||||
def _build_upstream_ratelimit_exception(self):
|
||||
"""Produce the exception the proxy sees when an Anthropic-compatible
|
||||
upstream returns 429, through the real litellm exception mapping so
|
||||
`litellm_response_headers` is populated exactly as in production."""
|
||||
import httpx
|
||||
|
||||
from litellm.llms.anthropic.common_utils import AnthropicError
|
||||
|
||||
upstream_headers = httpx.Headers(
|
||||
{
|
||||
"anthropic-ratelimit-unified-status": "rejected",
|
||||
"retry-after": "287441",
|
||||
"request-id": "20260821091810b67f7ae6443d450c",
|
||||
"anthropic-ratelimit-requests-limit": "1000",
|
||||
"set-cookie": "secret=1; HttpOnly",
|
||||
"access-control-allow-origin": "*",
|
||||
"content-type": "application/json",
|
||||
}
|
||||
)
|
||||
return AnthropicError(
|
||||
status_code=429,
|
||||
message="Rate limit exceeded",
|
||||
headers=upstream_headers,
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_response_forwards_upstream_rate_limit_headers(self):
|
||||
import litellm.proxy.anthropic_endpoints.endpoints as ep
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
mapped = self._build_upstream_ratelimit_exception()
|
||||
|
||||
req = _messages_request({"model": "glm", "messages": [{"role": "user", "content": "hi"}]})
|
||||
with patch.object(
|
||||
ep.ProxyBaseLLMRequestProcessing,
|
||||
"base_process_llm_request",
|
||||
new=AsyncMock(side_effect=mapped),
|
||||
):
|
||||
response = await ep.anthropic_response(
|
||||
fastapi_response=MagicMock(),
|
||||
request=req,
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
)
|
||||
|
||||
headers = {k.lower(): v for k, v in response.headers.items()}
|
||||
assert response.status_code == 429
|
||||
assert headers.get("anthropic-ratelimit-unified-status") == "rejected"
|
||||
assert headers.get("retry-after") == "287441"
|
||||
assert headers.get("anthropic-ratelimit-requests-limit") == "1000"
|
||||
assert headers.get("request-id") == "20260821091810b67f7ae6443d450c"
|
||||
assert "set-cookie" not in headers
|
||||
assert "access-control-allow-origin" not in headers
|
||||
assert "x-litellm-call-id" in headers or "x-litellm-version" in headers
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upstream_cannot_spoof_litellm_headers(self):
|
||||
import litellm.proxy.anthropic_endpoints.endpoints as ep
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
mapped = self._build_upstream_ratelimit_exception()
|
||||
mapped.litellm_response_headers = {
|
||||
"retry-after": "287441",
|
||||
"x-litellm-version": "spoofed",
|
||||
"X-LiteLLM-Model-ID": "forged-model",
|
||||
"x-litellm-call-id": "fake-call-id",
|
||||
}
|
||||
|
||||
req = _messages_request({"model": "glm", "messages": [{"role": "user", "content": "hi"}]})
|
||||
with patch.object(
|
||||
ep.ProxyBaseLLMRequestProcessing,
|
||||
"base_process_llm_request",
|
||||
new=AsyncMock(side_effect=mapped),
|
||||
):
|
||||
response = await ep.anthropic_response(
|
||||
fastapi_response=MagicMock(),
|
||||
request=req,
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
)
|
||||
|
||||
headers = {k.lower(): v for k, v in response.headers.items()}
|
||||
assert headers.get("retry-after") == "287441"
|
||||
assert headers.get("x-litellm-version") != "spoofed"
|
||||
assert headers.get("x-litellm-model-id") != "forged-model"
|
||||
|
||||
def test_extract_upstream_headers_edge_cases(self):
|
||||
from litellm.proxy.anthropic_endpoints.endpoints import _extract_upstream_anthropic_headers
|
||||
|
||||
e = Exception("plain error")
|
||||
assert _extract_upstream_anthropic_headers(e) == {}
|
||||
|
||||
e.litellm_response_headers = "not-a-dict"
|
||||
assert _extract_upstream_anthropic_headers(e) == {}
|
||||
|
||||
e.litellm_response_headers = {
|
||||
"retry-after": "60",
|
||||
"Anthropic-RateLimit-Unified-Status": "rejected",
|
||||
"set-cookie": "secret",
|
||||
"cross-origin-opener-policy": "same-origin",
|
||||
}
|
||||
res = _extract_upstream_anthropic_headers(e)
|
||||
assert res == {
|
||||
"retry-after": "60",
|
||||
"anthropic-ratelimit-unified-status": "rejected",
|
||||
}
|
||||
|
|
|
|||
|
|
@ -338,6 +338,26 @@ class TestProxyHeaderExtraction:
|
|||
assert headers.get("llm_provider-x-request-id") == "req-abc123"
|
||||
assert headers.get("llm_provider-x-ms-region") == "eastus"
|
||||
|
||||
def test_get_response_headers_preserves_anthropic_rate_limit_headers(self):
|
||||
"""Anthropic-compatible and rate-limited clients need these headers on downstream 429."""
|
||||
from litellm.litellm_core_utils.llm_response_utils.get_headers import (
|
||||
get_response_headers,
|
||||
)
|
||||
|
||||
result = get_response_headers(
|
||||
{
|
||||
"Anthropic-RateLimit-Unified-Status": "rejected",
|
||||
"Retry-After": "287441",
|
||||
"x-request-id": "req-123",
|
||||
}
|
||||
)
|
||||
|
||||
assert result["Anthropic-RateLimit-Unified-Status"] == "rejected"
|
||||
assert result["Retry-After"] == "287441"
|
||||
assert result["llm_provider-Anthropic-RateLimit-Unified-Status"] == "rejected"
|
||||
assert result["llm_provider-Retry-After"] == "287441"
|
||||
assert result["llm_provider-x-request-id"] == "req-123"
|
||||
|
||||
|
||||
class TestBedrockErrorHeaders:
|
||||
"""A BedrockError built with headers but no response still exposes them (LIT-5428)."""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue