mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
1009976c49
* fix(bedrock): keep x-amzn-RequestId on chat error responses Bedrock chat error paths built BedrockError from only a status code and a message, so the provider response headers were gone before exception mapping ran and the proxy had nothing to forward. AWS support needs x-amzn-RequestId to investigate a server-side error. - converse and invoke chat handlers pass the real headers and response when they turn an httpx.HTTPStatusError into a BedrockError, and read the body through error_response_text so a streamed body nobody read does not throw - every bedrock chat get_error_class honors the headers it is already handed: invoke, moonshot, bedrock-hosted openai, agentcore and the invoke agent - BedrockError carries those headers into the response it synthesizes when a caller has headers but no response, skipping values httpx cannot carry - the bedrock 500 mapping forwards the provider response like its 4xx and 503 siblings instead of fabricating a blank one The proxy now returns llm_provider-x-amzn-requestid on Bedrock chat errors. * fix(bedrock): keep request-id on text-classified errors The context-window and image branches of _map_bedrock_exception built their litellm exception without the provider response, so a Bedrock 400 classified by its body text lost x-amzn-RequestId while the sibling branches kept it. Also narrows the new BedrockError types and trims its docstrings. * chore(bedrock): drop the docstrings on the new error helpers * fix(bedrock): keep request-id on every error path that has one The ticket's root cause is that every BedrockError raise site under litellm/llms/bedrock/ was built from status and message alone. The first commits covered the chat and invoke handlers; this covers the rest. Embeddings, rerank, image generation, image edit, count tokens, search and the transformation layers now hand on the provider response or its headers, and both bedrock_mantle configs return a BedrockError instead of the OpenAI error that drops them. Two blockers surfaced while verifying the streaming path. The trailing `except Exception` in make_call and make_sync_call swallowed the BedrockError raised a few lines above, relabelling a provider status as a 500, and the non-200 branch read an unread streamed body, which throws. The raise sites left alone have no provider response to carry: timeouts, credential and config errors, and mid-stream event frames. * fix(bedrock): forward provider headers from the count tokens route The count tokens route converts BedrockError into an HTTPException, and dropped the headers the handler had just kept, so that route still lost the request id. get_response_headers now takes a Mapping so an httpx.Headers can be handed to it without a copy. * fix(bedrock): classify every bedrock surface through BedrockError Eleven bedrock configs still inherited a provider-agnostic get_error_class that builds a blank response, so the request id was gone before the proxy read it. Claude platform, bedrock anthropic-messages, both image edit configs, passthrough, realtime, vector stores and agentcore search now return BedrockError, and a parametrized audit drives all 36 configs. * fix(proxy): keep provider headers on the httpx status error branch _handle_llm_api_exception forwards safe_headers on every branch except the httpx.HTTPStatusError one, which the bedrock passthrough route reaches, so the request id was dropped before the client saw the response. * fix(bedrock): keep the request id on the timeout mappings Timeout takes no response argument, so the three bedrock timeout branches dropped the provider headers even when the upstream answered 408 or 504 with an x-amzn-RequestId. They now ride on the exception, already llm_provider-prefixed, which is the form the proxy emits. * fix(bedrock): keep the provider response on mapped timeouts The previous round attached llm_provider-prefixed headers directly to the Timeout. That shadowed the raw upstream headers for _get_response_headers, so router cooldown and fallback cooldown stopped honouring retry-after on bedrock 408/504 replies. Give Timeout an optional response instead, the way every other mapped bedrock exception already carries one. Retry logic reads the raw retry-after off the response, and the proxy prefixes those headers on the way out, so clients still see llm_provider-x-amzn-requestid. * chore(bedrock): drop the explanatory comment on Timeout.response
397 lines
15 KiB
Python
397 lines
15 KiB
Python
"""
|
|
Tests for exception header preservation.
|
|
|
|
These tests verify that when LLM providers return error responses with headers,
|
|
those headers are preserved in the exception and can be returned to clients.
|
|
|
|
This is important for debugging and observability - headers like x-request-id,
|
|
x-ms-region, rate limit headers, etc. should be available even when errors occur.
|
|
"""
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from litellm.exceptions import (
|
|
BadRequestError,
|
|
ContentPolicyViolationError,
|
|
ContextWindowExceededError,
|
|
ImageFetchError,
|
|
MidStreamFallbackError,
|
|
RateLimitError,
|
|
ServiceUnavailableError,
|
|
)
|
|
|
|
|
|
class TestExceptionHeaderPreservation:
|
|
"""Test that exception classes preserve headers from provider responses."""
|
|
|
|
@pytest.fixture
|
|
def mock_response_with_headers(self) -> httpx.Response:
|
|
"""Create a mock response with typical provider headers."""
|
|
return httpx.Response(
|
|
status_code=400,
|
|
headers={
|
|
"x-request-id": "req-abc123",
|
|
"x-ms-region": "eastus",
|
|
"x-ratelimit-remaining-requests": "99",
|
|
"x-ratelimit-remaining-tokens": "9999",
|
|
},
|
|
request=httpx.Request("POST", "https://api.openai.com/v1/chat/completions"),
|
|
)
|
|
|
|
def test_bad_request_error_preserves_headers(
|
|
self, mock_response_with_headers: httpx.Response
|
|
):
|
|
"""BadRequestError should preserve headers from the provider response."""
|
|
error = BadRequestError(
|
|
message="Invalid request",
|
|
model="gpt-4",
|
|
llm_provider="azure",
|
|
response=mock_response_with_headers,
|
|
)
|
|
|
|
assert error.response is not None
|
|
assert error.response.headers.get("x-request-id") == "req-abc123"
|
|
assert error.response.headers.get("x-ms-region") == "eastus"
|
|
assert error.response.headers.get("x-ratelimit-remaining-requests") == "99"
|
|
|
|
def test_content_policy_violation_error_preserves_headers(
|
|
self, mock_response_with_headers: httpx.Response
|
|
):
|
|
"""ContentPolicyViolationError should preserve headers from the provider response."""
|
|
error = ContentPolicyViolationError(
|
|
message="Content policy violation",
|
|
model="gpt-4",
|
|
llm_provider="azure",
|
|
response=mock_response_with_headers,
|
|
)
|
|
|
|
assert error.response is not None
|
|
assert error.response.headers.get("x-request-id") == "req-abc123"
|
|
assert error.response.headers.get("x-ms-region") == "eastus"
|
|
|
|
def test_context_window_exceeded_error_preserves_headers(
|
|
self, mock_response_with_headers: httpx.Response
|
|
):
|
|
"""ContextWindowExceededError should preserve headers from the provider response."""
|
|
error = ContextWindowExceededError(
|
|
message="Context window exceeded",
|
|
model="gpt-4",
|
|
llm_provider="azure",
|
|
response=mock_response_with_headers,
|
|
)
|
|
|
|
assert error.response is not None
|
|
assert error.response.headers.get("x-request-id") == "req-abc123"
|
|
assert error.response.headers.get("x-ms-region") == "eastus"
|
|
|
|
def test_image_fetch_error_preserves_headers(
|
|
self, mock_response_with_headers: httpx.Response
|
|
):
|
|
"""ImageFetchError should preserve headers from the provider response."""
|
|
error = ImageFetchError(
|
|
message="Failed to fetch image",
|
|
model="gpt-4",
|
|
llm_provider="azure",
|
|
response=mock_response_with_headers,
|
|
)
|
|
|
|
assert error.response is not None
|
|
assert error.response.headers.get("x-request-id") == "req-abc123"
|
|
assert error.response.headers.get("x-ms-region") == "eastus"
|
|
|
|
def test_bad_request_error_handles_none_response(self):
|
|
"""BadRequestError should handle None response gracefully."""
|
|
error = BadRequestError(
|
|
message="Invalid request",
|
|
model="gpt-4",
|
|
llm_provider="azure",
|
|
response=None,
|
|
)
|
|
|
|
assert error.response is not None
|
|
# Headers should be empty but not cause an error
|
|
assert error.response.headers.get("x-request-id") is None
|
|
|
|
def test_content_policy_violation_error_handles_none_response(self):
|
|
"""ContentPolicyViolationError should handle None response gracefully."""
|
|
error = ContentPolicyViolationError(
|
|
message="Content policy violation",
|
|
model="gpt-4",
|
|
llm_provider="azure",
|
|
response=None,
|
|
)
|
|
|
|
assert error.response is not None
|
|
assert error.response.headers.get("x-request-id") is None
|
|
|
|
def test_context_window_exceeded_error_handles_none_response(self):
|
|
"""ContextWindowExceededError should handle None response gracefully."""
|
|
error = ContextWindowExceededError(
|
|
message="Context window exceeded",
|
|
model="gpt-4",
|
|
llm_provider="azure",
|
|
response=None,
|
|
)
|
|
|
|
assert error.response is not None
|
|
assert error.response.headers.get("x-request-id") is None
|
|
|
|
|
|
class TestExceptionMessageFormatting:
|
|
"""Test that exception messages are formatted correctly after refactoring."""
|
|
|
|
def test_bad_request_error_message_format(self):
|
|
"""BadRequestError should format message with litellm prefix."""
|
|
error = BadRequestError(
|
|
message="test error",
|
|
model="gpt-4",
|
|
llm_provider="azure",
|
|
)
|
|
|
|
assert "litellm.BadRequestError" in error.message
|
|
assert "test error" in error.message
|
|
|
|
def test_content_policy_violation_error_message_format(self):
|
|
"""ContentPolicyViolationError should format message with specific prefix."""
|
|
error = ContentPolicyViolationError(
|
|
message="test error",
|
|
model="gpt-4",
|
|
llm_provider="azure",
|
|
)
|
|
|
|
assert "litellm.ContentPolicyViolationError" in error.message
|
|
assert "test error" in error.message
|
|
|
|
def test_context_window_exceeded_error_message_format(self):
|
|
"""ContextWindowExceededError should format message with specific prefix."""
|
|
error = ContextWindowExceededError(
|
|
message="test error",
|
|
model="gpt-4",
|
|
llm_provider="azure",
|
|
)
|
|
|
|
assert "litellm.ContextWindowExceededError" in error.message
|
|
assert "test error" in error.message
|
|
|
|
|
|
class TestExceptionAttributes:
|
|
"""Test that exception attributes are set correctly."""
|
|
|
|
def test_content_policy_violation_error_provider_specific_fields(self):
|
|
"""ContentPolicyViolationError should preserve provider_specific_fields."""
|
|
provider_fields = {"innererror": {"code": "ResponsibleAIPolicyViolation"}}
|
|
|
|
error = ContentPolicyViolationError(
|
|
message="test error",
|
|
model="gpt-4",
|
|
llm_provider="azure",
|
|
provider_specific_fields=provider_fields,
|
|
)
|
|
|
|
assert error.provider_specific_fields == provider_fields
|
|
assert (
|
|
error.provider_specific_fields["innererror"]["code"]
|
|
== "ResponsibleAIPolicyViolation"
|
|
)
|
|
|
|
def test_bad_request_error_attributes(self):
|
|
"""BadRequestError should set all expected attributes."""
|
|
error = BadRequestError(
|
|
message="test error",
|
|
model="gpt-4",
|
|
llm_provider="azure",
|
|
litellm_debug_info="debug info",
|
|
max_retries=3,
|
|
num_retries=1,
|
|
)
|
|
|
|
assert error.model == "gpt-4"
|
|
assert error.llm_provider == "azure"
|
|
assert error.litellm_debug_info == "debug info"
|
|
assert error.max_retries == 3
|
|
assert error.num_retries == 1
|
|
assert error.status_code == 400
|
|
|
|
def test_midstream_fallback_error_status_code_propagation(self):
|
|
"""
|
|
MidStreamFallbackError should preserve the original status code and keep
|
|
message/request/response fields consistent after super().__init__().
|
|
"""
|
|
original_req = httpx.Request(
|
|
"POST", "https://api.openai.com/v1/chat/completions"
|
|
)
|
|
original_resp = httpx.Response(status_code=429, request=original_req)
|
|
|
|
rate_limit_error = RateLimitError(
|
|
message="Rate limit exceeded",
|
|
llm_provider="openai",
|
|
model="gpt-4o-mini",
|
|
response=original_resp,
|
|
)
|
|
|
|
midstream_error = MidStreamFallbackError(
|
|
message="stream broke",
|
|
model="gpt-4o-mini",
|
|
llm_provider="openai",
|
|
original_exception=rate_limit_error,
|
|
)
|
|
|
|
assert midstream_error.status_code == 429
|
|
assert midstream_error.response.status_code == 429
|
|
assert str(midstream_error.response.request.url) == "https://openai.com/v1/"
|
|
assert midstream_error.message == "litellm.MidStreamFallbackError: stream broke"
|
|
assert midstream_error.args == ("litellm.MidStreamFallbackError: stream broke",)
|
|
|
|
# With no original exception, should default to 503.
|
|
midstream_fallback = MidStreamFallbackError(
|
|
message="stream broke without original",
|
|
model="gpt-4o-mini",
|
|
llm_provider="openai",
|
|
original_exception=None,
|
|
)
|
|
|
|
assert midstream_fallback.status_code == 503
|
|
assert midstream_fallback.response.status_code == 503
|
|
assert str(midstream_fallback.response.request.url) == "https://openai.com/v1/"
|
|
|
|
|
|
class TestProxyHeaderExtraction:
|
|
"""Test that proxy correctly extracts headers from exceptions."""
|
|
|
|
def test_get_response_headers_adds_llm_provider_prefix(self):
|
|
"""get_response_headers should prefix non-OpenAI headers with llm_provider-."""
|
|
from litellm.litellm_core_utils.llm_response_utils.get_headers import (
|
|
get_response_headers,
|
|
)
|
|
|
|
response_headers = {
|
|
"x-request-id": "req-abc123",
|
|
"x-ms-region": "eastus",
|
|
"x-ratelimit-remaining-requests": "99", # OpenAI header - should not be prefixed
|
|
}
|
|
|
|
result = get_response_headers(response_headers)
|
|
|
|
# OpenAI ratelimit headers should be preserved as-is
|
|
assert result.get("x-ratelimit-remaining-requests") == "99"
|
|
# Other headers should be prefixed with llm_provider-
|
|
assert result.get("llm_provider-x-request-id") == "req-abc123"
|
|
assert result.get("llm_provider-x-ms-region") == "eastus"
|
|
|
|
def test_proxy_can_extract_headers_from_exception_response(self):
|
|
"""Simulate how proxy extracts headers from exception.response.headers."""
|
|
from litellm.litellm_core_utils.llm_response_utils.get_headers import (
|
|
get_response_headers,
|
|
)
|
|
|
|
# Create exception with headers in response
|
|
mock_response = httpx.Response(
|
|
status_code=400,
|
|
headers={
|
|
"x-request-id": "req-abc123",
|
|
"x-ms-region": "eastus",
|
|
},
|
|
request=httpx.Request("POST", "https://test.com"),
|
|
)
|
|
error = ContentPolicyViolationError(
|
|
message="test",
|
|
model="gpt-4",
|
|
llm_provider="azure",
|
|
response=mock_response,
|
|
)
|
|
|
|
# Simulate proxy header extraction logic
|
|
headers = getattr(error, "headers", None) or {}
|
|
if not headers:
|
|
_response = getattr(error, "response", None)
|
|
if _response is not None:
|
|
_response_headers = getattr(_response, "headers", None)
|
|
if _response_headers:
|
|
headers = get_response_headers(dict(_response_headers))
|
|
|
|
# Verify headers are extracted and prefixed correctly
|
|
assert headers.get("llm_provider-x-request-id") == "req-abc123"
|
|
assert headers.get("llm_provider-x-ms-region") == "eastus"
|
|
|
|
|
|
class TestBedrockErrorHeaders:
|
|
"""A BedrockError built with headers but no response still exposes them (LIT-5428)."""
|
|
|
|
def test_synthesized_response_carries_headers(self):
|
|
from litellm.llms.bedrock.common_utils import BedrockError
|
|
|
|
error = BedrockError(
|
|
status_code=500,
|
|
message="Amazon Bedrock is unable to process your request.",
|
|
headers={"x-amzn-RequestId": "req-base-500"},
|
|
)
|
|
|
|
assert error.response.headers["x-amzn-requestid"] == "req-base-500"
|
|
assert str(error.request.url) == str(BedrockError(status_code=500, message="boom").request.url)
|
|
assert str(error.response.request.url) == str(error.request.url)
|
|
|
|
def test_synthesized_response_without_headers_stays_empty(self):
|
|
from litellm.llms.bedrock.common_utils import BedrockError
|
|
|
|
error = BedrockError(status_code=500, message="boom")
|
|
|
|
assert dict(error.response.headers) == {}
|
|
|
|
def test_explicit_response_is_kept(self):
|
|
from litellm.llms.bedrock.common_utils import BedrockError
|
|
|
|
provider_response = httpx.Response(
|
|
status_code=500,
|
|
headers={"x-amzn-RequestId": "from-response"},
|
|
request=httpx.Request("POST", "https://bedrock-runtime.us-east-1.amazonaws.com/"),
|
|
)
|
|
error = BedrockError(
|
|
status_code=500,
|
|
message="boom",
|
|
headers={"x-amzn-RequestId": "from-headers"},
|
|
response=provider_response,
|
|
)
|
|
|
|
assert error.response is provider_response
|
|
|
|
def test_proxy_extraction_surfaces_bedrock_request_id(self):
|
|
"""End-to-end shape the proxy error handler returns to the caller."""
|
|
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
|
|
from litellm.litellm_core_utils.llm_response_utils.get_headers import (
|
|
get_response_headers,
|
|
)
|
|
from litellm.llms.bedrock.common_utils import BedrockError
|
|
|
|
provider_response = httpx.Response(
|
|
status_code=500,
|
|
headers={"x-amzn-RequestId": "req-proxy-500"},
|
|
text='{"message":"Amazon Bedrock is unable to process your request."}',
|
|
request=httpx.Request("POST", "https://bedrock-runtime.us-east-1.amazonaws.com/"),
|
|
)
|
|
|
|
with pytest.raises(ServiceUnavailableError) as exc_info:
|
|
exception_type(
|
|
model="anthropic.claude-haiku-4-5-20251001-v1:0",
|
|
original_exception=BedrockError(
|
|
status_code=500,
|
|
message=provider_response.text,
|
|
headers=provider_response.headers,
|
|
response=provider_response,
|
|
),
|
|
custom_llm_provider="bedrock",
|
|
completion_kwargs={},
|
|
extra_kwargs={},
|
|
)
|
|
|
|
# Mirrors ProxyBaseLLMRequestProcessing._handle_llm_api_exception
|
|
error = exc_info.value
|
|
headers = getattr(error, "headers", None) or {}
|
|
if not headers:
|
|
_response = getattr(error, "response", None)
|
|
if _response is not None:
|
|
_response_headers = getattr(_response, "headers", None)
|
|
if _response_headers:
|
|
headers = get_response_headers(dict(_response_headers))
|
|
|
|
assert headers.get("llm_provider-x-amzn-requestid") == "req-proxy-500"
|