mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_lit_7022_azure_ai_passthrough_config
# Conflicts: # tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py
This commit is contained in:
commit
22d7616fe7
39 changed files with 1100 additions and 35 deletions
|
|
@ -338,6 +338,7 @@ class Timeout(openai.APITimeoutError):
|
|||
num_retries: int | None = None,
|
||||
headers: dict | None = None,
|
||||
exception_status_code: int | None = None,
|
||||
response: httpx.Response | None = None,
|
||||
):
|
||||
request: Final = httpx.Request(
|
||||
method="POST",
|
||||
|
|
@ -352,6 +353,8 @@ class Timeout(openai.APITimeoutError):
|
|||
self.max_retries = max_retries
|
||||
self.num_retries = num_retries
|
||||
self.headers = headers
|
||||
if response is not None:
|
||||
self.response = response
|
||||
|
||||
# custom function to convert to str
|
||||
def __str__(self):
|
||||
|
|
|
|||
|
|
@ -860,6 +860,7 @@ def _map_bedrock_exception(
|
|||
message=mantle_context_window_message,
|
||||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
response=getattr(original_exception, "response", None),
|
||||
)
|
||||
if (
|
||||
"too many tokens" in error_str
|
||||
|
|
@ -873,6 +874,7 @@ def _map_bedrock_exception(
|
|||
message=f"BedrockException: Context Window Error - {error_str}",
|
||||
model=model,
|
||||
llm_provider="bedrock",
|
||||
response=getattr(original_exception, "response", None),
|
||||
)
|
||||
elif "Conversation blocks and tool result blocks cannot be provided in the same turn." in error_str:
|
||||
raise BadRequestError(
|
||||
|
|
@ -924,12 +926,14 @@ def _map_bedrock_exception(
|
|||
message=f"BedrockException: Timeout Error - {error_str}",
|
||||
model=model,
|
||||
llm_provider="bedrock",
|
||||
response=getattr(original_exception, "response", None),
|
||||
)
|
||||
elif "Could not process image" in error_str:
|
||||
raise litellm.InternalServerError(
|
||||
message=f"BedrockException - {error_str}",
|
||||
model=model,
|
||||
llm_provider="bedrock",
|
||||
response=getattr(original_exception, "response", None),
|
||||
)
|
||||
elif hasattr(original_exception, "status_code"):
|
||||
if original_exception.status_code == 500:
|
||||
|
|
@ -937,10 +941,7 @@ def _map_bedrock_exception(
|
|||
message=f"BedrockException - {original_exception.message}",
|
||||
llm_provider="bedrock",
|
||||
model=model,
|
||||
response=httpx.Response(
|
||||
status_code=500,
|
||||
request=httpx.Request(method="POST", url="https://api.openai.com/v1/"),
|
||||
),
|
||||
response=getattr(original_exception, "response", None),
|
||||
)
|
||||
elif original_exception.status_code == 401:
|
||||
raise AuthenticationError(
|
||||
|
|
@ -969,6 +970,7 @@ def _map_bedrock_exception(
|
|||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
litellm_debug_info=extra_information,
|
||||
response=getattr(original_exception, "response", None),
|
||||
)
|
||||
elif original_exception.status_code == 422:
|
||||
raise BadRequestError(
|
||||
|
|
@ -1001,6 +1003,7 @@ def _map_bedrock_exception(
|
|||
llm_provider=custom_llm_provider,
|
||||
litellm_debug_info=extra_information,
|
||||
exception_status_code=original_exception.status_code,
|
||||
response=getattr(original_exception, "response", None),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
|
||||
def get_response_headers(_response_headers: dict | None = None) -> dict:
|
||||
def get_response_headers(_response_headers: Mapping[str, str] | None = None) -> dict:
|
||||
"""
|
||||
|
||||
Sets the Appropriate OpenAI headers for the response and forward all headers as llm_provider-{header}
|
||||
|
|
@ -31,7 +32,7 @@ def get_response_headers(_response_headers: dict | None = None) -> dict:
|
|||
return {**llm_provider_headers, **openai_headers}
|
||||
|
||||
|
||||
def _get_llm_provider_headers(response_headers: dict) -> dict:
|
||||
def _get_llm_provider_headers(response_headers: Mapping[str, str]) -> dict:
|
||||
"""
|
||||
Adds a llm_provider-{header} to all headers that are not already prefixed with llm_provider
|
||||
|
||||
|
|
|
|||
|
|
@ -667,7 +667,12 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
|
|||
)
|
||||
|
||||
if response.status_code != 200:
|
||||
raise BedrockError(status_code=response.status_code, message=str(response.read()))
|
||||
raise BedrockError(
|
||||
status_code=response.status_code,
|
||||
message=str(response.read()),
|
||||
headers=response.headers,
|
||||
response=response,
|
||||
)
|
||||
|
||||
# LOGGING
|
||||
logging_obj.post_call(
|
||||
|
|
@ -690,6 +695,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
|
|||
raise BedrockError(
|
||||
status_code=response.status_code,
|
||||
message=f"AgentCore: Failed to read/parse JSON response body: {e}",
|
||||
headers=response.headers,
|
||||
)
|
||||
parsed: Final = self._parse_json_response(response_json)
|
||||
|
||||
|
|
@ -880,7 +886,12 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
|
|||
)
|
||||
|
||||
if response.status_code != 200:
|
||||
raise BedrockError(status_code=response.status_code, message=str(await response.aread()))
|
||||
raise BedrockError(
|
||||
status_code=response.status_code,
|
||||
message=str(await response.aread()),
|
||||
headers=response.headers,
|
||||
response=response,
|
||||
)
|
||||
|
||||
# LOGGING
|
||||
logging_obj.post_call(
|
||||
|
|
@ -903,6 +914,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
|
|||
raise BedrockError(
|
||||
status_code=response.status_code,
|
||||
message=f"AgentCore: Failed to read/parse JSON response body: {e}",
|
||||
headers=response.headers,
|
||||
)
|
||||
parsed: Final = self._parse_json_response(response_json)
|
||||
|
||||
|
|
@ -1031,6 +1043,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
|
|||
raise BedrockError(
|
||||
message=f"Error processing response: {e}",
|
||||
status_code=raw_response.status_code,
|
||||
headers=raw_response.headers,
|
||||
)
|
||||
|
||||
def validate_environment(
|
||||
|
|
@ -1046,7 +1059,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
|
|||
return headers
|
||||
|
||||
def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException:
|
||||
return BedrockError(status_code=status_code, message=error_message)
|
||||
return BedrockError(status_code=status_code, message=error_message, headers=headers)
|
||||
|
||||
def should_fake_stream(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ from litellm.types.utils import ModelResponse
|
|||
from litellm.utils import CustomStreamWrapper
|
||||
|
||||
from ..base_aws_llm import BaseAWSLLM, Credentials, bedrock_bearer_token
|
||||
from ..common_utils import BedrockError, _get_all_bedrock_regions
|
||||
from ..common_utils import BedrockError, _get_all_bedrock_regions, error_response_text
|
||||
from .invoke_handler import AWSEventStreamDecoder, MockResponseIterator, make_call
|
||||
|
||||
|
||||
|
|
@ -66,7 +66,12 @@ def make_sync_call(
|
|||
)
|
||||
|
||||
if response.status_code != 200:
|
||||
raise BedrockError(status_code=response.status_code, message=str(response.read()))
|
||||
raise BedrockError(
|
||||
status_code=response.status_code,
|
||||
message=str(response.read()),
|
||||
headers=response.headers,
|
||||
response=response,
|
||||
)
|
||||
|
||||
if fake_stream:
|
||||
model_response: Final[ModelResponse] = litellm.AmazonConverseConfig()._transform_response(
|
||||
|
|
@ -247,7 +252,12 @@ class BedrockConverseLLM(BaseAWSLLM):
|
|||
response.raise_for_status()
|
||||
except httpx.HTTPStatusError as err:
|
||||
error_code: Final = err.response.status_code
|
||||
raise BedrockError(status_code=error_code, message=err.response.text)
|
||||
raise BedrockError(
|
||||
status_code=error_code,
|
||||
message=error_response_text(err.response),
|
||||
headers=err.response.headers,
|
||||
response=err.response,
|
||||
)
|
||||
except httpx.TimeoutException:
|
||||
raise BedrockError(status_code=408, message="Timeout error occurred.")
|
||||
|
||||
|
|
@ -594,7 +604,12 @@ class BedrockConverseLLM(BaseAWSLLM):
|
|||
response.raise_for_status()
|
||||
except httpx.HTTPStatusError as err:
|
||||
error_code: Final = err.response.status_code
|
||||
raise BedrockError(status_code=error_code, message=err.response.text)
|
||||
raise BedrockError(
|
||||
status_code=error_code,
|
||||
message=error_response_text(err.response),
|
||||
headers=err.response.headers,
|
||||
response=err.response,
|
||||
)
|
||||
except httpx.TimeoutException:
|
||||
raise BedrockError(status_code=408, message="Timeout error occurred.")
|
||||
|
||||
|
|
|
|||
|
|
@ -2255,6 +2255,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
raise BedrockError(
|
||||
message=f"Error converting to valid response block={e}. File an issue if litellm error - https://github.com/BerriAI/litellm/issues",
|
||||
status_code=422,
|
||||
headers=response.headers,
|
||||
)
|
||||
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -470,6 +470,7 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM):
|
|||
raise BedrockError(
|
||||
message=f"Error processing response: {e}",
|
||||
status_code=raw_response.status_code,
|
||||
headers=raw_response.headers,
|
||||
)
|
||||
|
||||
def validate_environment(
|
||||
|
|
@ -485,7 +486,7 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM):
|
|||
return headers
|
||||
|
||||
def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException:
|
||||
return BedrockError(status_code=status_code, message=error_message)
|
||||
return BedrockError(status_code=status_code, message=error_message, headers=headers)
|
||||
|
||||
def should_fake_stream(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ from litellm.types.utils import GenericStreamingChunk as GChunk
|
|||
from ..common_utils import (
|
||||
BedrockError,
|
||||
build_bedrock_stream_error,
|
||||
error_response_text,
|
||||
get_bedrock_response_stream_shape,
|
||||
get_bedrock_tool_name,
|
||||
)
|
||||
|
|
@ -184,7 +185,12 @@ async def make_call(
|
|||
)
|
||||
|
||||
if response.status_code != 200:
|
||||
raise BedrockError(status_code=response.status_code, message=response.text)
|
||||
raise BedrockError(
|
||||
status_code=response.status_code,
|
||||
message=error_response_text(response),
|
||||
headers=response.headers,
|
||||
response=response,
|
||||
)
|
||||
|
||||
if fake_stream:
|
||||
model_response: Final[ModelResponse] = litellm.AmazonConverseConfig()._transform_response(
|
||||
|
|
@ -228,9 +234,16 @@ async def make_call(
|
|||
)
|
||||
|
||||
return completion_stream, response.headers
|
||||
except BedrockError:
|
||||
raise
|
||||
except httpx.HTTPStatusError as err:
|
||||
error_code: Final = err.response.status_code
|
||||
raise BedrockError(status_code=error_code, message=err.response.text)
|
||||
raise BedrockError(
|
||||
status_code=error_code,
|
||||
message=error_response_text(err.response),
|
||||
headers=err.response.headers,
|
||||
response=err.response,
|
||||
)
|
||||
except httpx.TimeoutException:
|
||||
raise BedrockError(status_code=408, message="Timeout error occurred.")
|
||||
except Exception as e:
|
||||
|
|
@ -270,7 +283,12 @@ def make_sync_call(
|
|||
)
|
||||
|
||||
if response.status_code != 200:
|
||||
raise BedrockError(status_code=response.status_code, message=response.text)
|
||||
raise BedrockError(
|
||||
status_code=response.status_code,
|
||||
message=error_response_text(response),
|
||||
headers=response.headers,
|
||||
response=response,
|
||||
)
|
||||
|
||||
if fake_stream:
|
||||
model_response: Final[ModelResponse] = litellm.AmazonConverseConfig()._transform_response(
|
||||
|
|
@ -314,9 +332,16 @@ def make_sync_call(
|
|||
)
|
||||
|
||||
return completion_stream, response.headers
|
||||
except BedrockError:
|
||||
raise
|
||||
except httpx.HTTPStatusError as err:
|
||||
error_code: Final = err.response.status_code
|
||||
raise BedrockError(status_code=error_code, message=err.response.text)
|
||||
raise BedrockError(
|
||||
status_code=error_code,
|
||||
message=error_response_text(err.response),
|
||||
headers=err.response.headers,
|
||||
response=err.response,
|
||||
)
|
||||
except httpx.TimeoutException:
|
||||
raise BedrockError(status_code=408, message="Timeout error occurred.")
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -247,4 +247,4 @@ class AmazonMoonshotConfig(AmazonInvokeConfig, MoonshotChatConfig):
|
|||
|
||||
def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BedrockError:
|
||||
"""Return the appropriate error class for Bedrock."""
|
||||
return BedrockError(status_code=status_code, message=error_message)
|
||||
return BedrockError(status_code=status_code, message=error_message, headers=headers)
|
||||
|
|
|
|||
|
|
@ -182,4 +182,4 @@ class AmazonBedrockOpenAIConfig(OpenAIGPTConfig, BaseAWSLLM):
|
|||
|
||||
def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BedrockError:
|
||||
"""Return the appropriate error class for Bedrock."""
|
||||
return BedrockError(status_code=status_code, message=error_message)
|
||||
return BedrockError(status_code=status_code, message=error_message, headers=headers)
|
||||
|
|
|
|||
|
|
@ -212,6 +212,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
|
|||
raise BedrockError(
|
||||
message=f"Error parsing response: {raw_response.text}, error: {e}",
|
||||
status_code=raw_response.status_code,
|
||||
headers=raw_response.headers,
|
||||
)
|
||||
|
||||
verbose_logger.debug(
|
||||
|
|
@ -241,6 +242,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
|
|||
raise BedrockError(
|
||||
message=f"Error setting response content: {e}. Response: {completion_response}",
|
||||
status_code=raw_response.status_code,
|
||||
headers=raw_response.headers,
|
||||
)
|
||||
|
||||
# Calculate usage from headers
|
||||
|
|
|
|||
|
|
@ -295,7 +295,11 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
|
|||
try:
|
||||
completion_response: Final = raw_response.json()
|
||||
except Exception:
|
||||
raise BedrockError(message=raw_response.text, status_code=raw_response.status_code)
|
||||
raise BedrockError(
|
||||
message=raw_response.text,
|
||||
status_code=raw_response.status_code,
|
||||
headers=raw_response.headers,
|
||||
)
|
||||
verbose_logger.debug(
|
||||
"bedrock invoke response % s",
|
||||
json.dumps(completion_response, indent=4, default=str),
|
||||
|
|
@ -363,6 +367,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
|
|||
raise BedrockError(
|
||||
message=f"Error processing={raw_response.text}, Received error={e}",
|
||||
status_code=422,
|
||||
headers=raw_response.headers,
|
||||
)
|
||||
|
||||
try:
|
||||
|
|
@ -384,6 +389,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
|
|||
raise BedrockError(
|
||||
message=f"Error parsing received text={outputText}.\nError-{e}",
|
||||
status_code=raw_response.status_code,
|
||||
headers=raw_response.headers,
|
||||
)
|
||||
|
||||
## CALCULATING USAGE - bedrock returns usage in the headers
|
||||
|
|
@ -431,7 +437,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
|
|||
return merge_bedrock_invoke_headers(headers, guardrail_headers, metadata_headers, owned_names)
|
||||
|
||||
def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException:
|
||||
return BedrockError(status_code=status_code, message=error_message)
|
||||
return BedrockError(status_code=status_code, message=error_message, headers=headers)
|
||||
|
||||
@track_llm_api_timing()
|
||||
async def get_async_custom_stream_wrapper(
|
||||
|
|
|
|||
|
|
@ -1,7 +1,10 @@
|
|||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
||||
import litellm
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
||||
CLAUDE_PLATFORM_SERVICE_NAME: Final = "aws-external-anthropic"
|
||||
|
|
@ -15,6 +18,14 @@ def strip_claude_platform_route(model: str) -> str:
|
|||
|
||||
|
||||
class BedrockClaudePlatformMixin(BaseAWSLLM):
|
||||
def get_error_class(
|
||||
self,
|
||||
error_message: str,
|
||||
status_code: int,
|
||||
headers: dict[str, object] | httpx.Headers, # mutable-ok: base passes response headers as a dict
|
||||
) -> BedrockError:
|
||||
return BedrockError(status_code=status_code, message=error_message, headers=headers)
|
||||
|
||||
@staticmethod
|
||||
def _get_workspace_id(optional_params: dict, litellm_params: dict) -> str | None:
|
||||
workspace_id = (
|
||||
|
|
|
|||
|
|
@ -33,8 +33,53 @@ if TYPE_CHECKING:
|
|||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
||||
|
||||
_ERROR_REQUEST_URL: Final = "https://docs.litellm.ai/docs"
|
||||
|
||||
|
||||
def error_response_text(response: httpx.Response) -> str:
|
||||
try:
|
||||
return response.text
|
||||
except httpx.ResponseNotRead:
|
||||
return response.reason_phrase
|
||||
|
||||
|
||||
def _synthesize_error_response(
|
||||
*, status_code: int, headers: dict[str, object] | httpx.Headers, request: httpx.Request | None
|
||||
) -> tuple[httpx.Request, httpx.Response]:
|
||||
error_request: Final = request or httpx.Request(method="POST", url=_ERROR_REQUEST_URL)
|
||||
safe_headers: Final = (
|
||||
headers
|
||||
if isinstance(headers, httpx.Headers)
|
||||
else tuple((key, value) for key, value in headers.items() if isinstance(value, (str, bytes)))
|
||||
)
|
||||
return error_request, httpx.Response(status_code=status_code, headers=safe_headers, request=error_request)
|
||||
|
||||
|
||||
class BedrockError(BaseLLMException):
|
||||
pass
|
||||
def __init__(
|
||||
self,
|
||||
status_code: int,
|
||||
message: str,
|
||||
headers: dict[str, object] | httpx.Headers | None = None,
|
||||
request: httpx.Request | None = None,
|
||||
response: httpx.Response | None = None,
|
||||
body: dict[str, object] | None = None,
|
||||
status_code_is_synthesized: bool = False,
|
||||
) -> None:
|
||||
error_request, error_response = (
|
||||
_synthesize_error_response(status_code=status_code, headers=headers, request=request)
|
||||
if response is None and headers
|
||||
else (request, response)
|
||||
)
|
||||
super().__init__(
|
||||
status_code=status_code,
|
||||
message=message,
|
||||
headers=headers,
|
||||
request=error_request,
|
||||
response=error_response,
|
||||
body=body,
|
||||
status_code_is_synthesized=status_code_is_synthesized,
|
||||
)
|
||||
|
||||
|
||||
_BEDROCK_AWS_AUTH_PARAMETER_KEYS: Final[tuple[str, ...]] = (
|
||||
|
|
|
|||
|
|
@ -102,6 +102,8 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
|
|||
raise BedrockError(
|
||||
status_code=response.status_code,
|
||||
message=error_text,
|
||||
headers=response.headers,
|
||||
response=response,
|
||||
)
|
||||
|
||||
bedrock_response: Final = response.json()
|
||||
|
|
@ -124,6 +126,8 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
|
|||
raise BedrockError(
|
||||
status_code=e.response.status_code,
|
||||
message=e.response.text,
|
||||
headers=e.response.headers,
|
||||
response=e.response,
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.error("Error in CountTokens handler: %s", e)
|
||||
|
|
|
|||
|
|
@ -132,7 +132,12 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
response.raise_for_status()
|
||||
except httpx.HTTPStatusError as err:
|
||||
error_code: Final = err.response.status_code
|
||||
raise BedrockError(status_code=error_code, message=err.response.text)
|
||||
raise BedrockError(
|
||||
status_code=error_code,
|
||||
message=err.response.text,
|
||||
headers=err.response.headers,
|
||||
response=err.response,
|
||||
)
|
||||
except httpx.TimeoutException:
|
||||
raise BedrockError(status_code=408, message="Timeout error occurred.")
|
||||
|
||||
|
|
@ -161,7 +166,12 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
response.raise_for_status()
|
||||
except httpx.HTTPStatusError as err:
|
||||
error_code: Final = err.response.status_code
|
||||
raise BedrockError(status_code=error_code, message=err.response.text)
|
||||
raise BedrockError(
|
||||
status_code=error_code,
|
||||
message=err.response.text,
|
||||
headers=err.response.headers,
|
||||
response=err.response,
|
||||
)
|
||||
except httpx.TimeoutException:
|
||||
raise BedrockError(status_code=408, message="Timeout error occurred.")
|
||||
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ import httpx
|
|||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
from litellm.types.images.main import ImageEditOptionalRequestParams
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import FileTypes, ImageObject, ImageResponse
|
||||
|
|
@ -228,6 +229,14 @@ class BedrockAmazonNovaCanvasImageEditConfig(BaseImageEditConfig):
|
|||
"""
|
||||
return _supports_nova_canvas_image_edit_from_model_cost(model or "")
|
||||
|
||||
def get_error_class(
|
||||
self,
|
||||
error_message: str,
|
||||
status_code: int,
|
||||
headers: dict[str, object] | httpx.Headers, # mutable-ok: base passes response headers as a dict
|
||||
) -> BedrockError:
|
||||
return BedrockError(status_code=status_code, message=error_message, headers=headers)
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list:
|
||||
return [
|
||||
"n",
|
||||
|
|
|
|||
|
|
@ -114,7 +114,12 @@ class BedrockImageEdit(BaseAWSLLM):
|
|||
response.raise_for_status()
|
||||
except httpx.HTTPStatusError as err:
|
||||
error_code: Final = err.response.status_code
|
||||
raise BedrockError(status_code=error_code, message=err.response.text)
|
||||
raise BedrockError(
|
||||
status_code=error_code,
|
||||
message=err.response.text,
|
||||
headers=err.response.headers,
|
||||
response=err.response,
|
||||
)
|
||||
except httpx.TimeoutException:
|
||||
raise BedrockError(status_code=408, message="Timeout error occurred.")
|
||||
|
||||
|
|
@ -156,7 +161,12 @@ class BedrockImageEdit(BaseAWSLLM):
|
|||
response.raise_for_status()
|
||||
except httpx.HTTPStatusError as err:
|
||||
error_code: Final = err.response.status_code
|
||||
raise BedrockError(status_code=error_code, message=err.response.text)
|
||||
raise BedrockError(
|
||||
status_code=error_code,
|
||||
message=err.response.text,
|
||||
headers=err.response.headers,
|
||||
response=err.response,
|
||||
)
|
||||
except httpx.TimeoutException:
|
||||
raise BedrockError(status_code=408, message="Timeout error occurred.")
|
||||
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ from typing import TYPE_CHECKING, Any, Final
|
|||
import httpx
|
||||
|
||||
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
from litellm.types.images.main import ImageEditOptionalRequestParams
|
||||
from litellm.types.llms.stability import (
|
||||
OPENAI_SIZE_TO_STABILITY_ASPECT_RATIO,
|
||||
|
|
@ -84,6 +85,14 @@ class BedrockStabilityImageEditConfig(BaseImageEditConfig):
|
|||
return True
|
||||
return False
|
||||
|
||||
def get_error_class(
|
||||
self,
|
||||
error_message: str,
|
||||
status_code: int,
|
||||
headers: dict[str, object] | httpx.Headers, # mutable-ok: base passes response headers as a dict
|
||||
) -> BedrockError:
|
||||
return BedrockError(status_code=status_code, message=error_message, headers=headers)
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list:
|
||||
"""
|
||||
Return list of OpenAI params supported by Bedrock Stability.
|
||||
|
|
|
|||
|
|
@ -119,7 +119,12 @@ class BedrockImageGeneration(BaseAWSLLM):
|
|||
response.raise_for_status()
|
||||
except httpx.HTTPStatusError as err:
|
||||
error_code: Final = err.response.status_code
|
||||
raise BedrockError(status_code=error_code, message=err.response.text)
|
||||
raise BedrockError(
|
||||
status_code=error_code,
|
||||
message=err.response.text,
|
||||
headers=err.response.headers,
|
||||
response=err.response,
|
||||
)
|
||||
except httpx.TimeoutException:
|
||||
raise BedrockError(status_code=408, message="Timeout error occurred.")
|
||||
### FORMAT RESPONSE TO OPENAI FORMAT ###
|
||||
|
|
@ -162,7 +167,12 @@ class BedrockImageGeneration(BaseAWSLLM):
|
|||
response.raise_for_status()
|
||||
except httpx.HTTPStatusError as err:
|
||||
error_code: Final = err.response.status_code
|
||||
raise BedrockError(status_code=error_code, message=err.response.text)
|
||||
raise BedrockError(
|
||||
status_code=error_code,
|
||||
message=err.response.text,
|
||||
headers=err.response.headers,
|
||||
response=err.response,
|
||||
)
|
||||
except httpx.TimeoutException:
|
||||
raise BedrockError(status_code=408, message="Timeout error occurred.")
|
||||
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation
|
|||
AmazonInvokeConfig,
|
||||
)
|
||||
from litellm.llms.bedrock.common_utils import (
|
||||
BedrockError,
|
||||
apply_bedrock_invoke_structured_output,
|
||||
ensure_bedrock_anthropic_messages_tool_names,
|
||||
get_anthropic_beta_from_headers,
|
||||
|
|
@ -79,6 +80,14 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
|
||||
BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS = frozenset(BedrockInvokeAnthropicMessagesRequest.__annotations__.keys())
|
||||
|
||||
def get_error_class(
|
||||
self,
|
||||
error_message: str,
|
||||
status_code: int,
|
||||
headers: dict[str, object] | httpx.Headers, # mutable-ok: base passes response headers as a dict
|
||||
) -> BedrockError:
|
||||
return BedrockError(status_code=status_code, message=error_message, headers=headers)
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
BaseAnthropicMessagesConfig.__init__(self, **kwargs)
|
||||
AmazonInvokeConfig.__init__(self, **kwargs)
|
||||
|
|
|
|||
|
|
@ -2,13 +2,14 @@ import json
|
|||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Final, Optional, cast
|
||||
|
||||
import httpx
|
||||
from httpx import Response
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
|
||||
|
||||
from ..base_aws_llm import BaseAWSLLM
|
||||
from ..common_utils import BedrockEventStreamDecoderBase, BedrockModelInfo
|
||||
from ..common_utils import BedrockError, BedrockEventStreamDecoderBase, BedrockModelInfo
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from httpx import URL
|
||||
|
|
@ -18,6 +19,14 @@ if TYPE_CHECKING:
|
|||
|
||||
|
||||
class BedrockPassthroughConfig(BaseAWSLLM, BedrockModelInfo, BedrockEventStreamDecoderBase, BasePassthroughConfig):
|
||||
def get_error_class(
|
||||
self,
|
||||
error_message: str,
|
||||
status_code: int,
|
||||
headers: dict[str, object] | httpx.Headers, # mutable-ok: base passes response headers as a dict
|
||||
) -> BedrockError:
|
||||
return BedrockError(status_code=status_code, message=error_message, headers=headers)
|
||||
|
||||
def is_streaming_request(self, endpoint: str, request_data: dict) -> bool:
|
||||
return "stream" in endpoint
|
||||
|
||||
|
|
|
|||
|
|
@ -9,12 +9,14 @@ import json
|
|||
import uuid as uuid_lib
|
||||
from typing import Final, cast
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm._uuid import uuid
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
from litellm.llms.bedrock.realtime.trigger_audio import ready_trigger_pcm
|
||||
from litellm.types.llms.openai import (
|
||||
OpenAIRealtimeContentPartDone,
|
||||
|
|
@ -121,6 +123,14 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
|
|||
self._cumulative_usage = BedrockUsageEvent()
|
||||
self._reported_usage = BedrockUsageEvent()
|
||||
|
||||
def get_error_class(
|
||||
self,
|
||||
error_message: str,
|
||||
status_code: int,
|
||||
headers: dict[str, object] | httpx.Headers, # mutable-ok: base passes response headers as a dict
|
||||
) -> BedrockError:
|
||||
return BedrockError(status_code=status_code, message=error_message, headers=headers)
|
||||
|
||||
def validate_environment(self, headers: dict, model: str, api_key: str | None = None) -> dict:
|
||||
"""Validate environment - no special validation needed for Bedrock."""
|
||||
return headers
|
||||
|
|
|
|||
|
|
@ -46,7 +46,12 @@ class BedrockRerankHandler(BaseAWSLLM):
|
|||
response.raise_for_status()
|
||||
except httpx.HTTPStatusError as err:
|
||||
error_code: Final = err.response.status_code
|
||||
raise BedrockError(status_code=error_code, message=err.response.text)
|
||||
raise BedrockError(
|
||||
status_code=error_code,
|
||||
message=err.response.text,
|
||||
headers=err.response.headers,
|
||||
response=err.response,
|
||||
)
|
||||
except httpx.TimeoutException:
|
||||
raise BedrockError(status_code=408, message="Timeout error occurred.")
|
||||
|
||||
|
|
@ -117,7 +122,12 @@ class BedrockRerankHandler(BaseAWSLLM):
|
|||
response.raise_for_status()
|
||||
except httpx.HTTPStatusError as err:
|
||||
error_code: Final = err.response.status_code
|
||||
raise BedrockError(status_code=error_code, message=err.response.text)
|
||||
raise BedrockError(
|
||||
status_code=error_code,
|
||||
message=err.response.text,
|
||||
headers=err.response.headers,
|
||||
response=err.response,
|
||||
)
|
||||
except httpx.TimeoutException:
|
||||
raise BedrockError(status_code=408, message="Timeout error occurred.")
|
||||
|
||||
|
|
|
|||
|
|
@ -39,7 +39,6 @@ from typing import Final
|
|||
import httpx
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.search.transformation import (
|
||||
BaseSearchConfig,
|
||||
SearchResponse,
|
||||
|
|
@ -380,6 +379,7 @@ class AgentCoreSearchConfig(BaseSearchConfig, BaseAWSLLM):
|
|||
raise BedrockError(
|
||||
status_code=raw_response.status_code if raw_response.status_code >= 400 else 502,
|
||||
message=f"AgentCore gateway MCP error: {error}",
|
||||
headers=raw_response.headers,
|
||||
)
|
||||
|
||||
# A failed tools/call is reported in-band, as HTTP 200 with result.isError
|
||||
|
|
@ -389,6 +389,7 @@ class AgentCoreSearchConfig(BaseSearchConfig, BaseAWSLLM):
|
|||
raise BedrockError(
|
||||
status_code=raw_response.status_code if raw_response.status_code >= 400 else 502,
|
||||
message=f"AgentCore web search tool error: {self._tool_error_message(response_json)}",
|
||||
headers=raw_response.headers,
|
||||
)
|
||||
|
||||
text_items: Final = tuple(
|
||||
|
|
@ -440,6 +441,7 @@ class AgentCoreSearchConfig(BaseSearchConfig, BaseAWSLLM):
|
|||
raise BedrockError(
|
||||
status_code=502,
|
||||
message=f"AgentCore gateway returned SSE without a JSON data frame: {text[:200]}",
|
||||
headers=raw_response.headers,
|
||||
)
|
||||
|
||||
def get_error_class(
|
||||
|
|
@ -448,7 +450,7 @@ class AgentCoreSearchConfig(BaseSearchConfig, BaseAWSLLM):
|
|||
status_code: int,
|
||||
headers: dict, # mutable-ok: BaseSearchConfig.get_error_class takes the response headers as a dict
|
||||
) -> Exception:
|
||||
return BaseLLMException(
|
||||
return BedrockError(
|
||||
status_code=status_code,
|
||||
message=error_message,
|
||||
headers=headers,
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from litellm._logging import verbose_logger
|
|||
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
|
||||
from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
from litellm.types.integrations.rag.bedrock_knowledgebase import (
|
||||
BedrockKBContent,
|
||||
BedrockKBResponse,
|
||||
|
|
@ -38,6 +39,14 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM):
|
|||
BaseVectorStoreConfig.__init__(self)
|
||||
BaseAWSLLM.__init__(self)
|
||||
|
||||
def get_error_class(
|
||||
self,
|
||||
error_message: str,
|
||||
status_code: int,
|
||||
headers: dict[str, object] | httpx.Headers, # mutable-ok: base passes response headers as a dict
|
||||
) -> BedrockError:
|
||||
return BedrockError(status_code=status_code, message=error_message, headers=headers)
|
||||
|
||||
def get_auth_credentials(self, litellm_params: dict) -> BaseVectorStoreAuthCredentials:
|
||||
return {}
|
||||
|
||||
|
|
|
|||
|
|
@ -13,6 +13,8 @@ Auth: Bearer token (litellm_params.api_key, BEDROCK_MANTLE_API_KEY, or the
|
|||
from collections.abc import AsyncIterator, Iterator
|
||||
from typing import Any, Final
|
||||
|
||||
import httpx
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
|
|
@ -24,6 +26,8 @@ from litellm.secret_managers.main import get_secret_str
|
|||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
from ...base_llm.chat.transformation import BaseLLMException
|
||||
from ...bedrock.common_utils import BedrockError
|
||||
from ...openai_like.chat.transformation import OpenAILikeChatConfig
|
||||
from ..common_utils import mantle_base_segment
|
||||
|
||||
|
|
@ -45,6 +49,11 @@ class BedrockMantleChatConfig(BedrockMantleAuthMixin, OpenAILikeChatConfig):
|
|||
def get_config(cls):
|
||||
return super().get_config()
|
||||
|
||||
def get_error_class(
|
||||
self, error_message: str, status_code: int, headers: dict[str, object] | httpx.Headers
|
||||
) -> BaseLLMException:
|
||||
return BedrockError(status_code=status_code, message=error_message, headers=headers)
|
||||
|
||||
def _get_openai_compatible_provider_info(
|
||||
self,
|
||||
api_base: str | None,
|
||||
|
|
|
|||
|
|
@ -19,11 +19,14 @@ import json
|
|||
from collections.abc import Mapping
|
||||
from typing import Any, Final
|
||||
|
||||
import httpx
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
from litellm.llms.bedrock_mantle.common_utils import (
|
||||
MANTLE_HOST_RE,
|
||||
BedrockMantleAuthMixin,
|
||||
|
|
@ -98,6 +101,11 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
|
|||
def custom_llm_provider(self) -> LlmProviders:
|
||||
return LlmProviders.BEDROCK_MANTLE
|
||||
|
||||
def get_error_class(
|
||||
self, error_message: str, status_code: int, headers: dict[str, object] | httpx.Headers
|
||||
) -> BaseLLMException:
|
||||
return BedrockError(status_code=status_code, message=error_message, headers=headers)
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: str | None,
|
||||
|
|
|
|||
|
|
@ -3508,9 +3508,13 @@ class ProxyBaseLLMRequestProcessing:
|
|||
error_body: Final = await http_status_error.response.aread()
|
||||
error_text: Final = error_body.decode("utf-8")
|
||||
|
||||
error_headers: Final = { # mutable-ok: HTTPException takes a plain header dict
|
||||
k: v if isinstance(v, str) else str(v) for k, v in safe_headers.items()
|
||||
}
|
||||
raise HTTPException(
|
||||
status_code=http_status_error.response.status_code,
|
||||
detail={"error": error_text},
|
||||
headers=error_headers,
|
||||
)
|
||||
error_msg: Final = f"{e}"
|
||||
# Check for AttributeError in the exception chain.
|
||||
|
|
|
|||
|
|
@ -947,7 +947,14 @@ async def handle_bedrock_count_tokens(
|
|||
except BedrockError as e:
|
||||
# Convert BedrockError to HTTPException for FastAPI
|
||||
verbose_proxy_logger.error("BedrockError in handle_bedrock_count_tokens: %s", e)
|
||||
raise HTTPException(status_code=e.status_code, detail={"error": e.message})
|
||||
from litellm.litellm_core_utils.llm_response_utils.get_headers import get_response_headers
|
||||
|
||||
provider_headers: Final = getattr(getattr(e, "response", None), "headers", None)
|
||||
raise HTTPException(
|
||||
status_code=e.status_code,
|
||||
detail={"error": e.message},
|
||||
headers=get_response_headers(provider_headers) if provider_headers else None,
|
||||
)
|
||||
except HTTPException:
|
||||
# Re-raise HTTP exceptions as-is
|
||||
raise
|
||||
|
|
|
|||
|
|
@ -9,9 +9,11 @@ import litellm
|
|||
from litellm.litellm_core_utils.exception_mapping_utils import (
|
||||
ExceptionCheckers,
|
||||
_get_body_error_code,
|
||||
_get_response_headers,
|
||||
exception_type,
|
||||
extract_and_raise_litellm_exception,
|
||||
)
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
from litellm.llms.openai.common_utils import OpenAIError
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
|
|
@ -1254,3 +1256,156 @@ def test_handle_error_marks_only_a_status_code_it_never_received():
|
|||
raise handler._handle_error(e=upstream, provider_config=None)
|
||||
assert received.value.status_code == 500
|
||||
assert received.value.status_code_is_synthesized is False
|
||||
|
||||
|
||||
def test_bedrock_500_preserves_provider_response_headers():
|
||||
"""A Bedrock 5xx must keep x-amzn-RequestId so AWS support can trace it (LIT-5428)."""
|
||||
provider_response = httpx.Response(
|
||||
status_code=500,
|
||||
headers={"x-amzn-RequestId": "req-map-500"},
|
||||
text='{"message":"Amazon Bedrock is unable to process your request."}',
|
||||
request=httpx.Request("POST", "https://bedrock-runtime.us-east-1.amazonaws.com/"),
|
||||
)
|
||||
original_exception = BedrockError(
|
||||
status_code=500,
|
||||
message=provider_response.text,
|
||||
headers=provider_response.headers,
|
||||
response=provider_response,
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.ServiceUnavailableError) as exc_info:
|
||||
exception_type(
|
||||
model="anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
original_exception=original_exception,
|
||||
custom_llm_provider="bedrock",
|
||||
completion_kwargs={},
|
||||
extra_kwargs={},
|
||||
)
|
||||
|
||||
assert exc_info.value.response.headers["x-amzn-requestid"] == "req-map-500"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"custom_llm_provider, status_code, provider_message, expected_exception",
|
||||
[
|
||||
(
|
||||
"bedrock_mantle",
|
||||
400,
|
||||
(
|
||||
'{"error":{"code":"validation_error",'
|
||||
'"message":"prompt tokens (1055489) exceed model maximum (1050000) for openai.gpt-5.6-sol",'
|
||||
'"param":null,"type":"invalid_request_error"}}'
|
||||
),
|
||||
litellm.ContextWindowExceededError,
|
||||
),
|
||||
(
|
||||
"bedrock",
|
||||
400,
|
||||
'{"message":"Input is too long for requested model."}',
|
||||
litellm.ContextWindowExceededError,
|
||||
),
|
||||
(
|
||||
"bedrock",
|
||||
400,
|
||||
'{"message":"Could not process image"}',
|
||||
litellm.InternalServerError,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_bedrock_classified_errors_preserve_provider_response_headers(
|
||||
custom_llm_provider, status_code, provider_message, expected_exception
|
||||
):
|
||||
"""Branches that classify a Bedrock error by its text must keep x-amzn-RequestId (LIT-5428)."""
|
||||
provider_response = httpx.Response(
|
||||
status_code=status_code,
|
||||
headers={"x-amzn-RequestId": "req-classified"},
|
||||
text=provider_message,
|
||||
request=httpx.Request("POST", "https://bedrock-runtime.us-east-1.amazonaws.com/"),
|
||||
)
|
||||
original_exception = BedrockError(
|
||||
status_code=status_code,
|
||||
message=provider_message,
|
||||
headers=provider_response.headers,
|
||||
response=provider_response,
|
||||
)
|
||||
|
||||
with pytest.raises(expected_exception) as exc_info:
|
||||
exception_type(
|
||||
model="anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
original_exception=original_exception,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
completion_kwargs={},
|
||||
extra_kwargs={},
|
||||
)
|
||||
|
||||
assert exc_info.value.response.headers["x-amzn-requestid"] == "req-classified"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"status_code, provider_message",
|
||||
[
|
||||
(504, '{"message":"Gateway timeout"}'),
|
||||
(408, '{"message":"Bedrock did not answer in time"}'),
|
||||
(408, '{"message":"Connect timeout on endpoint URL"}'),
|
||||
],
|
||||
)
|
||||
def test_bedrock_timeout_mapping_preserves_provider_headers(status_code, provider_message):
|
||||
"""A mapped bedrock timeout keeps the upstream response, like every other mapped bedrock error.
|
||||
|
||||
The proxy prefixes those headers on the way out, while retry and cooldown
|
||||
logic still reads the raw retry-after off the response.
|
||||
"""
|
||||
provider_response = httpx.Response(
|
||||
status_code=status_code,
|
||||
headers={"x-amzn-RequestId": "req-timeout", "set-cookie": "session=attacker"},
|
||||
text=provider_message,
|
||||
request=httpx.Request("POST", "https://bedrock-runtime.us-east-1.amazonaws.com/"),
|
||||
)
|
||||
original_exception = BedrockError(
|
||||
status_code=status_code,
|
||||
message=provider_message,
|
||||
headers=provider_response.headers,
|
||||
response=provider_response,
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.Timeout) as exc_info:
|
||||
exception_type(
|
||||
model="anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
original_exception=original_exception,
|
||||
custom_llm_provider="bedrock",
|
||||
completion_kwargs={},
|
||||
extra_kwargs={},
|
||||
)
|
||||
|
||||
assert exc_info.value.response.headers["x-amzn-requestid"] == "req-timeout"
|
||||
assert exc_info.value.headers is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status_code", [504, 408])
|
||||
def test_bedrock_timeout_mapping_keeps_retry_after_readable(status_code):
|
||||
"""Cooldown and retry timing read retry-after through _get_response_headers."""
|
||||
provider_response = httpx.Response(
|
||||
status_code=status_code,
|
||||
headers={"x-amzn-RequestId": "req-retry-after", "retry-after": "7"},
|
||||
text='{"message":"Bedrock did not answer in time"}',
|
||||
request=httpx.Request("POST", "https://bedrock-runtime.us-east-1.amazonaws.com/"),
|
||||
)
|
||||
original_exception = BedrockError(
|
||||
status_code=status_code,
|
||||
message='{"message":"Bedrock did not answer in time"}',
|
||||
headers=provider_response.headers,
|
||||
response=provider_response,
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.Timeout) as exc_info:
|
||||
exception_type(
|
||||
model="anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
original_exception=original_exception,
|
||||
custom_llm_provider="bedrock",
|
||||
completion_kwargs={},
|
||||
extra_kwargs={},
|
||||
)
|
||||
|
||||
exception_headers = _get_response_headers(original_exception=exc_info.value)
|
||||
assert exception_headers is not None
|
||||
assert litellm.utils._get_retry_after_from_exception_header(response_headers=exception_headers) == 7
|
||||
|
|
|
|||
|
|
@ -177,3 +177,16 @@ def test_guardrail_config_flows_to_headers_not_request_body(model):
|
|||
assert headers["X-Amzn-Bedrock-GuardrailIdentifier"] == "ff6ujrregl1q"
|
||||
assert headers["X-Amzn-Bedrock-GuardrailVersion"] == "DRAFT"
|
||||
assert headers["X-Amzn-Bedrock-Trace"] == "DISABLED"
|
||||
|
||||
|
||||
def test_get_error_class_preserves_provider_headers():
|
||||
"""The invoke handler path hands real provider headers to get_error_class (LIT-5428)."""
|
||||
error = AmazonInvokeConfig().get_error_class(
|
||||
error_message="Amazon Bedrock is unable to process your request.",
|
||||
status_code=500,
|
||||
headers={"x-amzn-RequestId": "req-invoke-500"},
|
||||
)
|
||||
|
||||
assert isinstance(error, BedrockError)
|
||||
assert error.headers == {"x-amzn-RequestId": "req-invoke-500"}
|
||||
assert error.response.headers["x-amzn-requestid"] == "req-invoke-500"
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import asyncio
|
|||
import json
|
||||
import os
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
|
|
@ -6039,6 +6040,8 @@ def test_transform_response_does_not_leak_body_on_parse_failure():
|
|||
leaky_body = {"output": {"message": {"content": [{"text": "secret content"}]}}}
|
||||
|
||||
class MockResponse:
|
||||
headers = httpx.Headers({"x-amzn-RequestId": "req-parse-failure"})
|
||||
|
||||
def json(self):
|
||||
return leaky_body
|
||||
|
||||
|
|
@ -6067,6 +6070,7 @@ def test_transform_response_does_not_leak_body_on_parse_failure():
|
|||
msg = str(exc_info.value)
|
||||
assert "secret content" not in msg
|
||||
assert "Error converting to valid response block" in msg
|
||||
assert exc_info.value.response.headers["x-amzn-requestid"] == "req-parse-failure"
|
||||
|
||||
|
||||
def test_converse_drops_sampling_params_for_models_that_removed_them():
|
||||
|
|
|
|||
|
|
@ -496,3 +496,171 @@ async def test_async_invoke_streaming_forwards_bedrock_response_headers():
|
|||
|
||||
assert stream._hidden_params["additional_headers"]["llm_provider-x-amzn-requestid"] == "req-987"
|
||||
|
||||
|
||||
def _bedrock_stream_error_response(status_code: int, request_id: str) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
status_code=status_code,
|
||||
headers={
|
||||
"x-amzn-RequestId": request_id,
|
||||
"x-amzn-ErrorType": "InternalServerException",
|
||||
},
|
||||
text='{"message":"Amazon Bedrock is unable to process your request."}',
|
||||
request=httpx.Request("POST", "https://bedrock-runtime.us-east-1.amazonaws.com/"),
|
||||
)
|
||||
|
||||
|
||||
def test_invoke_streaming_error_forwards_bedrock_response_headers():
|
||||
error_response = _bedrock_stream_error_response(500, "req-stream-err-1")
|
||||
client = HTTPHandler()
|
||||
client.post = MagicMock(
|
||||
side_effect=httpx.HTTPStatusError(
|
||||
"server error",
|
||||
request=error_response.request,
|
||||
response=error_response,
|
||||
)
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.ServiceUnavailableError) as exc_info:
|
||||
litellm.completion(
|
||||
model="bedrock/invoke/anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
client=client,
|
||||
aws_access_key_id="fake",
|
||||
aws_secret_access_key="fake",
|
||||
aws_region_name="us-east-1",
|
||||
)
|
||||
|
||||
assert exc_info.value.response.headers["x-amzn-requestid"] == "req-stream-err-1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_invoke_streaming_error_forwards_bedrock_response_headers():
|
||||
error_response = _bedrock_stream_error_response(500, "req-stream-err-2")
|
||||
client = AsyncHTTPHandler()
|
||||
client.post = AsyncMock(
|
||||
side_effect=httpx.HTTPStatusError(
|
||||
"server error",
|
||||
request=error_response.request,
|
||||
response=error_response,
|
||||
)
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.ServiceUnavailableError) as exc_info:
|
||||
await litellm.acompletion(
|
||||
model="bedrock/invoke/anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
client=client,
|
||||
aws_access_key_id="fake",
|
||||
aws_secret_access_key="fake",
|
||||
aws_region_name="us-east-1",
|
||||
)
|
||||
|
||||
assert exc_info.value.response.headers["x-amzn-requestid"] == "req-stream-err-2"
|
||||
|
||||
|
||||
def _unread_bedrock_stream_error_response(status_code: int, request_id: str) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
status_code=status_code,
|
||||
headers={
|
||||
"x-amzn-RequestId": request_id,
|
||||
"x-amzn-ErrorType": "InternalServerException",
|
||||
},
|
||||
stream=httpx.ByteStream(b'{"message":"Amazon Bedrock is unable to process your request."}'),
|
||||
request=httpx.Request("POST", "https://bedrock-runtime.us-east-1.amazonaws.com/"),
|
||||
)
|
||||
|
||||
|
||||
def test_invoke_streaming_error_forwards_headers_when_body_was_never_read():
|
||||
"""A retried streamed request raises HTTPStatusError over a body nobody read, so
|
||||
reading it for the error message throws and loses the request id (LIT-5428)."""
|
||||
error_response = _unread_bedrock_stream_error_response(500, "req-unread-sync")
|
||||
client = HTTPHandler()
|
||||
client.post = MagicMock(
|
||||
side_effect=httpx.HTTPStatusError(
|
||||
"server error",
|
||||
request=error_response.request,
|
||||
response=error_response,
|
||||
)
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.ServiceUnavailableError) as exc_info:
|
||||
litellm.completion(
|
||||
model="bedrock/invoke/anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
client=client,
|
||||
aws_access_key_id="fake",
|
||||
aws_secret_access_key="fake",
|
||||
aws_region_name="us-east-1",
|
||||
)
|
||||
|
||||
assert exc_info.value.response.headers["x-amzn-requestid"] == "req-unread-sync"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_invoke_streaming_error_forwards_headers_when_body_was_never_read():
|
||||
error_response = _unread_bedrock_stream_error_response(500, "req-unread-async")
|
||||
client = AsyncHTTPHandler()
|
||||
client.post = AsyncMock(
|
||||
side_effect=httpx.HTTPStatusError(
|
||||
"server error",
|
||||
request=error_response.request,
|
||||
response=error_response,
|
||||
)
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.ServiceUnavailableError) as exc_info:
|
||||
await litellm.acompletion(
|
||||
model="bedrock/invoke/anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
client=client,
|
||||
aws_access_key_id="fake",
|
||||
aws_secret_access_key="fake",
|
||||
aws_region_name="us-east-1",
|
||||
)
|
||||
|
||||
assert exc_info.value.response.headers["x-amzn-requestid"] == "req-unread-async"
|
||||
|
||||
|
||||
def test_invoke_streaming_non_200_forwards_bedrock_response_headers():
|
||||
"""A caller-supplied client that returns a failure instead of raising still reaches the
|
||||
provider's headers, and reading the streamed body for the message must not throw (LIT-5428)."""
|
||||
error_response = _unread_bedrock_stream_error_response(500, "req-non200-sync")
|
||||
client = HTTPHandler()
|
||||
client.post = MagicMock(return_value=error_response)
|
||||
|
||||
with pytest.raises(litellm.ServiceUnavailableError) as exc_info:
|
||||
litellm.completion(
|
||||
model="bedrock/invoke/anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
client=client,
|
||||
aws_access_key_id="fake",
|
||||
aws_secret_access_key="fake",
|
||||
aws_region_name="us-east-1",
|
||||
)
|
||||
|
||||
assert exc_info.value.response.headers["x-amzn-requestid"] == "req-non200-sync"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_invoke_streaming_non_200_forwards_bedrock_response_headers():
|
||||
error_response = _unread_bedrock_stream_error_response(500, "req-non200-async")
|
||||
client = AsyncHTTPHandler()
|
||||
client.post = AsyncMock(return_value=error_response)
|
||||
|
||||
with pytest.raises(litellm.ServiceUnavailableError) as exc_info:
|
||||
await litellm.acompletion(
|
||||
model="bedrock/invoke/anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
client=client,
|
||||
aws_access_key_id="fake",
|
||||
aws_secret_access_key="fake",
|
||||
aws_region_name="us-east-1",
|
||||
)
|
||||
|
||||
assert exc_info.value.response.headers["x-amzn-requestid"] == "req-non200-async"
|
||||
|
|
|
|||
|
|
@ -614,3 +614,260 @@ def test_sign_aws_request_assumes_role_with_external_id(monkeypatch):
|
|||
authorization = {key.lower(): value for key, value in signed_headers.items()}["authorization"]
|
||||
assert "ASIABATCHSIGNROLE" in authorization
|
||||
assert signed_data == b'{"jobName": "litellm-batch-job"}'
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Provider error headers (LIT-5428) #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _bedrock_chat_error_configs():
|
||||
from litellm.llms.bedrock.chat.agentcore.transformation import AmazonAgentCoreConfig
|
||||
from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
|
||||
from litellm.llms.bedrock.chat.invoke_agent.transformation import AmazonInvokeAgentConfig
|
||||
from litellm.llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation import (
|
||||
AmazonMoonshotConfig,
|
||||
)
|
||||
from litellm.llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import (
|
||||
AmazonBedrockOpenAIConfig,
|
||||
)
|
||||
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
|
||||
AmazonInvokeConfig,
|
||||
)
|
||||
|
||||
return [
|
||||
AmazonInvokeConfig,
|
||||
AmazonConverseConfig,
|
||||
AmazonMoonshotConfig,
|
||||
AmazonBedrockOpenAIConfig,
|
||||
AmazonAgentCoreConfig,
|
||||
AmazonInvokeAgentConfig,
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("config", _bedrock_chat_error_configs())
|
||||
def test_bedrock_chat_get_error_class_keeps_provider_headers(config):
|
||||
"""Every Bedrock chat route must carry x-amzn-RequestId out to the caller (LIT-5428).
|
||||
|
||||
A config that drops the headers it is handed shadows the fix for its own models.
|
||||
"""
|
||||
error = config().get_error_class(
|
||||
error_message="Amazon Bedrock is unable to process your request.",
|
||||
status_code=500,
|
||||
headers={"x-amzn-RequestId": "req-chat-500"},
|
||||
)
|
||||
|
||||
assert error.response.headers["x-amzn-requestid"] == "req-chat-500"
|
||||
|
||||
|
||||
def test_error_response_text_reads_a_read_response():
|
||||
import httpx
|
||||
|
||||
from litellm.llms.bedrock.common_utils import error_response_text
|
||||
|
||||
response = httpx.Response(status_code=500, text="Amazon Bedrock is unable to process your request.")
|
||||
|
||||
assert error_response_text(response) == "Amazon Bedrock is unable to process your request."
|
||||
|
||||
|
||||
def test_error_response_text_falls_back_when_a_streamed_response_was_never_read():
|
||||
"""A retried streamed request raises HTTPStatusError over an unread body; reading it
|
||||
throws ResponseNotRead and would lose the status and headers this fix preserves."""
|
||||
import httpx
|
||||
|
||||
from litellm.llms.bedrock.common_utils import error_response_text
|
||||
|
||||
request = httpx.Request(method="POST", url="https://bedrock-runtime.amazonaws.com")
|
||||
response = httpx.Response(
|
||||
status_code=500,
|
||||
headers={"x-amzn-RequestId": "req-unread-500"},
|
||||
stream=httpx.ByteStream(b"never read"),
|
||||
request=request,
|
||||
)
|
||||
|
||||
with pytest.raises(httpx.ResponseNotRead):
|
||||
_ = response.text
|
||||
|
||||
assert error_response_text(response) == "Internal Server Error"
|
||||
|
||||
|
||||
def test_bedrock_error_skips_header_values_httpx_cannot_carry():
|
||||
"""The shared HTTP handler copies an arbitrary exception's header values in verbatim,
|
||||
so a non-str value must not take down the whole error (LIT-5428)."""
|
||||
import httpx
|
||||
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
|
||||
error = BedrockError(
|
||||
status_code=500,
|
||||
message="boom",
|
||||
headers={"x-amzn-RequestId": "req-mixed-500", "x-retry-count": 3, "x-nothing": None},
|
||||
)
|
||||
|
||||
assert error.response.headers["x-amzn-requestid"] == "req-mixed-500"
|
||||
assert "x-retry-count" not in error.response.headers
|
||||
assert isinstance(error.response, httpx.Response)
|
||||
|
||||
|
||||
def test_bedrock_error_keeps_duplicate_httpx_header_values():
|
||||
import httpx
|
||||
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
|
||||
error = BedrockError(
|
||||
status_code=500,
|
||||
message="boom",
|
||||
headers=httpx.Headers([("x-amzn-RequestId", "req-dup-500"), ("set-cookie", "a=1"), ("set-cookie", "b=2")]),
|
||||
)
|
||||
|
||||
assert error.response.headers.get_list("set-cookie") == ["a=1", "b=2"]
|
||||
|
||||
|
||||
def _bedrock_httpx_status_error_sites():
|
||||
"""Every `except httpx.HTTPStatusError as err` that raises a BedrockError, across bedrock."""
|
||||
import ast
|
||||
import pathlib
|
||||
|
||||
sites = []
|
||||
for path in sorted(pathlib.Path("litellm/llms/bedrock").rglob("*.py")):
|
||||
tree = ast.parse(path.read_text())
|
||||
for handler in (n for n in ast.walk(tree) if isinstance(n, ast.ExceptHandler)):
|
||||
caught = ast.unparse(handler.type) if handler.type is not None else ""
|
||||
if "HTTPStatusError" not in caught or handler.name is None:
|
||||
continue
|
||||
for call in (
|
||||
n
|
||||
for n in ast.walk(handler)
|
||||
if isinstance(n, ast.Call) and isinstance(n.func, ast.Name) and n.func.id == "BedrockError"
|
||||
):
|
||||
sites.append((str(path), call.lineno, handler.name, {k.arg for k in call.keywords}))
|
||||
return sites
|
||||
|
||||
|
||||
def test_every_bedrock_httpx_status_error_site_keeps_provider_headers():
|
||||
"""A raise site holding the provider's failed response must hand its headers on (LIT-5428).
|
||||
|
||||
These sites are the only place x-amzn-RequestId still exists; a site that drops it
|
||||
silently shadows the fix for that whole surface.
|
||||
"""
|
||||
sites = _bedrock_httpx_status_error_sites()
|
||||
|
||||
assert len(sites) >= 12
|
||||
dropped = [f"{path}:{lineno}" for path, lineno, _, kwargs in sites if "headers" not in kwargs]
|
||||
assert dropped == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_async", [False, True])
|
||||
@pytest.mark.asyncio
|
||||
async def test_bedrock_embedding_call_keeps_provider_headers(is_async):
|
||||
"""The embeddings surface raises from the same shape as chat and lost the same header."""
|
||||
import httpx
|
||||
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
from litellm.llms.bedrock.embed.embedding import BedrockEmbedding
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
|
||||
failure = httpx.Response(
|
||||
status_code=500,
|
||||
headers={"x-amzn-RequestId": "req-embed-500"},
|
||||
text='{"message":"Amazon Bedrock is unable to process your request."}',
|
||||
request=httpx.Request("POST", "https://bedrock-runtime.us-east-1.amazonaws.com/"),
|
||||
)
|
||||
|
||||
class _SyncUpstream(HTTPHandler):
|
||||
def post(self, *args, **kwargs):
|
||||
return failure
|
||||
|
||||
class _AsyncUpstream(AsyncHTTPHandler):
|
||||
async def post(self, *args, **kwargs):
|
||||
return failure
|
||||
|
||||
async def _drive():
|
||||
embedding = BedrockEmbedding()
|
||||
kwargs = dict(
|
||||
timeout=None,
|
||||
api_base="https://bedrock-runtime.us-east-1.amazonaws.com/",
|
||||
headers={},
|
||||
data={},
|
||||
)
|
||||
if is_async:
|
||||
return await embedding._make_async_call(client=_AsyncUpstream(), **kwargs)
|
||||
return embedding._make_sync_call(client=_SyncUpstream(), **kwargs)
|
||||
|
||||
with pytest.raises(BedrockError) as exc_info:
|
||||
await _drive()
|
||||
|
||||
assert exc_info.value.response.headers["x-amzn-requestid"] == "req-embed-500"
|
||||
|
||||
|
||||
def _bedrock_mantle_error_configs():
|
||||
from litellm.llms.bedrock_mantle.chat.transformation import BedrockMantleChatConfig
|
||||
from litellm.llms.bedrock_mantle.responses.transformation import BedrockMantleResponsesAPIConfig
|
||||
|
||||
return [BedrockMantleChatConfig, BedrockMantleResponsesAPIConfig]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("config", _bedrock_mantle_error_configs())
|
||||
def test_bedrock_mantle_get_error_class_keeps_provider_headers(config):
|
||||
"""bedrock_mantle rides the OpenAI-compatible surfaces, whose errors drop the headers.
|
||||
|
||||
A chat request for a responses-API model is bridged onto the responses config, so
|
||||
fixing only the chat one leaves the model the customer actually calls uncovered.
|
||||
"""
|
||||
error = config().get_error_class(
|
||||
error_message="prompt tokens exceed model maximum",
|
||||
status_code=400,
|
||||
headers={"x-amzn-RequestId": "req-mantle-400"},
|
||||
)
|
||||
|
||||
assert error.response.headers["x-amzn-requestid"] == "req-mantle-400"
|
||||
|
||||
|
||||
def _bedrock_configs_with_get_error_class():
|
||||
import importlib
|
||||
import inspect
|
||||
import pathlib
|
||||
|
||||
import litellm
|
||||
|
||||
llms_root = pathlib.Path(inspect.getfile(litellm)).parent / "llms"
|
||||
configs = []
|
||||
for package in ("bedrock", "bedrock_mantle"):
|
||||
for path in sorted((llms_root / package).rglob("*.py")):
|
||||
module_name = "litellm.llms." + ".".join(path.relative_to(llms_root).with_suffix("").parts)
|
||||
module = importlib.import_module(module_name)
|
||||
for name, obj in vars(module).items():
|
||||
if not inspect.isclass(obj) or obj.__module__ != module_name:
|
||||
continue
|
||||
if getattr(obj, "get_error_class", None) is None:
|
||||
continue
|
||||
configs.append(pytest.param(obj, id=f"{module_name}.{name}"))
|
||||
return configs
|
||||
|
||||
|
||||
@pytest.mark.parametrize("config", _bedrock_configs_with_get_error_class())
|
||||
def test_every_bedrock_config_get_error_class_keeps_provider_headers(config):
|
||||
"""Every bedrock surface must classify errors through BedrockError, not a header-dropping base.
|
||||
|
||||
A config that inherits get_error_class from a provider-agnostic base builds a blank
|
||||
response, so the request id is gone before the proxy ever reads it.
|
||||
"""
|
||||
try:
|
||||
instance = config()
|
||||
except Exception:
|
||||
instance = config.__new__(config)
|
||||
|
||||
try:
|
||||
error = instance.get_error_class(
|
||||
error_message="boom",
|
||||
status_code=500,
|
||||
headers={"x-amzn-RequestId": "req-audit-500"},
|
||||
)
|
||||
except Exception as raised: # some bases raise the exception instead of returning it
|
||||
error = raised
|
||||
|
||||
assert error.response.headers["x-amzn-requestid"] == "req-audit-500"
|
||||
|
||||
|
||||
def test_bedrock_get_error_class_audit_covers_every_surface():
|
||||
assert len(_bedrock_configs_with_get_error_class()) >= 30
|
||||
|
|
|
|||
|
|
@ -308,3 +308,64 @@ def test_completion_plumbs_stream_chunk_size_through_converse():
|
|||
stream_chunk_size=2048,
|
||||
)
|
||||
iter_bytes_spy.assert_called_once_with(chunk_size=2048)
|
||||
|
||||
|
||||
def _bedrock_error_response(status_code: int, request_id: str) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
status_code=status_code,
|
||||
headers={
|
||||
"x-amzn-RequestId": request_id,
|
||||
"x-amzn-ErrorType": "InternalServerException",
|
||||
},
|
||||
text=json.dumps({"message": "Amazon Bedrock is unable to process your request."}),
|
||||
request=httpx.Request("POST", "https://bedrock-runtime.us-east-1.amazonaws.com/"),
|
||||
)
|
||||
|
||||
|
||||
def test_converse_completion_error_forwards_bedrock_response_headers():
|
||||
error_response = _bedrock_error_response(500, "req-err-123")
|
||||
client = HTTPHandler()
|
||||
client.post = MagicMock(
|
||||
side_effect=httpx.HTTPStatusError(
|
||||
"server error",
|
||||
request=error_response.request,
|
||||
response=error_response,
|
||||
)
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.ServiceUnavailableError) as exc_info:
|
||||
litellm.completion(
|
||||
model="bedrock/converse/anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
client=client,
|
||||
aws_access_key_id="fake",
|
||||
aws_secret_access_key="fake",
|
||||
aws_region_name="us-east-1",
|
||||
)
|
||||
|
||||
assert exc_info.value.response.headers["x-amzn-requestid"] == "req-err-123"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_converse_completion_error_forwards_bedrock_response_headers():
|
||||
error_response = _bedrock_error_response(500, "req-err-456")
|
||||
client = AsyncHTTPHandler()
|
||||
client.post = AsyncMock(
|
||||
side_effect=httpx.HTTPStatusError(
|
||||
"server error",
|
||||
request=error_response.request,
|
||||
response=error_response,
|
||||
)
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.ServiceUnavailableError) as exc_info:
|
||||
await litellm.acompletion(
|
||||
model="bedrock/converse/anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
client=client,
|
||||
aws_access_key_id="fake",
|
||||
aws_secret_access_key="fake",
|
||||
aws_region_name="us-east-1",
|
||||
)
|
||||
|
||||
assert exc_info.value.response.headers["x-amzn-requestid"] == "req-err-456"
|
||||
|
|
|
|||
|
|
@ -5293,3 +5293,37 @@ class TestRouterModelRelayUpstreamContract:
|
|||
assert result.status_code == 404
|
||||
assert json.loads(result.body) == upstream_body
|
||||
assert result.headers["x-ms-request-id"] == "req-1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bedrock_count_tokens_error_forwards_provider_headers():
|
||||
"""The count tokens route converts BedrockError into an HTTPException, and dropping the
|
||||
headers there loses x-amzn-RequestId after the handler went to the trouble of keeping it."""
|
||||
from fastapi import HTTPException
|
||||
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
|
||||
handle_bedrock_count_tokens,
|
||||
)
|
||||
|
||||
failure = BedrockError(
|
||||
status_code=500,
|
||||
message="Amazon Bedrock is unable to process your request.",
|
||||
headers={"x-amzn-RequestId": "req-count-tokens-500"},
|
||||
)
|
||||
|
||||
with patch( # test-quality-ok: the route's BedrockError branch is only reachable when the handler raises
|
||||
"litellm.llms.bedrock.count_tokens.handler.BedrockCountTokensHandler.handle_count_tokens_request",
|
||||
new=AsyncMock(side_effect=failure),
|
||||
):
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await handle_bedrock_count_tokens(
|
||||
endpoint="v1/messages/count_tokens",
|
||||
request=MagicMock(),
|
||||
fastapi_response=MagicMock(),
|
||||
user_api_key_dict=MagicMock(),
|
||||
request_body={"model": "anthropic.claude-haiku-4-5-20251001-v1:0"},
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 500
|
||||
assert exc_info.value.headers["llm_provider-x-amzn-requestid"] == "req-count-tokens-500"
|
||||
|
|
|
|||
|
|
@ -8259,3 +8259,38 @@ class TestPassthroughHeadersAcceptImmutableMappings:
|
|||
assert merged["content-type"] == "text/event-stream"
|
||||
# the excluded hop-by-hop header is still dropped
|
||||
assert "transfer-encoding" not in merged
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_llm_api_exception_forwards_provider_headers_on_http_status_error():
|
||||
"""The httpx.HTTPStatusError branch dropped the headers its sibling branches forward.
|
||||
|
||||
A Bedrock passthrough failure reaches this branch, so the request id was gone
|
||||
before the client saw the response.
|
||||
"""
|
||||
import httpx
|
||||
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
request = httpx.Request("POST", "https://bedrock-runtime.us-east-1.amazonaws.com/model/m/converse")
|
||||
response = httpx.Response(
|
||||
status_code=500,
|
||||
headers={"x-amzn-RequestId": "req-passthrough-500"},
|
||||
content=b'{"message": "Amazon Bedrock is unable to process your request."}',
|
||||
request=request,
|
||||
)
|
||||
|
||||
processor = ProxyBaseLLMRequestProcessing(data={})
|
||||
proxy_logging_obj = MagicMock()
|
||||
proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None)
|
||||
proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={})
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await processor._handle_llm_api_exception(
|
||||
e=httpx.HTTPStatusError("boom", request=request, response=response),
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
|
||||
assert exc_info.value.headers is not None
|
||||
assert exc_info.value.headers["llm_provider-x-amzn-requestid"] == "req-passthrough-500"
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from litellm.exceptions import (
|
|||
ImageFetchError,
|
||||
MidStreamFallbackError,
|
||||
RateLimitError,
|
||||
ServiceUnavailableError,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -312,3 +313,85 @@ class TestProxyHeaderExtraction:
|
|||
# Verify headers are extracted and prefixed correctly
|
||||
assert headers.get("llm_provider-x-request-id") == "req-abc123"
|
||||
assert headers.get("llm_provider-x-ms-region") == "eastus"
|
||||
|
||||
|
||||
class TestBedrockErrorHeaders:
|
||||
"""A BedrockError built with headers but no response still exposes them (LIT-5428)."""
|
||||
|
||||
def test_synthesized_response_carries_headers(self):
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
|
||||
error = BedrockError(
|
||||
status_code=500,
|
||||
message="Amazon Bedrock is unable to process your request.",
|
||||
headers={"x-amzn-RequestId": "req-base-500"},
|
||||
)
|
||||
|
||||
assert error.response.headers["x-amzn-requestid"] == "req-base-500"
|
||||
assert str(error.request.url) == str(BedrockError(status_code=500, message="boom").request.url)
|
||||
assert str(error.response.request.url) == str(error.request.url)
|
||||
|
||||
def test_synthesized_response_without_headers_stays_empty(self):
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
|
||||
error = BedrockError(status_code=500, message="boom")
|
||||
|
||||
assert dict(error.response.headers) == {}
|
||||
|
||||
def test_explicit_response_is_kept(self):
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
|
||||
provider_response = httpx.Response(
|
||||
status_code=500,
|
||||
headers={"x-amzn-RequestId": "from-response"},
|
||||
request=httpx.Request("POST", "https://bedrock-runtime.us-east-1.amazonaws.com/"),
|
||||
)
|
||||
error = BedrockError(
|
||||
status_code=500,
|
||||
message="boom",
|
||||
headers={"x-amzn-RequestId": "from-headers"},
|
||||
response=provider_response,
|
||||
)
|
||||
|
||||
assert error.response is provider_response
|
||||
|
||||
def test_proxy_extraction_surfaces_bedrock_request_id(self):
|
||||
"""End-to-end shape the proxy error handler returns to the caller."""
|
||||
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
|
||||
from litellm.litellm_core_utils.llm_response_utils.get_headers import (
|
||||
get_response_headers,
|
||||
)
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
|
||||
provider_response = httpx.Response(
|
||||
status_code=500,
|
||||
headers={"x-amzn-RequestId": "req-proxy-500"},
|
||||
text='{"message":"Amazon Bedrock is unable to process your request."}',
|
||||
request=httpx.Request("POST", "https://bedrock-runtime.us-east-1.amazonaws.com/"),
|
||||
)
|
||||
|
||||
with pytest.raises(ServiceUnavailableError) as exc_info:
|
||||
exception_type(
|
||||
model="anthropic.claude-haiku-4-5-20251001-v1:0",
|
||||
original_exception=BedrockError(
|
||||
status_code=500,
|
||||
message=provider_response.text,
|
||||
headers=provider_response.headers,
|
||||
response=provider_response,
|
||||
),
|
||||
custom_llm_provider="bedrock",
|
||||
completion_kwargs={},
|
||||
extra_kwargs={},
|
||||
)
|
||||
|
||||
# Mirrors ProxyBaseLLMRequestProcessing._handle_llm_api_exception
|
||||
error = exc_info.value
|
||||
headers = getattr(error, "headers", None) or {}
|
||||
if not headers:
|
||||
_response = getattr(error, "response", None)
|
||||
if _response is not None:
|
||||
_response_headers = getattr(_response, "headers", None)
|
||||
if _response_headers:
|
||||
headers = get_response_headers(dict(_response_headers))
|
||||
|
||||
assert headers.get("llm_provider-x-amzn-requestid") == "req-proxy-500"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue