diff --git a/litellm/llms/sagemaker/chat/transformation.py b/litellm/llms/sagemaker/chat/transformation.py index af184c89060..76eafcd6bb6 100644 --- a/litellm/llms/sagemaker/chat/transformation.py +++ b/litellm/llms/sagemaker/chat/transformation.py @@ -28,7 +28,7 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import LlmProviders from ...openai.chat.gpt_transformation import OpenAIGPTConfig -from ..common_utils import INFERENCE_COMPONENT_HEADER, STR_KEYED_MAPPING, AWSEventStreamDecoder, SagemakerError +from ..common_utils import INFERENCE_COMPONENT_HEADER, OBJECT_KEYED_MAPPING, AWSEventStreamDecoder, SagemakerError if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -61,8 +61,8 @@ class SagemakerChatConfig(OpenAIGPTConfig, BaseAWSLLM): return headers other_headers: Final = { name: value - for name, value in STR_KEYED_MAPPING.validate_python(headers).items() - if name.lower() != INFERENCE_COMPONENT_HEADER.lower() + for name, value in OBJECT_KEYED_MAPPING.validate_python(headers).items() + if not isinstance(name, str) or name.lower() != INFERENCE_COMPONENT_HEADER.lower() } return {**other_headers, INFERENCE_COMPONENT_HEADER: inference_component_name} diff --git a/litellm/llms/sagemaker/common_utils.py b/litellm/llms/sagemaker/common_utils.py index f309f9b2af0..bb7719a19a1 100644 --- a/litellm/llms/sagemaker/common_utils.py +++ b/litellm/llms/sagemaker/common_utils.py @@ -17,7 +17,7 @@ from litellm.types.utils import GenericStreamingChunk as GChunk from litellm.types.utils import LlmProviders, StreamingChatCompletionChunk STR_KEYED_MAPPING: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) -_OBJECT_KEYED_MAPPING: Final = TypeAdapter(Mapping[object, object], config=ConfigDict(hide_input_in_errors=True)) +OBJECT_KEYED_MAPPING: Final = TypeAdapter(Mapping[object, object], config=ConfigDict(hide_input_in_errors=True)) @with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True)) @@ -191,7 +191,7 @@ def deployment_inference_component(litellm_params: Mapping[str, object]) -> str return next( ( value - for name, value in _OBJECT_KEYED_MAPPING.validate_python(headers).items() + for name, value in OBJECT_KEYED_MAPPING.validate_python(headers).items() if isinstance(name, str) and name.lower() == INFERENCE_COMPONENT_HEADER.lower() and isinstance(value, str) ), None, diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 9383e4b125b..a06860e7786 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -36,7 +36,6 @@ _V = TypeVar("_V") MID_STREAM_CONTINUATION_KWARG: Final = "_mid_stream_continuation" ROUTER_ONLY_CALL_KWARGS: Final = frozenset({"silent_model", "include_fallback_errors", MID_STREAM_CONTINUATION_KWARG}) _OBJECT_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapter(Mapping[object, object]) -_CLIENTSIDE_PARAMS_ADAPTER: Final = TypeAdapter(tuple[object, ...]) _DEPLOYMENT_ROWS_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter( tuple[Mapping[str, object], ...] ) @@ -525,10 +524,8 @@ _LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) def deployment_model_id_pin(deployment_litellm_params: Mapping[str, object]) -> object | None: - clientside_params: Final = _CLIENTSIDE_PARAMS_ADAPTER.validate_python( - deployment_litellm_params.get("configurable_clientside_auth_params") or () - ) - if "model_id" in clientside_params: + clientside_params: Final = deployment_litellm_params.get("configurable_clientside_auth_params") + if isinstance(clientside_params, list | tuple) and "model_id" in clientside_params: return None own_model_id: Final = deployment_litellm_params.get("model_id") if own_model_id is not None: diff --git a/tests/unit/llms/sagemaker/test_sagemaker_chat_transformation.py b/tests/unit/llms/sagemaker/test_sagemaker_chat_transformation.py index 7df26d770b6..356528d8909 100644 --- a/tests/unit/llms/sagemaker/test_sagemaker_chat_transformation.py +++ b/tests/unit/llms/sagemaker/test_sagemaker_chat_transformation.py @@ -351,6 +351,18 @@ def test_inference_component_travels_only_in_the_header(monkeypatch: pytest.Monk assert "model_id" not in client.request_body +def test_inference_component_header_keeps_header_keys_that_are_not_strings(): + headers = SagemakerChatConfig().validate_environment( + headers={7: "other", "x-amzn-sagemaker-inference-component": "client-component"}, + model="my-endpoint", + messages=[], + optional_params={"model_id": "my-inference-component"}, + litellm_params={}, + ) + + assert headers == {7: "other", _INFERENCE_COMPONENT_HEADER: "my-inference-component"} + + def test_body_model_stays_the_endpoint_name_when_hf_model_name_is_unset(monkeypatch): """Without `hf_model_name` the body must keep the model it has today.""" client = _invoke_sagemaker_chat(monkeypatch) diff --git a/tests/unit/proxy/test_health_check_max_tokens.py b/tests/unit/proxy/test_health_check_max_tokens.py index 73aa5456d9d..23242e9009c 100644 --- a/tests/unit/proxy/test_health_check_max_tokens.py +++ b/tests/unit/proxy/test_health_check_max_tokens.py @@ -1620,3 +1620,26 @@ async def test_run_model_health_check_probes_a_legacy_sagemaker_embedding_at_the assert "error" not in result, result assert invoke_endpoint.call_count == 1 assert invoke_endpoint.call_args.kwargs.get("InferenceComponentName") == probed_component + + +@pytest.mark.asyncio +async def test_run_model_health_check_reports_why_a_string_configurable_clientside_auth_params_fails() -> None: + invoke_endpoint: Final = MagicMock() + deployment: Final = { + "litellm_params": { + "model": "sagemaker/openai/embed-endpoint", + "model_id": "own-component", + "aws_region_name": "us-east-1", + "aws_access_key_id": "test-key", + "aws_secret_access_key": "test-secret", + }, + "model_info": {"mode": "embedding", "health_check_params": {"configurable_clientside_auth_params": "model_id"}}, + } + + client: Final = MagicMock(invoke_endpoint=invoke_endpoint) + + with patch("boto3.Session", return_value=MagicMock(client=MagicMock(return_value=client))): + result: Final = await hc_module._run_model_health_check(deployment) + + assert "configurable_clientside_auth_params" in result["error"] + invoke_endpoint.assert_not_called()