fix(sagemaker): keep header keys that aren't strings and report a string opt-in in health checks

The chat transformation now keeps extra_headers keys that aren't strings, so such a deployment fails the way it did before instead of answering 500, and the model_id pin checks configurable_clientside_auth_params with isinstance, so a string there reaches the health probe's own error capture again.
This commit is contained in:
mateo-berri 2026-10-29 19:59:40 -07:00
parent 22b0c808d5
commit 2be68ea2ed
5 changed files with 42 additions and 10 deletions

View file

@ -28,7 +28,7 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import LlmProviders
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
from ..common_utils import INFERENCE_COMPONENT_HEADER, STR_KEYED_MAPPING, AWSEventStreamDecoder, SagemakerError
from ..common_utils import INFERENCE_COMPONENT_HEADER, OBJECT_KEYED_MAPPING, AWSEventStreamDecoder, SagemakerError
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@ -61,8 +61,8 @@ class SagemakerChatConfig(OpenAIGPTConfig, BaseAWSLLM):
return headers
other_headers: Final = {
name: value
for name, value in STR_KEYED_MAPPING.validate_python(headers).items()
if name.lower() != INFERENCE_COMPONENT_HEADER.lower()
for name, value in OBJECT_KEYED_MAPPING.validate_python(headers).items()
if not isinstance(name, str) or name.lower() != INFERENCE_COMPONENT_HEADER.lower()
}
return {**other_headers, INFERENCE_COMPONENT_HEADER: inference_component_name}

View file

@ -17,7 +17,7 @@ from litellm.types.utils import GenericStreamingChunk as GChunk
from litellm.types.utils import LlmProviders, StreamingChatCompletionChunk
STR_KEYED_MAPPING: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
_OBJECT_KEYED_MAPPING: Final = TypeAdapter(Mapping[object, object], config=ConfigDict(hide_input_in_errors=True))
OBJECT_KEYED_MAPPING: Final = TypeAdapter(Mapping[object, object], config=ConfigDict(hide_input_in_errors=True))
@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
@ -191,7 +191,7 @@ def deployment_inference_component(litellm_params: Mapping[str, object]) -> str
return next(
(
value
for name, value in _OBJECT_KEYED_MAPPING.validate_python(headers).items()
for name, value in OBJECT_KEYED_MAPPING.validate_python(headers).items()
if isinstance(name, str) and name.lower() == INFERENCE_COMPONENT_HEADER.lower() and isinstance(value, str)
),
None,

View file

@ -36,7 +36,6 @@ _V = TypeVar("_V")
MID_STREAM_CONTINUATION_KWARG: Final = "_mid_stream_continuation"
ROUTER_ONLY_CALL_KWARGS: Final = frozenset({"silent_model", "include_fallback_errors", MID_STREAM_CONTINUATION_KWARG})
_OBJECT_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapter(Mapping[object, object])
_CLIENTSIDE_PARAMS_ADAPTER: Final = TypeAdapter(tuple[object, ...])
_DEPLOYMENT_ROWS_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter(
tuple[Mapping[str, object], ...]
)
@ -525,10 +524,8 @@ _LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
def deployment_model_id_pin(deployment_litellm_params: Mapping[str, object]) -> object | None:
clientside_params: Final = _CLIENTSIDE_PARAMS_ADAPTER.validate_python(
deployment_litellm_params.get("configurable_clientside_auth_params") or ()
)
if "model_id" in clientside_params:
clientside_params: Final = deployment_litellm_params.get("configurable_clientside_auth_params")
if isinstance(clientside_params, list | tuple) and "model_id" in clientside_params:
return None
own_model_id: Final = deployment_litellm_params.get("model_id")
if own_model_id is not None:

View file

@ -351,6 +351,18 @@ def test_inference_component_travels_only_in_the_header(monkeypatch: pytest.Monk
assert "model_id" not in client.request_body
def test_inference_component_header_keeps_header_keys_that_are_not_strings():
headers = SagemakerChatConfig().validate_environment(
headers={7: "other", "x-amzn-sagemaker-inference-component": "client-component"},
model="my-endpoint",
messages=[],
optional_params={"model_id": "my-inference-component"},
litellm_params={},
)
assert headers == {7: "other", _INFERENCE_COMPONENT_HEADER: "my-inference-component"}
def test_body_model_stays_the_endpoint_name_when_hf_model_name_is_unset(monkeypatch):
"""Without `hf_model_name` the body must keep the model it has today."""
client = _invoke_sagemaker_chat(monkeypatch)

View file

@ -1620,3 +1620,26 @@ async def test_run_model_health_check_probes_a_legacy_sagemaker_embedding_at_the
assert "error" not in result, result
assert invoke_endpoint.call_count == 1
assert invoke_endpoint.call_args.kwargs.get("InferenceComponentName") == probed_component
@pytest.mark.asyncio
async def test_run_model_health_check_reports_why_a_string_configurable_clientside_auth_params_fails() -> None:
invoke_endpoint: Final = MagicMock()
deployment: Final = {
"litellm_params": {
"model": "sagemaker/openai/embed-endpoint",
"model_id": "own-component",
"aws_region_name": "us-east-1",
"aws_access_key_id": "test-key",
"aws_secret_access_key": "test-secret",
},
"model_info": {"mode": "embedding", "health_check_params": {"configurable_clientside_auth_params": "model_id"}},
}
client: Final = MagicMock(invoke_endpoint=invoke_endpoint)
with patch("boto3.Session", return_value=MagicMock(client=MagicMock(return_value=client))):
result: Final = await hc_module._run_model_health_check(deployment)
assert "configurable_clientside_auth_params" in result["error"]
invoke_endpoint.assert_not_called()