diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 7e24292a87e..883b2b634dc 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -869,6 +869,13 @@ def get_bedrock_base_model(model: str) -> str: return model +def get_bedrock_openai_model(model: object) -> str | None: + if not isinstance(model, str): + return None + normalized: Final = get_bedrock_base_model(model.removeprefix("bedrock_mantle/")) + return normalized if normalized.startswith("openai.gpt-") else None + + def bedrock_converse_supports_parallel_tool_use_config(model: str) -> bool: return any( (litellm.model_cost.get(candidate) or {}).get("supports_parallel_tool_use_config") is True diff --git a/litellm/llms/bedrock/reasoning_affinity.py b/litellm/llms/bedrock/reasoning_affinity.py new file mode 100644 index 00000000000..f20d3f3ed56 --- /dev/null +++ b/litellm/llms/bedrock/reasoning_affinity.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from collections.abc import Callable, Mapping, Sequence +from typing import Final, cast # noqa: TID251 # narrows the untyped Responses payload + +from litellm.llms.bedrock.common_utils import get_bedrock_openai_model +from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.types.router import Deployment + + +def strip_incompatible_bedrock_openai_reasoning( + request_input: object, + healthy_deployments: Sequence[Mapping[str, object]], + get_routed_model_ids: Callable[[], frozenset[str]], + get_deployment: Callable[[str], Deployment | None], +) -> None: + if not isinstance(request_input, list): + return + target_models: Final = frozenset( + get_bedrock_openai_model(cast(Mapping[str, object], params).get("model")) + if isinstance(params, Mapping) + else None + for deployment in healthy_deployments + for params in (deployment.get("litellm_params"),) + ) + if len(target_models) != 1 or None in target_models: + return + target_model: Final = next(iter(target_models)) + items: Final = cast(list[object], request_input) # cast-ok: narrowed by isinstance + item_model_ids: Final = frozenset( + model_id + for item in items + if (model_id := ResponsesAPIRequestUtils.get_encrypted_content_model_id(item)) is not None + ) + routed_model_ids: Final = get_routed_model_ids() + foreign_model_ids: Final = frozenset( + model_id + for model_id in item_model_ids + if model_id not in routed_model_ids + if (origin := get_deployment(model_id)) is None + or get_bedrock_openai_model(origin.litellm_params.model) != target_model + ) + if foreign_model_ids: + ResponsesAPIRequestUtils.strip_encrypted_reasoning_from_input(items, originating_model_ids=foreign_model_ids) diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index a2642795cea..34d840d1de5 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -556,7 +556,24 @@ class ResponsesAPIRequestUtils: return request_input @staticmethod - def strip_encrypted_reasoning_from_input(request_input: object) -> None: + def get_encrypted_content_model_id(item: object) -> str | None: + if not _is_object_dict(item): + return None + item_id: Final = item.get("id") + if isinstance(item_id, str): + decoded: Final = ResponsesAPIRequestUtils._decode_encrypted_item_id(item_id) + if decoded: + return decoded.get("model_id") + encrypted_content: Final = item.get("encrypted_content") + if isinstance(encrypted_content, str): + model_id, _ = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(encrypted_content) + return model_id or None + return None + + @staticmethod + def strip_encrypted_reasoning_from_input( + request_input: object, *, originating_model_ids: frozenset[str] | None = None + ) -> None: """Drop reasoning items the routed deployment cannot decrypt, keeping their readable summary. Mutates ``request_input`` in place: the router's fallback snapshot shares this @@ -565,7 +582,13 @@ class ResponsesAPIRequestUtils: if not isinstance(request_input, list): return items: Final = cast(list[object], request_input) # cast-ok: untyped client json - stripped: Final = tuple(ResponsesAPIRequestUtils._without_encrypted_reasoning(item) for item in items) + stripped: Final = tuple( + ResponsesAPIRequestUtils._without_encrypted_reasoning(item) + if originating_model_ids is None + or ResponsesAPIRequestUtils.get_encrypted_content_model_id(item) in originating_model_ids + else item + for item in items + ) items[:] = (item for item in stripped if item is not None) # rebind-ok: list shared with fallback snapshot @staticmethod diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index cb5c3089685..9c9588514fb 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -129,25 +129,15 @@ class EncryptedContentAffinityCheck(CustomLogger): if not isinstance(request_input, list): return None - for item in request_input: - if not isinstance(item, dict): - continue - - # First, try to decode from item ID (if present) - item_id = item.get("id") - if item_id and isinstance(item_id, str): - decoded = ResponsesAPIRequestUtils._decode_encrypted_item_id(item_id) - if decoded: - return decoded.get("model_id") - - # If no encoded ID, check if encrypted_content itself is wrapped - encrypted_content = item.get("encrypted_content") - if encrypted_content and isinstance(encrypted_content, str): - model_id = EncryptedContentAffinityCheck._model_id_from_wrapped_encrypted_content(encrypted_content) - if model_id: - return model_id - - return None + items: Final = cast(list[object], request_input) # cast-ok: narrowed by isinstance + return next( + ( + model_id + for item in items + if (model_id := ResponsesAPIRequestUtils.get_encrypted_content_model_id(item)) is not None + ), + None, + ) @staticmethod def _anthropic_content_blocks(messages: object) -> Iterator[Mapping[str, object]]: @@ -216,9 +206,10 @@ class EncryptedContentAffinityCheck(CustomLogger): @staticmethod def _encryption_boundary_key( litellm_params: object, - ) -> tuple[object, object] | None: + ) -> tuple[object, ...] | None: """ ``(api_base, api_key)`` identifies an upstream encryption boundary. + Bedrock OpenAI reasoning also requires the same underlying model. The values are resolved from the deployment and its named credential without modifying the deployment. @@ -241,18 +232,23 @@ class EncryptedContentAffinityCheck(CustomLogger): if isinstance(credential_name, str) and credential_name else None ) - effective_api_base: Final = ( + effective_api_base: Final[object] = ( credential_values.get("api_base") if credential_values is not None and "api_base" in credential_values else api_base ) - effective_api_key: Final = ( + effective_api_key: Final[object] = ( credential_values.get("api_key") if credential_values is not None and "api_key" in credential_values else api_key ) if not effective_api_base or not effective_api_key: return None + from litellm.llms.bedrock.common_utils import get_bedrock_openai_model + + bedrock_model: Final = get_bedrock_openai_model(getter("model")) + if bedrock_model is not None: + return (effective_api_base, effective_api_key, bedrock_model) return (effective_api_base, effective_api_key) def _find_deployments_on_same_encryption_boundary( @@ -290,7 +286,7 @@ class EncryptedContentAffinityCheck(CustomLogger): model: str, healthy_deployments: list, messages: list[AllMessageValues] | None, - request_kwargs: dict | None = None, + request_kwargs: dict[str, object] | None = None, parent_otel_span: Span | None = None, ) -> list[dict]: """ @@ -310,7 +306,9 @@ class EncryptedContentAffinityCheck(CustomLogger): retry after the deployment is eligible again. """ request_kwargs = request_kwargs or {} - typed_healthy_deployments: Final = cast(list[dict], healthy_deployments) + typed_healthy_deployments: Final = cast( + list[dict[str, object]], healthy_deployments + ) # cast-ok: router deployment dictionaries if not self._is_enabled_for_model_group(model): return typed_healthy_deployments @@ -321,10 +319,20 @@ class EncryptedContentAffinityCheck(CustomLogger): # completions / embeddings, which breaks tag-based routing because # _get_metadata_variable_name_from_kwargs would pick "litellm_metadata" # over "metadata" where tags are actually stored. - if "litellm_metadata" in request_kwargs: - request_kwargs["litellm_metadata"]["encrypted_content_affinity_enabled"] = True + metadata: Final = request_kwargs.get("litellm_metadata") + if isinstance(metadata, dict): + metadata["encrypted_content_affinity_enabled"] = True request_input: Final = request_kwargs.get("input") + if self.router is not None: + from litellm.llms.bedrock.reasoning_affinity import strip_incompatible_bedrock_openai_reasoning + + strip_incompatible_bedrock_openai_reasoning( + request_input, + typed_healthy_deployments, + lambda: self._routed_group_candidate_model_ids(request_kwargs, model), + lambda model_id: self.router.get_deployment(model_id=model_id), + ) anthropic_messages: Final = messages or request_kwargs.get("messages") model_id: Final = self._extract_model_id_from_input( request_input diff --git a/tests/test_litellm/responses/test_responses_utils.py b/tests/test_litellm/responses/test_responses_utils.py index 7a482488706..d2d9fdb3d5a 100644 --- a/tests/test_litellm/responses/test_responses_utils.py +++ b/tests/test_litellm/responses/test_responses_utils.py @@ -1,5 +1,6 @@ from importlib import import_module import base64 +from typing import Final from unittest.mock import MagicMock, patch import pytest @@ -13,6 +14,33 @@ from litellm.types.utils import Usage class TestResponsesAPIRequestUtils: + def test_selective_reasoning_removal_preserves_other_origins_and_compaction(self) -> None: + foreign: Final = { + "type": "reasoning", + "id": ResponsesAPIRequestUtils._build_encrypted_item_id("sol", "rs_sol"), + "encrypted_content": "opaque-sol", + "summary": [{"type": "summary_text", "text": "Investigated the failing test"}], + } + native: Final = { + "type": "reasoning", + "encrypted_content": ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("opaque-luna", "luna"), + "summary": [], + } + compaction: Final = { + "type": "compaction", + "encrypted_content": ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("opaque-state", "sol"), + } + untagged: Final = {"type": "reasoning", "encrypted_content": "opaque-unknown", "summary": []} + history: Final = [foreign, native, compaction, untagged] + + ResponsesAPIRequestUtils.strip_encrypted_reasoning_from_input(history, originating_model_ids=frozenset({"sol"})) + + assert history == [{"type": "reasoning", "summary": foreign["summary"]}, native, compaction, untagged] + assert foreign["encrypted_content"] == "opaque-sol" + assert history[1] is native + assert history[2] is compaction + assert history[3] is untagged + def test_get_optional_params_responses_api(self): """Test that optional parameters are correctly processed for responses API""" # Setup diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index b93b8c1cdfc..83fc3f0eea3 100644 --- a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -16,6 +16,8 @@ The mechanism works without any cache and supports two encoding strategies: """ import time +from copy import deepcopy +from typing import Final from unittest.mock import AsyncMock, patch import pytest @@ -2120,3 +2122,155 @@ async def test_affinity_honors_router_candidate_ids_for_team_and_pattern_routes( assert request_kwargs["input"][1].get("encrypted_content") mock_router.get_candidate_model_ids_for_route.assert_called_once_with(model="team-public-model", team_id="teamA") + + +@pytest.fixture +def bedrock_switch_router() -> litellm.Router: + return litellm.Router( + model_list=[ + { + "model_name": name, + "model_info": {"id": name}, + "litellm_params": { + "model": f"bedrock_mantle/openai.gpt-5.6-{upstream}", + "api_base": "https://bedrock-mantle.us-east-2.api.aws", + "api_key": "test-shared-key", + }, + } + for name, upstream in (("sol", "sol"), ("luna", "luna"), ("sol-alias", "sol")) + ] + ) + + +def _bedrock_reasoning_item(origin: str, include_id: bool) -> dict[str, object]: + return { + "type": "reasoning", + "summary": [], + "encrypted_content": ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id( + f"opaque-{origin}", origin + ), + **({"id": ResponsesAPIRequestUtils._build_encrypted_item_id(origin, f"rs_{origin}")} if include_id else {}), + } + + +@pytest.mark.parametrize("include_id", [True, False]) +@pytest.mark.parametrize(("source", "target"), [("sol", "luna"), ("luna", "sol"), ("sol", "sol"), ("sol", "sol-alias")]) +async def test_bedrock_model_switch_only_removes_foreign_reasoning( + bedrock_switch_router: litellm.Router, source: str, target: str, include_id: bool +) -> None: + check: Final = EncryptedContentAffinityCheck(router=bedrock_switch_router) + pool: Final = [d for d in bedrock_switch_router.model_list if d["model_name"] == target] + reasoning: Final = _bedrock_reasoning_item(source, include_id) + request: Final = { + "litellm_metadata": {}, + "reasoning": {"effort": "high", "context": "all_turns", "summary": "auto"}, + "input": [ + {"role": "user", "content": "Inspect the bug"}, + reasoning, + {"role": "assistant", "phase": "final_answer", "content": "Found it"}, + {"role": "user", "content": "Fix it"}, + ], + } + original: Final = deepcopy(request) + + result: Final = await check.async_filter_deployments(target, pool, None, request) + + assert result == pool + assert request["reasoning"] == original["reasoning"] + expected: Final = ( + original["input"] + if source == target or target == "sol-alias" + else [item for item in original["input"] if item.get("type") != "reasoning"] + ) + assert request["input"] == expected + assert request["litellm_metadata"]["encrypted_content_affinity_enabled"] is True + + +@pytest.mark.parametrize("target", ["sol", "luna"]) +async def test_bedrock_mixed_history_keeps_selected_models_reasoning( + bedrock_switch_router: litellm.Router, target: str +) -> None: + check: Final = EncryptedContentAffinityCheck(router=bedrock_switch_router) + pool: Final = [d for d in bedrock_switch_router.model_list if d["model_name"] == target] + sol: Final = _bedrock_reasoning_item("sol", False) + luna: Final = _bedrock_reasoning_item("luna", False) + tool_call: Final = {"type": "function_call", "call_id": "call_1", "name": "inspect", "arguments": "{}"} + tool_result: Final = {"type": "function_call_output", "call_id": "call_1", "output": "inspected"} + history: Final = [ + {"role": "user", "content": "Inspect"}, sol, tool_call, tool_result, + {"role": "assistant", "phase": "final_answer", "content": "Done"}, + {"role": "user", "content": "Fix"}, luna, + {"role": "user", "content": "Continue"}, + ] + request: Final = {"litellm_metadata": {}, "input": deepcopy(history)} + shared_input: Final = request["input"] + + result: Final = await check.async_filter_deployments(target, pool, None, request) + + assert result == pool + assert request["input"] is shared_input + assert shared_input == [item for item in history if item != (sol if target == "luna" else luna)] + + +async def test_bedrock_switch_filter_respects_opt_in(bedrock_switch_router: litellm.Router) -> None: + check: Final = EncryptedContentAffinityCheck(router=bedrock_switch_router, enable_global_affinity=False) + pool: Final = [d for d in bedrock_switch_router.model_list if d["model_name"] == "luna"] + request: Final = {"litellm_metadata": {}, "input": [_bedrock_reasoning_item("sol", False)]} + original: Final = deepcopy(request) + + assert await check.async_filter_deployments("luna", pool, None, request) == pool + assert request == original + + +@pytest.mark.parametrize("untrusted_origin", ["unknown-deployment", "sol"]) +async def test_bedrock_unknown_marker_uses_same_safe_path_as_untrusted_foreign_marker( + bedrock_switch_router: litellm.Router, + untrusted_origin: str, +) -> None: + check: Final = EncryptedContentAffinityCheck(router=bedrock_switch_router) + pool: Final = [d for d in bedrock_switch_router.model_list if d["model_name"] == "luna"] + request: Final = { + "litellm_metadata": {}, + "input": [ + _bedrock_reasoning_item(untrusted_origin, False), + _bedrock_reasoning_item("luna", False), + ], + } + + assert await check.async_filter_deployments("luna", pool, None, request) == pool + assert request["input"] == [_bedrock_reasoning_item("luna", False)] + + +async def test_bedrock_cooldown_does_not_become_a_model_switch() -> None: + router: Final = litellm.Router( + model_list=[ + { + "model_name": "auto", + "model_info": {"id": name}, + "litellm_params": { + "model": f"bedrock_mantle/openai.gpt-5.6-{name}", + "api_base": "https://bedrock-mantle.us-east-2.api.aws", + "api_key": "test-shared-key", + }, + } + for name in ("sol", "luna") + ] + ) + check: Final = EncryptedContentAffinityCheck(router=router) + pool: Final = [d for d in router.model_list if d["model_info"]["id"] == "luna"] + request: Final = {"input": [_bedrock_reasoning_item("sol", False)]} + original: Final = deepcopy(request) + + with pytest.raises(litellm.ServiceUnavailableError): + await check.async_filter_deployments("auto", pool, None, request) + assert request == original + + +@pytest.mark.parametrize("prefix", ["bedrock_mantle/", "bedrock/us.", "openai/us."]) +def test_bedrock_shared_credentials_do_not_make_different_models_compatible(prefix: str) -> None: + shared: Final = {"api_base": "https://gateway.example/openai/v1", "api_key": "test-shared-key"} + sol: Final = {**shared, "model": f"{prefix}openai.gpt-5.6-sol"} + luna: Final = {**shared, "model": f"{prefix}openai.gpt-5.6-luna"} + + assert EncryptedContentAffinityCheck._encryption_boundary_key(sol) != EncryptedContentAffinityCheck._encryption_boundary_key(luna) + assert EncryptedContentAffinityCheck._encryption_boundary_key(sol) == EncryptedContentAffinityCheck._encryption_boundary_key(dict(sol))