From 03b2e2e00aa2847f7839c880defcf3d719958952 Mon Sep 17 00:00:00 2001 From: luisgallego-aily <162148370+luisgallego-aily@users.noreply.github.com> Date: Fri, 18 Sep 2026 10:14:22 +0200 Subject: [PATCH 1/4] fix: preserve compatible reasoning on Bedrock model switches --- .../codex_bedrock_model_switch.md | 60 ++++++++ litellm/responses/utils.py | 27 +++- .../encrypted_content_affinity_check.py | 98 +++++++++---- .../responses/test_responses_utils.py | 28 ++++ .../test_encrypted_content_affinity_check.py | 135 ++++++++++++++++++ 5 files changed, 319 insertions(+), 29 deletions(-) create mode 100644 cookbook/ai_coding_tool_guides/codex_bedrock_model_switch.md diff --git a/cookbook/ai_coding_tool_guides/codex_bedrock_model_switch.md b/cookbook/ai_coding_tool_guides/codex_bedrock_model_switch.md new file mode 100644 index 00000000000..ee6cb463ad2 --- /dev/null +++ b/cookbook/ai_coding_tool_guides/codex_bedrock_model_switch.md @@ -0,0 +1,60 @@ +# Switch Bedrock OpenAI models without replaying incompatible reasoning + +Bedrock can reject encrypted reasoning when a Responses API conversation changes models, even within GPT-5.6. A Sol response replayed to Luna can return HTTP 400 with `encrypted reasoning was created for a different account or model`, while replaying the same input to Sol succeeds + +Enable encrypted-content affinity on the proxy before starting the conversation: + +```yaml +model_list: + - model_name: sol + litellm_params: + model: bedrock_mantle/openai.gpt-5.6-sol + aws_region_name: us-east-2 + api_key: os.environ/AWS_BEARER_TOKEN_BEDROCK + - model_name: luna + litellm_params: + model: bedrock_mantle/openai.gpt-5.6-luna + aws_region_name: us-east-2 + api_key: os.environ/AWS_BEARER_TOKEN_BEDROCK + +router_settings: + optional_pre_call_checks: + - encrypted_content_affinity +``` + +For SigV4 authentication, omit `api_key` and configure the usual AWS credentials. The callback also recognizes Bedrock OpenAI model IDs routed through an OpenAI-compatible upstream, such as `openai/us.openai.gpt-5.6-sol` + +Continue replaying the response output items in their original order. LiteLLM tags encrypted items with their originating deployment and removes the tracking wrapper before forwarding to the provider. Codex can omit item IDs because the encrypted-content wrapper also carries the origin + +When the selected route has one underlying Bedrock OpenAI model, LiteLLM removes encrypted reasoning from other known Bedrock OpenAI models outside that route. It preserves readable reasoning summaries, messages, assistant phases, function calls, and function outputs. Reasoning from the selected model remains available, including in histories containing several models or when switching back to an earlier model. `reasoning.context`, effort, and summary settings are not overridden + +Calls that stay on the same model retain encrypted reasoning. Aliases of the same underlying model retain it when their encryption boundary matches. If the originating deployment remains in the requested model group but is unavailable, the existing affinity/cooldown behavior applies; an unavailable deployment is not treated as an explicit model switch + +The tradeoff occurs only for incompatible reasoning: the destination model must reconstruct any analysis that existed only in that reasoning. The HTTP client's original history remains unchanged, so switching back can reuse the earlier model's compatible items. This does not make encrypted reasoning portable between models + +## Existing conversations and other opaque state + +The proxy cannot infer the origin of an untagged encrypted item. Conversations started before affinity was enabled need a fresh conversation with a visible handoff, or client-side removal of the incompatible reasoning. Retain stable deployment IDs and do not repurpose an existing ID for another upstream model while clients still reference it + +This mitigation targets reasoning items. Compaction items and other opaque state are preserved and may have their own compatibility restrictions. Routes containing several underlying models retain the existing deployment-affinity behavior + +## Why not set `current_turn` globally? + +Direct Bedrock Runtime tests on 2026-09-18 returned the following results with fixed credentials in `us-east-2` and `store: false`: + +| Replay | Default `all_turns` | `current_turn` | Reasoning removed | +| --- | --- | --- | --- | +| Sol to Luna, next user turn | 400 | 200 | 200 | +| Luna to Sol, next user turn | 400 | 200 | 200 | +| Luna to Terra, next user turn | 400 | 200 | 200 | +| Terra to Luna, next user turn | 400 | 200 | 200 | +| Sol to Luna, active tool loop | 400 | 400 | 200 | +| Luna to Sol, active tool loop | 400 | 400 | 200 | + +Same-model controls returned 200. A reasoning item from a Codex conversation reproduced the same Luna-to-Terra failure. These observations conflict with the [documented Sol/Terra/Luna reasoning reuse](https://developers.openai.com/api/docs/guides/reasoning#preserve-reasoning-across-calls); the provider's internal cause remains unconfirmed + +`current_turn` excludes reasoning from completed user turns, even when the model stays the same. It also leaves incompatible reasoning in the payload and still fails for a switch during the active tool loop. Selective removal preserves compatible reasoning without imposing that global loss of continuity + +The [Codex configuration reference](https://learn.chatgpt.com/docs/config-file/config-reference) and schema checked on 2026-09-18 expose effort and summary controls, but no documented `reasoning.context` override. This mitigation is configured in LiteLLM + +The direct Mantle reproduction was reported separately. The Runtime tests above were independently executed; Mantle retesting was blocked by missing `bedrock-mantle:CreateInference` permission diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 41a3ded7022..c746a5cae29 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -544,7 +544,24 @@ class ResponsesAPIRequestUtils: return request_input @staticmethod - def strip_encrypted_reasoning_from_input(request_input: object) -> None: + def get_encrypted_content_model_id(item: object) -> str | None: + if not _is_object_dict(item): + return None + item_id: Final = item.get("id") + if isinstance(item_id, str): + decoded: Final = ResponsesAPIRequestUtils._decode_encrypted_item_id(item_id) + if decoded: + return decoded.get("model_id") + encrypted_content: Final = item.get("encrypted_content") + if isinstance(encrypted_content, str): + model_id, _ = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(encrypted_content) + return model_id or None + return None + + @staticmethod + def strip_encrypted_reasoning_from_input( + request_input: object, *, originating_model_ids: frozenset[str] | None = None + ) -> None: """Drop reasoning items the routed deployment cannot decrypt, keeping their readable summary. Mutates ``request_input`` in place: the router's fallback snapshot shares this @@ -553,7 +570,13 @@ class ResponsesAPIRequestUtils: if not isinstance(request_input, list): return items: Final = cast(list[object], request_input) # cast-ok: untyped client json - stripped: Final = tuple(ResponsesAPIRequestUtils._without_encrypted_reasoning(item) for item in items) + stripped: Final = tuple( + ResponsesAPIRequestUtils._without_encrypted_reasoning(item) + if originating_model_ids is None + or ResponsesAPIRequestUtils.get_encrypted_content_model_id(item) in originating_model_ids + else item + for item in items + ) items[:] = (item for item in stripped if item is not None) # rebind-ok: list shared with fallback snapshot @staticmethod diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index cb5c3089685..63cd621eb32 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -37,7 +37,7 @@ Safe to enable globally: """ import time -from collections.abc import Iterator, Mapping +from collections.abc import Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Final, Optional, Protocol, cast import httpx @@ -129,25 +129,15 @@ class EncryptedContentAffinityCheck(CustomLogger): if not isinstance(request_input, list): return None - for item in request_input: - if not isinstance(item, dict): - continue - - # First, try to decode from item ID (if present) - item_id = item.get("id") - if item_id and isinstance(item_id, str): - decoded = ResponsesAPIRequestUtils._decode_encrypted_item_id(item_id) - if decoded: - return decoded.get("model_id") - - # If no encoded ID, check if encrypted_content itself is wrapped - encrypted_content = item.get("encrypted_content") - if encrypted_content and isinstance(encrypted_content, str): - model_id = EncryptedContentAffinityCheck._model_id_from_wrapped_encrypted_content(encrypted_content) - if model_id: - return model_id - - return None + items: Final = cast(list[object], request_input) # cast-ok: narrowed by isinstance + return next( + ( + model_id + for item in items + if (model_id := ResponsesAPIRequestUtils.get_encrypted_content_model_id(item)) is not None + ), + None, + ) @staticmethod def _anthropic_content_blocks(messages: object) -> Iterator[Mapping[str, object]]: @@ -213,12 +203,59 @@ class EncryptedContentAffinityCheck(CustomLogger): return frozenset() return self.router.get_candidate_model_ids_for_route(model=model, team_id=self._request_team_id(request_kwargs)) + @staticmethod + def _bedrock_openai_model(model: object) -> str | None: + from litellm.llms.bedrock.common_utils import get_bedrock_base_model + + if not isinstance(model, str): + return None + normalized: Final = get_bedrock_base_model(model.removeprefix("bedrock_mantle/")) + return normalized if normalized.startswith("openai.gpt-") else None + + def _strip_bedrock_model_switch_reasoning( + self, + request_input: object, + model: str, + healthy_deployments: Sequence[Mapping[str, object]], + request_kwargs: Mapping[str, object], + ) -> None: + if self.router is None or not isinstance(request_input, list): + return + target_models: Final = frozenset( + self._bedrock_openai_model( + cast(Mapping[str, object], params).get("model") # cast-ok: narrowed by isinstance + ) + if isinstance(params, Mapping) + else None + for deployment in healthy_deployments + for params in (deployment.get("litellm_params"),) + ) + if len(target_models) != 1 or None in target_models: + return + target_model: Final = next(iter(target_models)) + routed_model_ids: Final = self._routed_group_candidate_model_ids(request_kwargs, model) + items: Final = cast(list[object], request_input) # cast-ok: narrowed by isinstance + foreign_model_ids: Final = frozenset( + model_id + for item in items + if (model_id := ResponsesAPIRequestUtils.get_encrypted_content_model_id(item)) is not None + if model_id not in routed_model_ids + if (origin := self.router.get_deployment(model_id=model_id)) is not None + if (origin_model := self._bedrock_openai_model(origin.litellm_params.model)) is not None + if origin_model != target_model + ) + if foreign_model_ids: + ResponsesAPIRequestUtils.strip_encrypted_reasoning_from_input( + items, originating_model_ids=foreign_model_ids + ) + @staticmethod def _encryption_boundary_key( litellm_params: object, - ) -> tuple[object, object] | None: + ) -> tuple[object, ...] | None: """ ``(api_base, api_key)`` identifies an upstream encryption boundary. + Bedrock OpenAI reasoning also requires the same underlying model. The values are resolved from the deployment and its named credential without modifying the deployment. @@ -241,18 +278,21 @@ class EncryptedContentAffinityCheck(CustomLogger): if isinstance(credential_name, str) and credential_name else None ) - effective_api_base: Final = ( + effective_api_base: Final[object] = ( credential_values.get("api_base") if credential_values is not None and "api_base" in credential_values else api_base ) - effective_api_key: Final = ( + effective_api_key: Final[object] = ( credential_values.get("api_key") if credential_values is not None and "api_key" in credential_values else api_key ) if not effective_api_base or not effective_api_key: return None + bedrock_model: Final = EncryptedContentAffinityCheck._bedrock_openai_model(getter("model")) + if bedrock_model is not None: + return (effective_api_base, effective_api_key, bedrock_model) return (effective_api_base, effective_api_key) def _find_deployments_on_same_encryption_boundary( @@ -290,7 +330,7 @@ class EncryptedContentAffinityCheck(CustomLogger): model: str, healthy_deployments: list, messages: list[AllMessageValues] | None, - request_kwargs: dict | None = None, + request_kwargs: dict[str, object] | None = None, parent_otel_span: Span | None = None, ) -> list[dict]: """ @@ -310,7 +350,9 @@ class EncryptedContentAffinityCheck(CustomLogger): retry after the deployment is eligible again. """ request_kwargs = request_kwargs or {} - typed_healthy_deployments: Final = cast(list[dict], healthy_deployments) + typed_healthy_deployments: Final = cast( + list[dict[str, object]], healthy_deployments + ) # cast-ok: router deployment dictionaries if not self._is_enabled_for_model_group(model): return typed_healthy_deployments @@ -321,10 +363,12 @@ class EncryptedContentAffinityCheck(CustomLogger): # completions / embeddings, which breaks tag-based routing because # _get_metadata_variable_name_from_kwargs would pick "litellm_metadata" # over "metadata" where tags are actually stored. - if "litellm_metadata" in request_kwargs: - request_kwargs["litellm_metadata"]["encrypted_content_affinity_enabled"] = True + metadata: Final = request_kwargs.get("litellm_metadata") + if isinstance(metadata, dict): + metadata["encrypted_content_affinity_enabled"] = True request_input: Final = request_kwargs.get("input") + self._strip_bedrock_model_switch_reasoning(request_input, model, typed_healthy_deployments, request_kwargs) anthropic_messages: Final = messages or request_kwargs.get("messages") model_id: Final = self._extract_model_id_from_input( request_input diff --git a/tests/test_litellm/responses/test_responses_utils.py b/tests/test_litellm/responses/test_responses_utils.py index 4d06b5e7bdc..3a224976790 100644 --- a/tests/test_litellm/responses/test_responses_utils.py +++ b/tests/test_litellm/responses/test_responses_utils.py @@ -1,5 +1,6 @@ from importlib import import_module import base64 +from typing import Final from unittest.mock import MagicMock, patch import pytest @@ -13,6 +14,33 @@ from litellm.types.utils import Usage class TestResponsesAPIRequestUtils: + def test_selective_reasoning_removal_preserves_other_origins_and_compaction(self) -> None: + foreign: Final = { + "type": "reasoning", + "id": ResponsesAPIRequestUtils._build_encrypted_item_id("sol", "rs_sol"), + "encrypted_content": "opaque-sol", + "summary": [{"type": "summary_text", "text": "Investigated the failing test"}], + } + native: Final = { + "type": "reasoning", + "encrypted_content": ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("opaque-luna", "luna"), + "summary": [], + } + compaction: Final = { + "type": "compaction", + "encrypted_content": ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("opaque-state", "sol"), + } + untagged: Final = {"type": "reasoning", "encrypted_content": "opaque-unknown", "summary": []} + history: Final = [foreign, native, compaction, untagged] + + ResponsesAPIRequestUtils.strip_encrypted_reasoning_from_input(history, originating_model_ids=frozenset({"sol"})) + + assert history == [{"type": "reasoning", "summary": foreign["summary"]}, native, compaction, untagged] + assert foreign["encrypted_content"] == "opaque-sol" + assert history[1] is native + assert history[2] is compaction + assert history[3] is untagged + def test_get_optional_params_responses_api(self): """Test that optional parameters are correctly processed for responses API""" # Setup diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index b93b8c1cdfc..0c5b470152b 100644 --- a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -16,6 +16,8 @@ The mechanism works without any cache and supports two encoding strategies: """ import time +from copy import deepcopy +from typing import Final from unittest.mock import AsyncMock, patch import pytest @@ -2120,3 +2122,136 @@ async def test_affinity_honors_router_candidate_ids_for_team_and_pattern_routes( assert request_kwargs["input"][1].get("encrypted_content") mock_router.get_candidate_model_ids_for_route.assert_called_once_with(model="team-public-model", team_id="teamA") + + +@pytest.fixture +def bedrock_switch_router() -> litellm.Router: + return litellm.Router( + model_list=[ + { + "model_name": name, + "model_info": {"id": name}, + "litellm_params": { + "model": f"bedrock_mantle/openai.gpt-5.6-{upstream}", + "api_base": "https://bedrock-mantle.us-east-2.api.aws", + "api_key": "test-shared-key", + }, + } + for name, upstream in (("sol", "sol"), ("luna", "luna"), ("sol-alias", "sol")) + ] + ) + + +def _bedrock_reasoning_item(origin: str, include_id: bool) -> dict[str, object]: + return { + "type": "reasoning", + "summary": [], + "encrypted_content": ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id( + f"opaque-{origin}", origin + ), + **({"id": ResponsesAPIRequestUtils._build_encrypted_item_id(origin, f"rs_{origin}")} if include_id else {}), + } + + +@pytest.mark.parametrize("include_id", [True, False]) +@pytest.mark.parametrize(("source", "target"), [("sol", "luna"), ("luna", "sol"), ("sol", "sol"), ("sol", "sol-alias")]) +async def test_bedrock_model_switch_only_removes_foreign_reasoning( + bedrock_switch_router: litellm.Router, source: str, target: str, include_id: bool +) -> None: + check: Final = EncryptedContentAffinityCheck(router=bedrock_switch_router) + pool: Final = [d for d in bedrock_switch_router.model_list if d["model_name"] == target] + reasoning: Final = _bedrock_reasoning_item(source, include_id) + request: Final = { + "litellm_metadata": {}, + "reasoning": {"effort": "high", "context": "all_turns", "summary": "auto"}, + "input": [ + {"role": "user", "content": "Inspect the bug"}, + reasoning, + {"role": "assistant", "phase": "final_answer", "content": "Found it"}, + {"role": "user", "content": "Fix it"}, + ], + } + original: Final = deepcopy(request) + + result: Final = await check.async_filter_deployments(target, pool, None, request) + + assert result == pool + assert request["reasoning"] == original["reasoning"] + expected: Final = ( + original["input"] + if source == target or target == "sol-alias" + else [item for item in original["input"] if item.get("type") != "reasoning"] + ) + assert request["input"] == expected + assert request["litellm_metadata"]["encrypted_content_affinity_enabled"] is True + + +@pytest.mark.parametrize("target", ["sol", "luna"]) +async def test_bedrock_mixed_history_keeps_selected_models_reasoning( + bedrock_switch_router: litellm.Router, target: str +) -> None: + check: Final = EncryptedContentAffinityCheck(router=bedrock_switch_router) + pool: Final = [d for d in bedrock_switch_router.model_list if d["model_name"] == target] + sol: Final = _bedrock_reasoning_item("sol", False) + luna: Final = _bedrock_reasoning_item("luna", False) + tool_call: Final = {"type": "function_call", "call_id": "call_1", "name": "inspect", "arguments": "{}"} + tool_result: Final = {"type": "function_call_output", "call_id": "call_1", "output": "inspected"} + history: Final = [ + {"role": "user", "content": "Inspect"}, sol, tool_call, tool_result, + {"role": "assistant", "phase": "final_answer", "content": "Done"}, + {"role": "user", "content": "Fix"}, luna, + {"role": "user", "content": "Continue"}, + ] + request: Final = {"litellm_metadata": {}, "input": deepcopy(history)} + shared_input: Final = request["input"] + + result: Final = await check.async_filter_deployments(target, pool, None, request) + + assert result == pool + assert request["input"] is shared_input + assert shared_input == [item for item in history if item != (sol if target == "luna" else luna)] + + +async def test_bedrock_switch_filter_respects_opt_in(bedrock_switch_router: litellm.Router) -> None: + check: Final = EncryptedContentAffinityCheck(router=bedrock_switch_router, enable_global_affinity=False) + pool: Final = [d for d in bedrock_switch_router.model_list if d["model_name"] == "luna"] + request: Final = {"litellm_metadata": {}, "input": [_bedrock_reasoning_item("sol", False)]} + original: Final = deepcopy(request) + + assert await check.async_filter_deployments("luna", pool, None, request) == pool + assert request == original + + +async def test_bedrock_cooldown_does_not_become_a_model_switch() -> None: + router: Final = litellm.Router( + model_list=[ + { + "model_name": "auto", + "model_info": {"id": name}, + "litellm_params": { + "model": f"bedrock_mantle/openai.gpt-5.6-{name}", + "api_base": "https://bedrock-mantle.us-east-2.api.aws", + "api_key": "test-shared-key", + }, + } + for name in ("sol", "luna") + ] + ) + check: Final = EncryptedContentAffinityCheck(router=router) + pool: Final = [d for d in router.model_list if d["model_info"]["id"] == "luna"] + request: Final = {"input": [_bedrock_reasoning_item("sol", False)]} + original: Final = deepcopy(request) + + with pytest.raises(litellm.ServiceUnavailableError): + await check.async_filter_deployments("auto", pool, None, request) + assert request == original + + +@pytest.mark.parametrize("prefix", ["bedrock_mantle/", "bedrock/us.", "openai/us."]) +def test_bedrock_shared_credentials_do_not_make_different_models_compatible(prefix: str) -> None: + shared: Final = {"api_base": "https://gateway.example/openai/v1", "api_key": "test-shared-key"} + sol: Final = {**shared, "model": f"{prefix}openai.gpt-5.6-sol"} + luna: Final = {**shared, "model": f"{prefix}openai.gpt-5.6-luna"} + + assert EncryptedContentAffinityCheck._encryption_boundary_key(sol) != EncryptedContentAffinityCheck._encryption_boundary_key(luna) + assert EncryptedContentAffinityCheck._encryption_boundary_key(sol) == EncryptedContentAffinityCheck._encryption_boundary_key(dict(sol)) From f872dc5c4f9806f274f22e9acd4775bb03e37219 Mon Sep 17 00:00:00 2001 From: luisgallego-aily <162148370+luisgallego-aily@users.noreply.github.com> Date: Fri, 18 Sep 2026 12:44:44 +0200 Subject: [PATCH 2/4] docs: remove misplaced Bedrock model-switch guide --- .../codex_bedrock_model_switch.md | 60 ------------------- 1 file changed, 60 deletions(-) delete mode 100644 cookbook/ai_coding_tool_guides/codex_bedrock_model_switch.md diff --git a/cookbook/ai_coding_tool_guides/codex_bedrock_model_switch.md b/cookbook/ai_coding_tool_guides/codex_bedrock_model_switch.md deleted file mode 100644 index ee6cb463ad2..00000000000 --- a/cookbook/ai_coding_tool_guides/codex_bedrock_model_switch.md +++ /dev/null @@ -1,60 +0,0 @@ -# Switch Bedrock OpenAI models without replaying incompatible reasoning - -Bedrock can reject encrypted reasoning when a Responses API conversation changes models, even within GPT-5.6. A Sol response replayed to Luna can return HTTP 400 with `encrypted reasoning was created for a different account or model`, while replaying the same input to Sol succeeds - -Enable encrypted-content affinity on the proxy before starting the conversation: - -```yaml -model_list: - - model_name: sol - litellm_params: - model: bedrock_mantle/openai.gpt-5.6-sol - aws_region_name: us-east-2 - api_key: os.environ/AWS_BEARER_TOKEN_BEDROCK - - model_name: luna - litellm_params: - model: bedrock_mantle/openai.gpt-5.6-luna - aws_region_name: us-east-2 - api_key: os.environ/AWS_BEARER_TOKEN_BEDROCK - -router_settings: - optional_pre_call_checks: - - encrypted_content_affinity -``` - -For SigV4 authentication, omit `api_key` and configure the usual AWS credentials. The callback also recognizes Bedrock OpenAI model IDs routed through an OpenAI-compatible upstream, such as `openai/us.openai.gpt-5.6-sol` - -Continue replaying the response output items in their original order. LiteLLM tags encrypted items with their originating deployment and removes the tracking wrapper before forwarding to the provider. Codex can omit item IDs because the encrypted-content wrapper also carries the origin - -When the selected route has one underlying Bedrock OpenAI model, LiteLLM removes encrypted reasoning from other known Bedrock OpenAI models outside that route. It preserves readable reasoning summaries, messages, assistant phases, function calls, and function outputs. Reasoning from the selected model remains available, including in histories containing several models or when switching back to an earlier model. `reasoning.context`, effort, and summary settings are not overridden - -Calls that stay on the same model retain encrypted reasoning. Aliases of the same underlying model retain it when their encryption boundary matches. If the originating deployment remains in the requested model group but is unavailable, the existing affinity/cooldown behavior applies; an unavailable deployment is not treated as an explicit model switch - -The tradeoff occurs only for incompatible reasoning: the destination model must reconstruct any analysis that existed only in that reasoning. The HTTP client's original history remains unchanged, so switching back can reuse the earlier model's compatible items. This does not make encrypted reasoning portable between models - -## Existing conversations and other opaque state - -The proxy cannot infer the origin of an untagged encrypted item. Conversations started before affinity was enabled need a fresh conversation with a visible handoff, or client-side removal of the incompatible reasoning. Retain stable deployment IDs and do not repurpose an existing ID for another upstream model while clients still reference it - -This mitigation targets reasoning items. Compaction items and other opaque state are preserved and may have their own compatibility restrictions. Routes containing several underlying models retain the existing deployment-affinity behavior - -## Why not set `current_turn` globally? - -Direct Bedrock Runtime tests on 2026-09-18 returned the following results with fixed credentials in `us-east-2` and `store: false`: - -| Replay | Default `all_turns` | `current_turn` | Reasoning removed | -| --- | --- | --- | --- | -| Sol to Luna, next user turn | 400 | 200 | 200 | -| Luna to Sol, next user turn | 400 | 200 | 200 | -| Luna to Terra, next user turn | 400 | 200 | 200 | -| Terra to Luna, next user turn | 400 | 200 | 200 | -| Sol to Luna, active tool loop | 400 | 400 | 200 | -| Luna to Sol, active tool loop | 400 | 400 | 200 | - -Same-model controls returned 200. A reasoning item from a Codex conversation reproduced the same Luna-to-Terra failure. These observations conflict with the [documented Sol/Terra/Luna reasoning reuse](https://developers.openai.com/api/docs/guides/reasoning#preserve-reasoning-across-calls); the provider's internal cause remains unconfirmed - -`current_turn` excludes reasoning from completed user turns, even when the model stays the same. It also leaves incompatible reasoning in the payload and still fails for a switch during the active tool loop. Selective removal preserves compatible reasoning without imposing that global loss of continuity - -The [Codex configuration reference](https://learn.chatgpt.com/docs/config-file/config-reference) and schema checked on 2026-09-18 expose effort and summary controls, but no documented `reasoning.context` override. This mitigation is configured in LiteLLM - -The direct Mantle reproduction was reported separately. The Runtime tests above were independently executed; Mantle retesting was blocked by missing `bedrock-mantle:CreateInference` permission From b50a1239b91a9cf3be32f3928eaf557309215a3d Mon Sep 17 00:00:00 2001 From: luisgallego-aily <162148370+luisgallego-aily@users.noreply.github.com> Date: Fri, 18 Sep 2026 12:53:09 +0200 Subject: [PATCH 3/4] fix(router): move Bedrock reasoning affinity into provider layer --- litellm/llms/bedrock/common_utils.py | 7 +++ litellm/llms/bedrock/reasoning_affinity.py | 44 +++++++++++++ .../encrypted_content_affinity_check.py | 62 ++++--------------- .../test_encrypted_content_affinity_check.py | 19 ++++++ 4 files changed, 83 insertions(+), 49 deletions(-) create mode 100644 litellm/llms/bedrock/reasoning_affinity.py diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 4f030b156e7..fbcf317a8b9 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -868,6 +868,13 @@ def get_bedrock_base_model(model: str) -> str: return model +def get_bedrock_openai_model(model: object) -> str | None: + if not isinstance(model, str): + return None + normalized: Final = get_bedrock_base_model(model.removeprefix("bedrock_mantle/")) + return normalized if normalized.startswith("openai.gpt-") else None + + def bedrock_converse_supports_parallel_tool_use_config(model: str) -> bool: return any( (litellm.model_cost.get(candidate) or {}).get("supports_parallel_tool_use_config") is True diff --git a/litellm/llms/bedrock/reasoning_affinity.py b/litellm/llms/bedrock/reasoning_affinity.py new file mode 100644 index 00000000000..ff47426f5c1 --- /dev/null +++ b/litellm/llms/bedrock/reasoning_affinity.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from collections.abc import Callable, Mapping, Sequence +from typing import Final, cast + +from litellm.llms.bedrock.common_utils import get_bedrock_openai_model +from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.types.router import Deployment + + +def strip_incompatible_bedrock_openai_reasoning( + request_input: object, + healthy_deployments: Sequence[Mapping[str, object]], + get_routed_model_ids: Callable[[], frozenset[str]], + get_deployment: Callable[[str], Deployment | None], +) -> None: + if not isinstance(request_input, list): + return + target_models: Final = frozenset( + get_bedrock_openai_model(cast(Mapping[str, object], params).get("model")) + if isinstance(params, Mapping) + else None + for deployment in healthy_deployments + for params in (deployment.get("litellm_params"),) + ) + if len(target_models) != 1 or None in target_models: + return + target_model: Final = next(iter(target_models)) + items: Final = cast(list[object], request_input) # cast-ok: narrowed by isinstance + item_model_ids: Final = frozenset( + model_id + for item in items + if (model_id := ResponsesAPIRequestUtils.get_encrypted_content_model_id(item)) is not None + ) + routed_model_ids: Final = get_routed_model_ids() + foreign_model_ids: Final = frozenset( + model_id + for model_id in item_model_ids + if model_id not in routed_model_ids + if (origin := get_deployment(model_id)) is None + or get_bedrock_openai_model(origin.litellm_params.model) != target_model + ) + if foreign_model_ids: + ResponsesAPIRequestUtils.strip_encrypted_reasoning_from_input(items, originating_model_ids=foreign_model_ids) diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index 63cd621eb32..9c9588514fb 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -37,7 +37,7 @@ Safe to enable globally: """ import time -from collections.abc import Iterator, Mapping, Sequence +from collections.abc import Iterator, Mapping from typing import TYPE_CHECKING, Final, Optional, Protocol, cast import httpx @@ -203,52 +203,6 @@ class EncryptedContentAffinityCheck(CustomLogger): return frozenset() return self.router.get_candidate_model_ids_for_route(model=model, team_id=self._request_team_id(request_kwargs)) - @staticmethod - def _bedrock_openai_model(model: object) -> str | None: - from litellm.llms.bedrock.common_utils import get_bedrock_base_model - - if not isinstance(model, str): - return None - normalized: Final = get_bedrock_base_model(model.removeprefix("bedrock_mantle/")) - return normalized if normalized.startswith("openai.gpt-") else None - - def _strip_bedrock_model_switch_reasoning( - self, - request_input: object, - model: str, - healthy_deployments: Sequence[Mapping[str, object]], - request_kwargs: Mapping[str, object], - ) -> None: - if self.router is None or not isinstance(request_input, list): - return - target_models: Final = frozenset( - self._bedrock_openai_model( - cast(Mapping[str, object], params).get("model") # cast-ok: narrowed by isinstance - ) - if isinstance(params, Mapping) - else None - for deployment in healthy_deployments - for params in (deployment.get("litellm_params"),) - ) - if len(target_models) != 1 or None in target_models: - return - target_model: Final = next(iter(target_models)) - routed_model_ids: Final = self._routed_group_candidate_model_ids(request_kwargs, model) - items: Final = cast(list[object], request_input) # cast-ok: narrowed by isinstance - foreign_model_ids: Final = frozenset( - model_id - for item in items - if (model_id := ResponsesAPIRequestUtils.get_encrypted_content_model_id(item)) is not None - if model_id not in routed_model_ids - if (origin := self.router.get_deployment(model_id=model_id)) is not None - if (origin_model := self._bedrock_openai_model(origin.litellm_params.model)) is not None - if origin_model != target_model - ) - if foreign_model_ids: - ResponsesAPIRequestUtils.strip_encrypted_reasoning_from_input( - items, originating_model_ids=foreign_model_ids - ) - @staticmethod def _encryption_boundary_key( litellm_params: object, @@ -290,7 +244,9 @@ class EncryptedContentAffinityCheck(CustomLogger): ) if not effective_api_base or not effective_api_key: return None - bedrock_model: Final = EncryptedContentAffinityCheck._bedrock_openai_model(getter("model")) + from litellm.llms.bedrock.common_utils import get_bedrock_openai_model + + bedrock_model: Final = get_bedrock_openai_model(getter("model")) if bedrock_model is not None: return (effective_api_base, effective_api_key, bedrock_model) return (effective_api_base, effective_api_key) @@ -368,7 +324,15 @@ class EncryptedContentAffinityCheck(CustomLogger): metadata["encrypted_content_affinity_enabled"] = True request_input: Final = request_kwargs.get("input") - self._strip_bedrock_model_switch_reasoning(request_input, model, typed_healthy_deployments, request_kwargs) + if self.router is not None: + from litellm.llms.bedrock.reasoning_affinity import strip_incompatible_bedrock_openai_reasoning + + strip_incompatible_bedrock_openai_reasoning( + request_input, + typed_healthy_deployments, + lambda: self._routed_group_candidate_model_ids(request_kwargs, model), + lambda model_id: self.router.get_deployment(model_id=model_id), + ) anthropic_messages: Final = messages or request_kwargs.get("messages") model_id: Final = self._extract_model_id_from_input( request_input diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index 0c5b470152b..83fc3f0eea3 100644 --- a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -2222,6 +2222,25 @@ async def test_bedrock_switch_filter_respects_opt_in(bedrock_switch_router: lite assert request == original +@pytest.mark.parametrize("untrusted_origin", ["unknown-deployment", "sol"]) +async def test_bedrock_unknown_marker_uses_same_safe_path_as_untrusted_foreign_marker( + bedrock_switch_router: litellm.Router, + untrusted_origin: str, +) -> None: + check: Final = EncryptedContentAffinityCheck(router=bedrock_switch_router) + pool: Final = [d for d in bedrock_switch_router.model_list if d["model_name"] == "luna"] + request: Final = { + "litellm_metadata": {}, + "input": [ + _bedrock_reasoning_item(untrusted_origin, False), + _bedrock_reasoning_item("luna", False), + ], + } + + assert await check.async_filter_deployments("luna", pool, None, request) == pool + assert request["input"] == [_bedrock_reasoning_item("luna", False)] + + async def test_bedrock_cooldown_does_not_become_a_model_switch() -> None: router: Final = litellm.Router( model_list=[ From ecc3110339a55b4954faf4f69b51dfaec3990f43 Mon Sep 17 00:00:00 2001 From: luisgallego-aily <162148370+luisgallego-aily@users.noreply.github.com> Date: Fri, 18 Sep 2026 14:54:15 +0200 Subject: [PATCH 4/4] fix(router): justify Bedrock affinity cast --- litellm/llms/bedrock/reasoning_affinity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/bedrock/reasoning_affinity.py b/litellm/llms/bedrock/reasoning_affinity.py index ff47426f5c1..f20d3f3ed56 100644 --- a/litellm/llms/bedrock/reasoning_affinity.py +++ b/litellm/llms/bedrock/reasoning_affinity.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections.abc import Callable, Mapping, Sequence -from typing import Final, cast +from typing import Final, cast # noqa: TID251 # narrows the untyped Responses payload from litellm.llms.bedrock.common_utils import get_bedrock_openai_model from litellm.responses.utils import ResponsesAPIRequestUtils