This commit is contained in:
luisgallego-aily 2026-09-20 03:39:23 -04:00 • committed by GitHub
commit e07002a1e8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 292 additions and 28 deletions

View file

@ -869,6 +869,13 @@ def get_bedrock_base_model(model: str) -> str:
return model
def get_bedrock_openai_model(model: object) -> str | None:
if not isinstance(model, str):
return None
normalized: Final = get_bedrock_base_model(model.removeprefix("bedrock_mantle/"))
return normalized if normalized.startswith("openai.gpt-") else None
def bedrock_converse_supports_parallel_tool_use_config(model: str) -> bool:
return any(
(litellm.model_cost.get(candidate) or {}).get("supports_parallel_tool_use_config") is True

View file

@ -0,0 +1,44 @@
from __future__ import annotations
from collections.abc import Callable, Mapping, Sequence
from typing import Final, cast # noqa: TID251 # narrows the untyped Responses payload
from litellm.llms.bedrock.common_utils import get_bedrock_openai_model
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.router import Deployment
def strip_incompatible_bedrock_openai_reasoning(
request_input: object,
healthy_deployments: Sequence[Mapping[str, object]],
get_routed_model_ids: Callable[[], frozenset[str]],
get_deployment: Callable[[str], Deployment | None],
) -> None:
if not isinstance(request_input, list):
return
target_models: Final = frozenset(
get_bedrock_openai_model(cast(Mapping[str, object], params).get("model"))
if isinstance(params, Mapping)
else None
for deployment in healthy_deployments
for params in (deployment.get("litellm_params"),)
)
if len(target_models) != 1 or None in target_models:
return
target_model: Final = next(iter(target_models))
items: Final = cast(list[object], request_input) # cast-ok: narrowed by isinstance
item_model_ids: Final = frozenset(
model_id
for item in items
if (model_id := ResponsesAPIRequestUtils.get_encrypted_content_model_id(item)) is not None
)
routed_model_ids: Final = get_routed_model_ids()
foreign_model_ids: Final = frozenset(
model_id
for model_id in item_model_ids
if model_id not in routed_model_ids
if (origin := get_deployment(model_id)) is None
or get_bedrock_openai_model(origin.litellm_params.model) != target_model
)
if foreign_model_ids:
ResponsesAPIRequestUtils.strip_encrypted_reasoning_from_input(items, originating_model_ids=foreign_model_ids)

View file

@ -556,7 +556,24 @@ class ResponsesAPIRequestUtils:
return request_input
@staticmethod
def strip_encrypted_reasoning_from_input(request_input: object) -> None:
def get_encrypted_content_model_id(item: object) -> str | None:
if not _is_object_dict(item):
return None
item_id: Final = item.get("id")
if isinstance(item_id, str):
decoded: Final = ResponsesAPIRequestUtils._decode_encrypted_item_id(item_id)
if decoded:
return decoded.get("model_id")
encrypted_content: Final = item.get("encrypted_content")
if isinstance(encrypted_content, str):
model_id, _ = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(encrypted_content)
return model_id or None
return None
@staticmethod
def strip_encrypted_reasoning_from_input(
request_input: object, *, originating_model_ids: frozenset[str] | None = None
) -> None:
"""Drop reasoning items the routed deployment cannot decrypt, keeping their readable summary.
Mutates ``request_input`` in place: the router's fallback snapshot shares this
@ -565,7 +582,13 @@ class ResponsesAPIRequestUtils:
if not isinstance(request_input, list):
return
items: Final = cast(list[object], request_input) # cast-ok: untyped client json
stripped: Final = tuple(ResponsesAPIRequestUtils._without_encrypted_reasoning(item) for item in items)
stripped: Final = tuple(
ResponsesAPIRequestUtils._without_encrypted_reasoning(item)
if originating_model_ids is None
or ResponsesAPIRequestUtils.get_encrypted_content_model_id(item) in originating_model_ids
else item
for item in items
)
items[:] = (item for item in stripped if item is not None) # rebind-ok: list shared with fallback snapshot
@staticmethod

View file

@ -129,25 +129,15 @@ class EncryptedContentAffinityCheck(CustomLogger):
if not isinstance(request_input, list):
return None
for item in request_input:
if not isinstance(item, dict):
continue
# First, try to decode from item ID (if present)
item_id = item.get("id")
if item_id and isinstance(item_id, str):
decoded = ResponsesAPIRequestUtils._decode_encrypted_item_id(item_id)
if decoded:
return decoded.get("model_id")
# If no encoded ID, check if encrypted_content itself is wrapped
encrypted_content = item.get("encrypted_content")
if encrypted_content and isinstance(encrypted_content, str):
model_id = EncryptedContentAffinityCheck._model_id_from_wrapped_encrypted_content(encrypted_content)
if model_id:
return model_id
return None
items: Final = cast(list[object], request_input) # cast-ok: narrowed by isinstance
return next(
(
model_id
for item in items
if (model_id := ResponsesAPIRequestUtils.get_encrypted_content_model_id(item)) is not None
),
None,
)
@staticmethod
def _anthropic_content_blocks(messages: object) -> Iterator[Mapping[str, object]]:
@ -216,9 +206,10 @@ class EncryptedContentAffinityCheck(CustomLogger):
@staticmethod
def _encryption_boundary_key(
litellm_params: object,
) -> tuple[object, object] | None:
) -> tuple[object, ...] | None:
"""
``(api_base, api_key)`` identifies an upstream encryption boundary.
Bedrock OpenAI reasoning also requires the same underlying model.
The values are resolved from the deployment and its named credential
without modifying the deployment.
@ -241,18 +232,23 @@ class EncryptedContentAffinityCheck(CustomLogger):
if isinstance(credential_name, str) and credential_name
else None
)
effective_api_base: Final = (
effective_api_base: Final[object] = (
credential_values.get("api_base")
if credential_values is not None and "api_base" in credential_values
else api_base
)
effective_api_key: Final = (
effective_api_key: Final[object] = (
credential_values.get("api_key")
if credential_values is not None and "api_key" in credential_values
else api_key
)
if not effective_api_base or not effective_api_key:
return None
from litellm.llms.bedrock.common_utils import get_bedrock_openai_model
bedrock_model: Final = get_bedrock_openai_model(getter("model"))
if bedrock_model is not None:
return (effective_api_base, effective_api_key, bedrock_model)
return (effective_api_base, effective_api_key)
def _find_deployments_on_same_encryption_boundary(
@ -290,7 +286,7 @@ class EncryptedContentAffinityCheck(CustomLogger):
model: str,
healthy_deployments: list,
messages: list[AllMessageValues] | None,
request_kwargs: dict | None = None,
request_kwargs: dict[str, object] | None = None,
parent_otel_span: Span | None = None,
) -> list[dict]:
"""
@ -310,7 +306,9 @@ class EncryptedContentAffinityCheck(CustomLogger):
retry after the deployment is eligible again.
"""
request_kwargs = request_kwargs or {}
typed_healthy_deployments: Final = cast(list[dict], healthy_deployments)
typed_healthy_deployments: Final = cast(
list[dict[str, object]], healthy_deployments
) # cast-ok: router deployment dictionaries
if not self._is_enabled_for_model_group(model):
return typed_healthy_deployments
@ -321,10 +319,20 @@ class EncryptedContentAffinityCheck(CustomLogger):
# completions / embeddings, which breaks tag-based routing because
# _get_metadata_variable_name_from_kwargs would pick "litellm_metadata"
# over "metadata" where tags are actually stored.
if "litellm_metadata" in request_kwargs:
request_kwargs["litellm_metadata"]["encrypted_content_affinity_enabled"] = True
metadata: Final = request_kwargs.get("litellm_metadata")
if isinstance(metadata, dict):
metadata["encrypted_content_affinity_enabled"] = True
request_input: Final = request_kwargs.get("input")
if self.router is not None:
from litellm.llms.bedrock.reasoning_affinity import strip_incompatible_bedrock_openai_reasoning
strip_incompatible_bedrock_openai_reasoning(
request_input,
typed_healthy_deployments,
lambda: self._routed_group_candidate_model_ids(request_kwargs, model),
lambda model_id: self.router.get_deployment(model_id=model_id),
)
anthropic_messages: Final = messages or request_kwargs.get("messages")
model_id: Final = self._extract_model_id_from_input(
request_input

View file

@ -1,5 +1,6 @@
from importlib import import_module
import base64
from typing import Final
from unittest.mock import MagicMock, patch
import pytest
@ -13,6 +14,33 @@ from litellm.types.utils import Usage
class TestResponsesAPIRequestUtils:
def test_selective_reasoning_removal_preserves_other_origins_and_compaction(self) -> None:
foreign: Final = {
"type": "reasoning",
"id": ResponsesAPIRequestUtils._build_encrypted_item_id("sol", "rs_sol"),
"encrypted_content": "opaque-sol",
"summary": [{"type": "summary_text", "text": "Investigated the failing test"}],
}
native: Final = {
"type": "reasoning",
"encrypted_content": ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("opaque-luna", "luna"),
"summary": [],
}
compaction: Final = {
"type": "compaction",
"encrypted_content": ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("opaque-state", "sol"),
}
untagged: Final = {"type": "reasoning", "encrypted_content": "opaque-unknown", "summary": []}
history: Final = [foreign, native, compaction, untagged]
ResponsesAPIRequestUtils.strip_encrypted_reasoning_from_input(history, originating_model_ids=frozenset({"sol"}))
assert history == [{"type": "reasoning", "summary": foreign["summary"]}, native, compaction, untagged]
assert foreign["encrypted_content"] == "opaque-sol"
assert history[1] is native
assert history[2] is compaction
assert history[3] is untagged
def test_get_optional_params_responses_api(self):
"""Test that optional parameters are correctly processed for responses API"""
# Setup

View file

@ -16,6 +16,8 @@ The mechanism works without any cache and supports two encoding strategies:
"""
import time
from copy import deepcopy
from typing import Final
from unittest.mock import AsyncMock, patch
import pytest
@ -2120,3 +2122,155 @@ async def test_affinity_honors_router_candidate_ids_for_team_and_pattern_routes(
assert request_kwargs["input"][1].get("encrypted_content")
mock_router.get_candidate_model_ids_for_route.assert_called_once_with(model="team-public-model", team_id="teamA")
@pytest.fixture
def bedrock_switch_router() -> litellm.Router:
return litellm.Router(
model_list=[
{
"model_name": name,
"model_info": {"id": name},
"litellm_params": {
"model": f"bedrock_mantle/openai.gpt-5.6-{upstream}",
"api_base": "https://bedrock-mantle.us-east-2.api.aws",
"api_key": "test-shared-key",
},
}
for name, upstream in (("sol", "sol"), ("luna", "luna"), ("sol-alias", "sol"))
]
)
def _bedrock_reasoning_item(origin: str, include_id: bool) -> dict[str, object]:
return {
"type": "reasoning",
"summary": [],
"encrypted_content": ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(
f"opaque-{origin}", origin
),
**({"id": ResponsesAPIRequestUtils._build_encrypted_item_id(origin, f"rs_{origin}")} if include_id else {}),
}
@pytest.mark.parametrize("include_id", [True, False])
@pytest.mark.parametrize(("source", "target"), [("sol", "luna"), ("luna", "sol"), ("sol", "sol"), ("sol", "sol-alias")])
async def test_bedrock_model_switch_only_removes_foreign_reasoning(
bedrock_switch_router: litellm.Router, source: str, target: str, include_id: bool
) -> None:
check: Final = EncryptedContentAffinityCheck(router=bedrock_switch_router)
pool: Final = [d for d in bedrock_switch_router.model_list if d["model_name"] == target]
reasoning: Final = _bedrock_reasoning_item(source, include_id)
request: Final = {
"litellm_metadata": {},
"reasoning": {"effort": "high", "context": "all_turns", "summary": "auto"},
"input": [
{"role": "user", "content": "Inspect the bug"},
reasoning,
{"role": "assistant", "phase": "final_answer", "content": "Found it"},
{"role": "user", "content": "Fix it"},
],
}
original: Final = deepcopy(request)
result: Final = await check.async_filter_deployments(target, pool, None, request)
assert result == pool
assert request["reasoning"] == original["reasoning"]
expected: Final = (
original["input"]
if source == target or target == "sol-alias"
else [item for item in original["input"] if item.get("type") != "reasoning"]
)
assert request["input"] == expected
assert request["litellm_metadata"]["encrypted_content_affinity_enabled"] is True
@pytest.mark.parametrize("target", ["sol", "luna"])
async def test_bedrock_mixed_history_keeps_selected_models_reasoning(
bedrock_switch_router: litellm.Router, target: str
) -> None:
check: Final = EncryptedContentAffinityCheck(router=bedrock_switch_router)
pool: Final = [d for d in bedrock_switch_router.model_list if d["model_name"] == target]
sol: Final = _bedrock_reasoning_item("sol", False)
luna: Final = _bedrock_reasoning_item("luna", False)
tool_call: Final = {"type": "function_call", "call_id": "call_1", "name": "inspect", "arguments": "{}"}
tool_result: Final = {"type": "function_call_output", "call_id": "call_1", "output": "inspected"}
history: Final = [
{"role": "user", "content": "Inspect"}, sol, tool_call, tool_result,
{"role": "assistant", "phase": "final_answer", "content": "Done"},
{"role": "user", "content": "Fix"}, luna,
{"role": "user", "content": "Continue"},
]
request: Final = {"litellm_metadata": {}, "input": deepcopy(history)}
shared_input: Final = request["input"]
result: Final = await check.async_filter_deployments(target, pool, None, request)
assert result == pool
assert request["input"] is shared_input
assert shared_input == [item for item in history if item != (sol if target == "luna" else luna)]
async def test_bedrock_switch_filter_respects_opt_in(bedrock_switch_router: litellm.Router) -> None:
check: Final = EncryptedContentAffinityCheck(router=bedrock_switch_router, enable_global_affinity=False)
pool: Final = [d for d in bedrock_switch_router.model_list if d["model_name"] == "luna"]
request: Final = {"litellm_metadata": {}, "input": [_bedrock_reasoning_item("sol", False)]}
original: Final = deepcopy(request)
assert await check.async_filter_deployments("luna", pool, None, request) == pool
assert request == original
@pytest.mark.parametrize("untrusted_origin", ["unknown-deployment", "sol"])
async def test_bedrock_unknown_marker_uses_same_safe_path_as_untrusted_foreign_marker(
bedrock_switch_router: litellm.Router,
untrusted_origin: str,
) -> None:
check: Final = EncryptedContentAffinityCheck(router=bedrock_switch_router)
pool: Final = [d for d in bedrock_switch_router.model_list if d["model_name"] == "luna"]
request: Final = {
"litellm_metadata": {},
"input": [
_bedrock_reasoning_item(untrusted_origin, False),
_bedrock_reasoning_item("luna", False),
],
}
assert await check.async_filter_deployments("luna", pool, None, request) == pool
assert request["input"] == [_bedrock_reasoning_item("luna", False)]
async def test_bedrock_cooldown_does_not_become_a_model_switch() -> None:
router: Final = litellm.Router(
model_list=[
{
"model_name": "auto",
"model_info": {"id": name},
"litellm_params": {
"model": f"bedrock_mantle/openai.gpt-5.6-{name}",
"api_base": "https://bedrock-mantle.us-east-2.api.aws",
"api_key": "test-shared-key",
},
}
for name in ("sol", "luna")
]
)
check: Final = EncryptedContentAffinityCheck(router=router)
pool: Final = [d for d in router.model_list if d["model_info"]["id"] == "luna"]
request: Final = {"input": [_bedrock_reasoning_item("sol", False)]}
original: Final = deepcopy(request)
with pytest.raises(litellm.ServiceUnavailableError):
await check.async_filter_deployments("auto", pool, None, request)
assert request == original
@pytest.mark.parametrize("prefix", ["bedrock_mantle/", "bedrock/us.", "openai/us."])
def test_bedrock_shared_credentials_do_not_make_different_models_compatible(prefix: str) -> None:
shared: Final = {"api_base": "https://gateway.example/openai/v1", "api_key": "test-shared-key"}
sol: Final = {**shared, "model": f"{prefix}openai.gpt-5.6-sol"}
luna: Final = {**shared, "model": f"{prefix}openai.gpt-5.6-luna"}
assert EncryptedContentAffinityCheck._encryption_boundary_key(sol) != EncryptedContentAffinityCheck._encryption_boundary_key(luna)
assert EncryptedContentAffinityCheck._encryption_boundary_key(sol) == EncryptedContentAffinityCheck._encryption_boundary_key(dict(sol))