mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge ecc3110339 into 776f8bc309
This commit is contained in:
commit
e07002a1e8
6 changed files with 292 additions and 28 deletions
|
|
@ -869,6 +869,13 @@ def get_bedrock_base_model(model: str) -> str:
|
|||
return model
|
||||
|
||||
|
||||
def get_bedrock_openai_model(model: object) -> str | None:
|
||||
if not isinstance(model, str):
|
||||
return None
|
||||
normalized: Final = get_bedrock_base_model(model.removeprefix("bedrock_mantle/"))
|
||||
return normalized if normalized.startswith("openai.gpt-") else None
|
||||
|
||||
|
||||
def bedrock_converse_supports_parallel_tool_use_config(model: str) -> bool:
|
||||
return any(
|
||||
(litellm.model_cost.get(candidate) or {}).get("supports_parallel_tool_use_config") is True
|
||||
|
|
|
|||
44
litellm/llms/bedrock/reasoning_affinity.py
Normal file
44
litellm/llms/bedrock/reasoning_affinity.py
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from typing import Final, cast # noqa: TID251 # narrows the untyped Responses payload
|
||||
|
||||
from litellm.llms.bedrock.common_utils import get_bedrock_openai_model
|
||||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
from litellm.types.router import Deployment
|
||||
|
||||
|
||||
def strip_incompatible_bedrock_openai_reasoning(
|
||||
request_input: object,
|
||||
healthy_deployments: Sequence[Mapping[str, object]],
|
||||
get_routed_model_ids: Callable[[], frozenset[str]],
|
||||
get_deployment: Callable[[str], Deployment | None],
|
||||
) -> None:
|
||||
if not isinstance(request_input, list):
|
||||
return
|
||||
target_models: Final = frozenset(
|
||||
get_bedrock_openai_model(cast(Mapping[str, object], params).get("model"))
|
||||
if isinstance(params, Mapping)
|
||||
else None
|
||||
for deployment in healthy_deployments
|
||||
for params in (deployment.get("litellm_params"),)
|
||||
)
|
||||
if len(target_models) != 1 or None in target_models:
|
||||
return
|
||||
target_model: Final = next(iter(target_models))
|
||||
items: Final = cast(list[object], request_input) # cast-ok: narrowed by isinstance
|
||||
item_model_ids: Final = frozenset(
|
||||
model_id
|
||||
for item in items
|
||||
if (model_id := ResponsesAPIRequestUtils.get_encrypted_content_model_id(item)) is not None
|
||||
)
|
||||
routed_model_ids: Final = get_routed_model_ids()
|
||||
foreign_model_ids: Final = frozenset(
|
||||
model_id
|
||||
for model_id in item_model_ids
|
||||
if model_id not in routed_model_ids
|
||||
if (origin := get_deployment(model_id)) is None
|
||||
or get_bedrock_openai_model(origin.litellm_params.model) != target_model
|
||||
)
|
||||
if foreign_model_ids:
|
||||
ResponsesAPIRequestUtils.strip_encrypted_reasoning_from_input(items, originating_model_ids=foreign_model_ids)
|
||||
|
|
@ -556,7 +556,24 @@ class ResponsesAPIRequestUtils:
|
|||
return request_input
|
||||
|
||||
@staticmethod
|
||||
def strip_encrypted_reasoning_from_input(request_input: object) -> None:
|
||||
def get_encrypted_content_model_id(item: object) -> str | None:
|
||||
if not _is_object_dict(item):
|
||||
return None
|
||||
item_id: Final = item.get("id")
|
||||
if isinstance(item_id, str):
|
||||
decoded: Final = ResponsesAPIRequestUtils._decode_encrypted_item_id(item_id)
|
||||
if decoded:
|
||||
return decoded.get("model_id")
|
||||
encrypted_content: Final = item.get("encrypted_content")
|
||||
if isinstance(encrypted_content, str):
|
||||
model_id, _ = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(encrypted_content)
|
||||
return model_id or None
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def strip_encrypted_reasoning_from_input(
|
||||
request_input: object, *, originating_model_ids: frozenset[str] | None = None
|
||||
) -> None:
|
||||
"""Drop reasoning items the routed deployment cannot decrypt, keeping their readable summary.
|
||||
|
||||
Mutates ``request_input`` in place: the router's fallback snapshot shares this
|
||||
|
|
@ -565,7 +582,13 @@ class ResponsesAPIRequestUtils:
|
|||
if not isinstance(request_input, list):
|
||||
return
|
||||
items: Final = cast(list[object], request_input) # cast-ok: untyped client json
|
||||
stripped: Final = tuple(ResponsesAPIRequestUtils._without_encrypted_reasoning(item) for item in items)
|
||||
stripped: Final = tuple(
|
||||
ResponsesAPIRequestUtils._without_encrypted_reasoning(item)
|
||||
if originating_model_ids is None
|
||||
or ResponsesAPIRequestUtils.get_encrypted_content_model_id(item) in originating_model_ids
|
||||
else item
|
||||
for item in items
|
||||
)
|
||||
items[:] = (item for item in stripped if item is not None) # rebind-ok: list shared with fallback snapshot
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -129,25 +129,15 @@ class EncryptedContentAffinityCheck(CustomLogger):
|
|||
if not isinstance(request_input, list):
|
||||
return None
|
||||
|
||||
for item in request_input:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
|
||||
# First, try to decode from item ID (if present)
|
||||
item_id = item.get("id")
|
||||
if item_id and isinstance(item_id, str):
|
||||
decoded = ResponsesAPIRequestUtils._decode_encrypted_item_id(item_id)
|
||||
if decoded:
|
||||
return decoded.get("model_id")
|
||||
|
||||
# If no encoded ID, check if encrypted_content itself is wrapped
|
||||
encrypted_content = item.get("encrypted_content")
|
||||
if encrypted_content and isinstance(encrypted_content, str):
|
||||
model_id = EncryptedContentAffinityCheck._model_id_from_wrapped_encrypted_content(encrypted_content)
|
||||
if model_id:
|
||||
return model_id
|
||||
|
||||
return None
|
||||
items: Final = cast(list[object], request_input) # cast-ok: narrowed by isinstance
|
||||
return next(
|
||||
(
|
||||
model_id
|
||||
for item in items
|
||||
if (model_id := ResponsesAPIRequestUtils.get_encrypted_content_model_id(item)) is not None
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _anthropic_content_blocks(messages: object) -> Iterator[Mapping[str, object]]:
|
||||
|
|
@ -216,9 +206,10 @@ class EncryptedContentAffinityCheck(CustomLogger):
|
|||
@staticmethod
|
||||
def _encryption_boundary_key(
|
||||
litellm_params: object,
|
||||
) -> tuple[object, object] | None:
|
||||
) -> tuple[object, ...] | None:
|
||||
"""
|
||||
``(api_base, api_key)`` identifies an upstream encryption boundary.
|
||||
Bedrock OpenAI reasoning also requires the same underlying model.
|
||||
The values are resolved from the deployment and its named credential
|
||||
without modifying the deployment.
|
||||
|
||||
|
|
@ -241,18 +232,23 @@ class EncryptedContentAffinityCheck(CustomLogger):
|
|||
if isinstance(credential_name, str) and credential_name
|
||||
else None
|
||||
)
|
||||
effective_api_base: Final = (
|
||||
effective_api_base: Final[object] = (
|
||||
credential_values.get("api_base")
|
||||
if credential_values is not None and "api_base" in credential_values
|
||||
else api_base
|
||||
)
|
||||
effective_api_key: Final = (
|
||||
effective_api_key: Final[object] = (
|
||||
credential_values.get("api_key")
|
||||
if credential_values is not None and "api_key" in credential_values
|
||||
else api_key
|
||||
)
|
||||
if not effective_api_base or not effective_api_key:
|
||||
return None
|
||||
from litellm.llms.bedrock.common_utils import get_bedrock_openai_model
|
||||
|
||||
bedrock_model: Final = get_bedrock_openai_model(getter("model"))
|
||||
if bedrock_model is not None:
|
||||
return (effective_api_base, effective_api_key, bedrock_model)
|
||||
return (effective_api_base, effective_api_key)
|
||||
|
||||
def _find_deployments_on_same_encryption_boundary(
|
||||
|
|
@ -290,7 +286,7 @@ class EncryptedContentAffinityCheck(CustomLogger):
|
|||
model: str,
|
||||
healthy_deployments: list,
|
||||
messages: list[AllMessageValues] | None,
|
||||
request_kwargs: dict | None = None,
|
||||
request_kwargs: dict[str, object] | None = None,
|
||||
parent_otel_span: Span | None = None,
|
||||
) -> list[dict]:
|
||||
"""
|
||||
|
|
@ -310,7 +306,9 @@ class EncryptedContentAffinityCheck(CustomLogger):
|
|||
retry after the deployment is eligible again.
|
||||
"""
|
||||
request_kwargs = request_kwargs or {}
|
||||
typed_healthy_deployments: Final = cast(list[dict], healthy_deployments)
|
||||
typed_healthy_deployments: Final = cast(
|
||||
list[dict[str, object]], healthy_deployments
|
||||
) # cast-ok: router deployment dictionaries
|
||||
if not self._is_enabled_for_model_group(model):
|
||||
return typed_healthy_deployments
|
||||
|
||||
|
|
@ -321,10 +319,20 @@ class EncryptedContentAffinityCheck(CustomLogger):
|
|||
# completions / embeddings, which breaks tag-based routing because
|
||||
# _get_metadata_variable_name_from_kwargs would pick "litellm_metadata"
|
||||
# over "metadata" where tags are actually stored.
|
||||
if "litellm_metadata" in request_kwargs:
|
||||
request_kwargs["litellm_metadata"]["encrypted_content_affinity_enabled"] = True
|
||||
metadata: Final = request_kwargs.get("litellm_metadata")
|
||||
if isinstance(metadata, dict):
|
||||
metadata["encrypted_content_affinity_enabled"] = True
|
||||
|
||||
request_input: Final = request_kwargs.get("input")
|
||||
if self.router is not None:
|
||||
from litellm.llms.bedrock.reasoning_affinity import strip_incompatible_bedrock_openai_reasoning
|
||||
|
||||
strip_incompatible_bedrock_openai_reasoning(
|
||||
request_input,
|
||||
typed_healthy_deployments,
|
||||
lambda: self._routed_group_candidate_model_ids(request_kwargs, model),
|
||||
lambda model_id: self.router.get_deployment(model_id=model_id),
|
||||
)
|
||||
anthropic_messages: Final = messages or request_kwargs.get("messages")
|
||||
model_id: Final = self._extract_model_id_from_input(
|
||||
request_input
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from importlib import import_module
|
||||
import base64
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -13,6 +14,33 @@ from litellm.types.utils import Usage
|
|||
|
||||
|
||||
class TestResponsesAPIRequestUtils:
|
||||
def test_selective_reasoning_removal_preserves_other_origins_and_compaction(self) -> None:
|
||||
foreign: Final = {
|
||||
"type": "reasoning",
|
||||
"id": ResponsesAPIRequestUtils._build_encrypted_item_id("sol", "rs_sol"),
|
||||
"encrypted_content": "opaque-sol",
|
||||
"summary": [{"type": "summary_text", "text": "Investigated the failing test"}],
|
||||
}
|
||||
native: Final = {
|
||||
"type": "reasoning",
|
||||
"encrypted_content": ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("opaque-luna", "luna"),
|
||||
"summary": [],
|
||||
}
|
||||
compaction: Final = {
|
||||
"type": "compaction",
|
||||
"encrypted_content": ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("opaque-state", "sol"),
|
||||
}
|
||||
untagged: Final = {"type": "reasoning", "encrypted_content": "opaque-unknown", "summary": []}
|
||||
history: Final = [foreign, native, compaction, untagged]
|
||||
|
||||
ResponsesAPIRequestUtils.strip_encrypted_reasoning_from_input(history, originating_model_ids=frozenset({"sol"}))
|
||||
|
||||
assert history == [{"type": "reasoning", "summary": foreign["summary"]}, native, compaction, untagged]
|
||||
assert foreign["encrypted_content"] == "opaque-sol"
|
||||
assert history[1] is native
|
||||
assert history[2] is compaction
|
||||
assert history[3] is untagged
|
||||
|
||||
def test_get_optional_params_responses_api(self):
|
||||
"""Test that optional parameters are correctly processed for responses API"""
|
||||
# Setup
|
||||
|
|
|
|||
|
|
@ -16,6 +16,8 @@ The mechanism works without any cache and supports two encoding strategies:
|
|||
"""
|
||||
|
||||
import time
|
||||
from copy import deepcopy
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -2120,3 +2122,155 @@ async def test_affinity_honors_router_candidate_ids_for_team_and_pattern_routes(
|
|||
|
||||
assert request_kwargs["input"][1].get("encrypted_content")
|
||||
mock_router.get_candidate_model_ids_for_route.assert_called_once_with(model="team-public-model", team_id="teamA")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def bedrock_switch_router() -> litellm.Router:
|
||||
return litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": name,
|
||||
"model_info": {"id": name},
|
||||
"litellm_params": {
|
||||
"model": f"bedrock_mantle/openai.gpt-5.6-{upstream}",
|
||||
"api_base": "https://bedrock-mantle.us-east-2.api.aws",
|
||||
"api_key": "test-shared-key",
|
||||
},
|
||||
}
|
||||
for name, upstream in (("sol", "sol"), ("luna", "luna"), ("sol-alias", "sol"))
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _bedrock_reasoning_item(origin: str, include_id: bool) -> dict[str, object]:
|
||||
return {
|
||||
"type": "reasoning",
|
||||
"summary": [],
|
||||
"encrypted_content": ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(
|
||||
f"opaque-{origin}", origin
|
||||
),
|
||||
**({"id": ResponsesAPIRequestUtils._build_encrypted_item_id(origin, f"rs_{origin}")} if include_id else {}),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("include_id", [True, False])
|
||||
@pytest.mark.parametrize(("source", "target"), [("sol", "luna"), ("luna", "sol"), ("sol", "sol"), ("sol", "sol-alias")])
|
||||
async def test_bedrock_model_switch_only_removes_foreign_reasoning(
|
||||
bedrock_switch_router: litellm.Router, source: str, target: str, include_id: bool
|
||||
) -> None:
|
||||
check: Final = EncryptedContentAffinityCheck(router=bedrock_switch_router)
|
||||
pool: Final = [d for d in bedrock_switch_router.model_list if d["model_name"] == target]
|
||||
reasoning: Final = _bedrock_reasoning_item(source, include_id)
|
||||
request: Final = {
|
||||
"litellm_metadata": {},
|
||||
"reasoning": {"effort": "high", "context": "all_turns", "summary": "auto"},
|
||||
"input": [
|
||||
{"role": "user", "content": "Inspect the bug"},
|
||||
reasoning,
|
||||
{"role": "assistant", "phase": "final_answer", "content": "Found it"},
|
||||
{"role": "user", "content": "Fix it"},
|
||||
],
|
||||
}
|
||||
original: Final = deepcopy(request)
|
||||
|
||||
result: Final = await check.async_filter_deployments(target, pool, None, request)
|
||||
|
||||
assert result == pool
|
||||
assert request["reasoning"] == original["reasoning"]
|
||||
expected: Final = (
|
||||
original["input"]
|
||||
if source == target or target == "sol-alias"
|
||||
else [item for item in original["input"] if item.get("type") != "reasoning"]
|
||||
)
|
||||
assert request["input"] == expected
|
||||
assert request["litellm_metadata"]["encrypted_content_affinity_enabled"] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("target", ["sol", "luna"])
|
||||
async def test_bedrock_mixed_history_keeps_selected_models_reasoning(
|
||||
bedrock_switch_router: litellm.Router, target: str
|
||||
) -> None:
|
||||
check: Final = EncryptedContentAffinityCheck(router=bedrock_switch_router)
|
||||
pool: Final = [d for d in bedrock_switch_router.model_list if d["model_name"] == target]
|
||||
sol: Final = _bedrock_reasoning_item("sol", False)
|
||||
luna: Final = _bedrock_reasoning_item("luna", False)
|
||||
tool_call: Final = {"type": "function_call", "call_id": "call_1", "name": "inspect", "arguments": "{}"}
|
||||
tool_result: Final = {"type": "function_call_output", "call_id": "call_1", "output": "inspected"}
|
||||
history: Final = [
|
||||
{"role": "user", "content": "Inspect"}, sol, tool_call, tool_result,
|
||||
{"role": "assistant", "phase": "final_answer", "content": "Done"},
|
||||
{"role": "user", "content": "Fix"}, luna,
|
||||
{"role": "user", "content": "Continue"},
|
||||
]
|
||||
request: Final = {"litellm_metadata": {}, "input": deepcopy(history)}
|
||||
shared_input: Final = request["input"]
|
||||
|
||||
result: Final = await check.async_filter_deployments(target, pool, None, request)
|
||||
|
||||
assert result == pool
|
||||
assert request["input"] is shared_input
|
||||
assert shared_input == [item for item in history if item != (sol if target == "luna" else luna)]
|
||||
|
||||
|
||||
async def test_bedrock_switch_filter_respects_opt_in(bedrock_switch_router: litellm.Router) -> None:
|
||||
check: Final = EncryptedContentAffinityCheck(router=bedrock_switch_router, enable_global_affinity=False)
|
||||
pool: Final = [d for d in bedrock_switch_router.model_list if d["model_name"] == "luna"]
|
||||
request: Final = {"litellm_metadata": {}, "input": [_bedrock_reasoning_item("sol", False)]}
|
||||
original: Final = deepcopy(request)
|
||||
|
||||
assert await check.async_filter_deployments("luna", pool, None, request) == pool
|
||||
assert request == original
|
||||
|
||||
|
||||
@pytest.mark.parametrize("untrusted_origin", ["unknown-deployment", "sol"])
|
||||
async def test_bedrock_unknown_marker_uses_same_safe_path_as_untrusted_foreign_marker(
|
||||
bedrock_switch_router: litellm.Router,
|
||||
untrusted_origin: str,
|
||||
) -> None:
|
||||
check: Final = EncryptedContentAffinityCheck(router=bedrock_switch_router)
|
||||
pool: Final = [d for d in bedrock_switch_router.model_list if d["model_name"] == "luna"]
|
||||
request: Final = {
|
||||
"litellm_metadata": {},
|
||||
"input": [
|
||||
_bedrock_reasoning_item(untrusted_origin, False),
|
||||
_bedrock_reasoning_item("luna", False),
|
||||
],
|
||||
}
|
||||
|
||||
assert await check.async_filter_deployments("luna", pool, None, request) == pool
|
||||
assert request["input"] == [_bedrock_reasoning_item("luna", False)]
|
||||
|
||||
|
||||
async def test_bedrock_cooldown_does_not_become_a_model_switch() -> None:
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "auto",
|
||||
"model_info": {"id": name},
|
||||
"litellm_params": {
|
||||
"model": f"bedrock_mantle/openai.gpt-5.6-{name}",
|
||||
"api_base": "https://bedrock-mantle.us-east-2.api.aws",
|
||||
"api_key": "test-shared-key",
|
||||
},
|
||||
}
|
||||
for name in ("sol", "luna")
|
||||
]
|
||||
)
|
||||
check: Final = EncryptedContentAffinityCheck(router=router)
|
||||
pool: Final = [d for d in router.model_list if d["model_info"]["id"] == "luna"]
|
||||
request: Final = {"input": [_bedrock_reasoning_item("sol", False)]}
|
||||
original: Final = deepcopy(request)
|
||||
|
||||
with pytest.raises(litellm.ServiceUnavailableError):
|
||||
await check.async_filter_deployments("auto", pool, None, request)
|
||||
assert request == original
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prefix", ["bedrock_mantle/", "bedrock/us.", "openai/us."])
|
||||
def test_bedrock_shared_credentials_do_not_make_different_models_compatible(prefix: str) -> None:
|
||||
shared: Final = {"api_base": "https://gateway.example/openai/v1", "api_key": "test-shared-key"}
|
||||
sol: Final = {**shared, "model": f"{prefix}openai.gpt-5.6-sol"}
|
||||
luna: Final = {**shared, "model": f"{prefix}openai.gpt-5.6-luna"}
|
||||
|
||||
assert EncryptedContentAffinityCheck._encryption_boundary_key(sol) != EncryptedContentAffinityCheck._encryption_boundary_key(luna)
|
||||
assert EncryptedContentAffinityCheck._encryption_boundary_key(sol) == EncryptedContentAffinityCheck._encryption_boundary_key(dict(sol))
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue