From 99e4ba34c578c1cc1c41b11145588c9c12a91bbe Mon Sep 17 00:00:00 2001 From: aiedwardyi <41576951+aiedwardyi@users.noreply.github.com> Date: Mon, 24 Aug 2026 01:33:28 +0900 Subject: [PATCH 01/21] fix: scope Bedrock guardrail bearer keys --- .../guardrail_hooks/bedrock_guardrails.py | 24 +++++++-- .../test_bedrock_guardrails.py | 49 +++++++++++++++++++ .../test_bedrock_invoke_guardrail_checks.py | 42 ++++++++++++++++ 3 files changed, 111 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index c70a2ee8a74..06772f0ad07 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -669,6 +669,24 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): # logic becomes shared across providers. #### CALL HOOKS - proxy only #### + @staticmethod + def _get_bedrock_api_key(request_data: Mapping[str, object] | None) -> str | None: + if not request_data: + return None + + top_level_provider: Final[object | None] = request_data.get("custom_llm_provider") + request_provider: Final[str | None] = ( + top_level_provider if isinstance(top_level_provider, str) else None + ) + model: Final[object | None] = request_data.get("model") + model_provider: Final = model.partition("/")[0] if isinstance(model, str) else None + custom_llm_provider: Final = request_provider or model_provider + if custom_llm_provider not in ("bedrock", "bedrock_converse"): + return None + + api_key: Final[object | None] = request_data.get("api_key") + return api_key if isinstance(api_key, str) else None + def _load_credentials( self, ): @@ -841,7 +859,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): bedrock_request_data: Final[dict] = dict( self.convert_to_bedrock_format(source=source, messages=messages, response=response) ) - api_key: str | None = None + api_key: Final = self._get_bedrock_api_key(request_data) if request_data: dynamic_request_body_params = self.get_guardrail_dynamic_request_body_params(request_data=request_data) bedrock_request_data.update( @@ -851,8 +869,6 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): if key not in _BEDROCK_DYNAMIC_BODY_DENYLIST } ) - if request_data.get("api_key") is not None: - api_key = request_data["api_key"] event_type: Final = ( logging_event_type @@ -1828,7 +1844,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): credentials, aws_region_name = self._load_credentials() body: Final[dict[str, Any]] = {"messages": checks_messages, "checks": self.checks} - api_key: Final[str | None] = request_data.get("api_key") if request_data else None + api_key: Final = self._get_bedrock_api_key(request_data) prepared_request: Final = self._prepare_request( credentials=credentials, diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index dd339d4e51f..c111072a06a 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -1114,6 +1114,55 @@ async def test_make_apply_guardrail_request_skips_scan_without_credentials(): mock_post.assert_not_called() +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("custom_llm_provider", "expected_auth_prefix"), + [("nvidia_nim", "AWS4-HMAC-SHA256"), ("bedrock", "Bearer bedrock-key")], +) +async def test_make_apply_guardrail_request_scopes_api_key_to_bedrock_provider( + custom_llm_provider, expected_auth_prefix, monkeypatch +): + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + guardrail = BedrockGuardrail( + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + ) + request_data = { + "model": f"{custom_llm_provider}/test-model", + "api_key": "bedrock-key", + } + mock_credentials = MagicMock() + mock_credentials.access_key = "test-access-key" + mock_credentials.secret_key = "test-secret-key" + mock_credentials.token = None + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"action": "NONE", "assessments": []} + + with ( + patch.object( + guardrail, + "_load_credentials", + return_value=(mock_credentials, "us-east-1"), + ), + patch.object( + guardrail.async_handler, + "post", + new=AsyncMock(return_value=mock_response), + ) as mock_post, + ): + result = await guardrail.make_bedrock_api_request( + source="INPUT", + messages=[{"role": "user", "content": "hello"}], + request_data=request_data, + ) + + assert result["action"] == "NONE" + assert mock_post.await_args.kwargs["headers"]["Authorization"].startswith( + expected_auth_prefix + ) + + @pytest.mark.asyncio async def test_bedrock_apply_guardrail_response_uses_OUTPUT_source(): """input_type='response' must call Bedrock with source=OUTPUT and assistant content. diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_invoke_guardrail_checks.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_invoke_guardrail_checks.py index d842a1ee5f9..634e2254187 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_invoke_guardrail_checks.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_invoke_guardrail_checks.py @@ -395,6 +395,48 @@ async def test_request_uses_checks_path_and_body(): ] +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("custom_llm_provider", "expected_auth_prefix"), + [("nvidia_nim", "AWS4-HMAC-SHA256"), ("bedrock", "Bearer bedrock-key")], +) +async def test_request_scopes_api_key_to_bedrock_provider( + custom_llm_provider, expected_auth_prefix, monkeypatch +): + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + g = BedrockGuardrail(checks=CONTENT_FILTER_CHECKS) + request_data = { + "model": f"{custom_llm_provider}/test-model", + "api_key": "bedrock-key", + } + mock_credentials = MagicMock() + mock_credentials.access_key = "test-access-key" + mock_credentials.secret_key = "test-secret-key" + mock_credentials.token = None + mock_post = AsyncMock( + return_value=_mock_http_response(200, {"results": {}}) + ) + + with ( + patch.object( + g, + "_load_credentials", + return_value=(mock_credentials, "us-east-1"), + ), + patch.object(g.async_handler, "post", new=mock_post), + ): + result = await g.make_bedrock_api_request( + source="INPUT", + messages=[{"role": "user", "content": "hello"}], + request_data=request_data, + ) + + assert result == BedrockGuardrailResponse() + assert mock_post.await_args.kwargs["headers"]["Authorization"].startswith( + expected_auth_prefix + ) + + @pytest.mark.asyncio async def test_empty_messages_passes_without_api_call(): g = BedrockGuardrail(checks=CONTENT_FILTER_CHECKS) From 31a529aee61ddd5c03192143b0b5cc5d0dfff2f5 Mon Sep 17 00:00:00 2001 From: aiedwardyi <41576951+aiedwardyi@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:01:02 +0900 Subject: [PATCH 02/21] fix: resolve native Bedrock guardrail providers --- .../guardrail_hooks/bedrock_guardrails.py | 14 ++++++++++++-- .../guardrail_hooks/test_bedrock_guardrails.py | 14 +++++++++----- .../test_bedrock_invoke_guardrail_checks.py | 14 +++++++++----- .../proxy/guardrails/test_guardrail_endpoints.py | 5 ++++- 4 files changed, 34 insertions(+), 13 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 06772f0ad07..8ade4a011f7 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -669,6 +669,14 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): # logic becomes shared across providers. #### CALL HOOKS - proxy only #### + @staticmethod + def _resolve_model_provider(model: str) -> str | None: + try: + _, custom_llm_provider, _, _ = litellm.get_llm_provider(model=model) + return custom_llm_provider + except Exception: + return model.partition("/")[0] + @staticmethod def _get_bedrock_api_key(request_data: Mapping[str, object] | None) -> str | None: if not request_data: @@ -679,8 +687,10 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): top_level_provider if isinstance(top_level_provider, str) else None ) model: Final[object | None] = request_data.get("model") - model_provider: Final = model.partition("/")[0] if isinstance(model, str) else None - custom_llm_provider: Final = request_provider or model_provider + model_provider: Final[str | None] = ( + BedrockGuardrail._resolve_model_provider(model) if isinstance(model, str) else None + ) + custom_llm_provider: Final[str | None] = request_provider or model_provider if custom_llm_provider not in ("bedrock", "bedrock_converse"): return None diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index c111072a06a..a1dbbaa5953 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -1116,19 +1116,23 @@ async def test_make_apply_guardrail_request_skips_scan_without_credentials(): @pytest.mark.asyncio @pytest.mark.parametrize( - ("custom_llm_provider", "expected_auth_prefix"), - [("nvidia_nim", "AWS4-HMAC-SHA256"), ("bedrock", "Bearer bedrock-key")], + ("model", "expected_auth_prefix"), + [ + ("nvidia_nim/test-model", "AWS4-HMAC-SHA256"), + ("bedrock/test-model", "Bearer bedrock-key"), + ("amazon.nova-lite-v1:0", "Bearer bedrock-key"), + ], ) async def test_make_apply_guardrail_request_scopes_api_key_to_bedrock_provider( - custom_llm_provider, expected_auth_prefix, monkeypatch -): + model: str, expected_auth_prefix: str, monkeypatch: pytest.MonkeyPatch +) -> None: monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) guardrail = BedrockGuardrail( guardrailIdentifier="test-guardrail", guardrailVersion="DRAFT", ) request_data = { - "model": f"{custom_llm_provider}/test-model", + "model": model, "api_key": "bedrock-key", } mock_credentials = MagicMock() diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_invoke_guardrail_checks.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_invoke_guardrail_checks.py index 634e2254187..a50e4bcffa0 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_invoke_guardrail_checks.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_invoke_guardrail_checks.py @@ -397,16 +397,20 @@ async def test_request_uses_checks_path_and_body(): @pytest.mark.asyncio @pytest.mark.parametrize( - ("custom_llm_provider", "expected_auth_prefix"), - [("nvidia_nim", "AWS4-HMAC-SHA256"), ("bedrock", "Bearer bedrock-key")], + ("model", "expected_auth_prefix"), + [ + ("nvidia_nim/test-model", "AWS4-HMAC-SHA256"), + ("bedrock/test-model", "Bearer bedrock-key"), + ("amazon.nova-lite-v1:0", "Bearer bedrock-key"), + ], ) async def test_request_scopes_api_key_to_bedrock_provider( - custom_llm_provider, expected_auth_prefix, monkeypatch -): + model: str, expected_auth_prefix: str, monkeypatch: pytest.MonkeyPatch +) -> None: monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) g = BedrockGuardrail(checks=CONTENT_FILTER_CHECKS) request_data = { - "model": f"{custom_llm_provider}/test-model", + "model": model, "api_key": "bedrock-key", } mock_credentials = MagicMock() diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py b/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py index 45f5afef1bc..ab7a8f8acac 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py @@ -889,7 +889,10 @@ async def test_bedrock_guardrail_make_api_request_passes_api_key(): mock_response.status_code = 200 mock_response.json.return_value = {"action": "NONE", "outputs": []} - test_request_data = {"api_key": "test-api-key-789"} + test_request_data = { + "model": "bedrock/test-model", + "api_key": "test-api-key-789", + } with ( patch.object( From 53c90c351b26b73ec8b4f86b935cf77c7df0234b Mon Sep 17 00:00:00 2001 From: aiedwardyi <41576951+aiedwardyi@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:13:56 +0900 Subject: [PATCH 03/21] fix: scope Bedrock bearer keys --- .../guardrail_hooks/bedrock_guardrails.py | 67 ++++++++++++++++--- .../test_bedrock_guardrails.py | 40 +++++++++++ 2 files changed, 97 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 8ade4a011f7..2c05bfce601 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -677,25 +677,72 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): except Exception: return model.partition("/")[0] + @staticmethod + def _router_allows_bedrock(request_data: Mapping[str, object]) -> bool | None: + model: Final[object | None] = request_data.get("model") + if not isinstance(model, str): + return False + + try: + from litellm.proxy.proxy_server import llm_router + except ImportError: + return None + if llm_router is None: + return None + + metadata: Final = request_data.get("metadata") + litellm_metadata: Final = request_data.get("litellm_metadata") + team_id: Final[object | None] = ( + metadata.get("user_api_key_team_id") + if isinstance(metadata, Mapping) + else litellm_metadata.get("user_api_key_team_id") + if isinstance(litellm_metadata, Mapping) + else None + ) + try: + deployments: Final = llm_router.get_model_list( + model_name=model, + team_id=team_id if isinstance(team_id, str) else None, + ) or [] + except Exception: + return False + if not deployments: + return False + + providers: list[str] = [] + for deployment in deployments: + params: object = deployment.get("litellm_params") if isinstance(deployment, Mapping) else None + provider: object = params.get("custom_llm_provider") if isinstance(params, Mapping) else None + if not isinstance(provider, str) and isinstance(params, Mapping): + deployment_model: Final[object | None] = params.get("model") + provider = ( + BedrockGuardrail._resolve_model_provider(deployment_model) + if isinstance(deployment_model, str) + else None + ) + if not isinstance(provider, str): + return False + providers.append(provider) + return bool(providers) and all(provider in ("bedrock", "bedrock_converse") for provider in providers) + @staticmethod def _get_bedrock_api_key(request_data: Mapping[str, object] | None) -> str | None: if not request_data: return None - top_level_provider: Final[object | None] = request_data.get("custom_llm_provider") - request_provider: Final[str | None] = ( - top_level_provider if isinstance(top_level_provider, str) else None - ) + api_key: Final[object | None] = request_data.get("api_key") + if not isinstance(api_key, str): + return None + + router_allows_bedrock: Final = BedrockGuardrail._router_allows_bedrock(request_data) + if router_allows_bedrock is not None: + return api_key if router_allows_bedrock else None + model: Final[object | None] = request_data.get("model") model_provider: Final[str | None] = ( BedrockGuardrail._resolve_model_provider(model) if isinstance(model, str) else None ) - custom_llm_provider: Final[str | None] = request_provider or model_provider - if custom_llm_provider not in ("bedrock", "bedrock_converse"): - return None - - api_key: Final[object | None] = request_data.get("api_key") - return api_key if isinstance(api_key, str) else None + return api_key if model_provider in ("bedrock", "bedrock_converse") else None def _load_credentials( self, diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index a1dbbaa5953..bf8b48625e8 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -1167,6 +1167,46 @@ async def test_make_apply_guardrail_request_scopes_api_key_to_bedrock_provider( ) +def test_bedrock_api_key_rejects_caller_provider_spoofing(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy import proxy_server + + router = MagicMock() + router.get_model_list.return_value = [ + {"litellm_params": {"model": "gpt-4o", "custom_llm_provider": "openai"}} + ] + monkeypatch.setattr(proxy_server, "llm_router", router) + + assert ( + BedrockGuardrail._get_bedrock_api_key( + { + "model": "shared-alias", + "custom_llm_provider": "bedrock", + "api_key": "bedrock-key", + } + ) + is None + ) + + +def test_bedrock_api_key_accepts_alias_with_only_bedrock_deployments( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy import proxy_server + + router = MagicMock() + router.get_model_list.return_value = [ + {"litellm_params": {"model": "amazon.nova-lite-v1:0", "custom_llm_provider": "bedrock"}} + ] + monkeypatch.setattr(proxy_server, "llm_router", router) + + assert ( + BedrockGuardrail._get_bedrock_api_key( + {"model": "bedrock-alias", "api_key": "bedrock-key"} + ) + == "bedrock-key" + ) + + @pytest.mark.asyncio async def test_bedrock_apply_guardrail_response_uses_OUTPUT_source(): """input_type='response' must call Bedrock with source=OUTPUT and assistant content. From 81c21fcb5ba4473b2f42658811365c15afc2e1e4 Mon Sep 17 00:00:00 2001 From: aiedwardyi <41576951+aiedwardyi@users.noreply.github.com> Date: Mon, 24 Aug 2026 17:26:33 +0900 Subject: [PATCH 04/21] fix: filter inactive Bedrock deployments --- .../guardrail_hooks/bedrock_guardrails.py | 33 +++++++++++++------ .../test_bedrock_guardrails.py | 28 ++++++++++++++++ 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 2c05bfce601..ab6fafbba9d 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -30,7 +30,10 @@ from litellm.caching import DualCache from litellm.constants import BEDROCK_APPLY_GUARDRAIL_CHUNK_BUDGET_CHARS from litellm.exceptions import ModifyResponseException from litellm.integrations.custom_guardrail import CustomGuardrail -from litellm.litellm_core_utils.core_helpers import redact_nested_match_and_regex_keys +from litellm.litellm_core_utils.core_helpers import ( + get_metadata_variable_name_from_kwargs, + redact_nested_match_and_regex_keys, +) from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import bedrock_guardrail_cost from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicMessagesHandler from litellm.llms.base_llm.guardrail_translation.utils import ( @@ -690,14 +693,10 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): if llm_router is None: return None - metadata: Final = request_data.get("metadata") - litellm_metadata: Final = request_data.get("litellm_metadata") + metadata_key: Final = get_metadata_variable_name_from_kwargs(dict(request_data)) + metadata: Final = request_data.get(metadata_key) team_id: Final[object | None] = ( - metadata.get("user_api_key_team_id") - if isinstance(metadata, Mapping) - else litellm_metadata.get("user_api_key_team_id") - if isinstance(litellm_metadata, Mapping) - else None + metadata.get("user_api_key_team_id") if isinstance(metadata, Mapping) else None ) try: deployments: Final = llm_router.get_model_list( @@ -706,11 +705,25 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): ) or [] except Exception: return False - if not deployments: + active_deployments = [] + for deployment in deployments: + model_info = ( + deployment.get("model_info") + if isinstance(deployment, Mapping) + else getattr(deployment, "model_info", None) + ) + blocked = ( + model_info.get("blocked") + if isinstance(model_info, Mapping) + else getattr(model_info, "blocked", None) + ) + if blocked is not True: + active_deployments.append(deployment) + if not active_deployments: return False providers: list[str] = [] - for deployment in deployments: + for deployment in active_deployments: params: object = deployment.get("litellm_params") if isinstance(deployment, Mapping) else None provider: object = params.get("custom_llm_provider") if isinstance(params, Mapping) else None if not isinstance(provider, str) and isinstance(params, Mapping): diff --git a/tests/guardrails_tests/test_bedrock_guardrails.py b/tests/guardrails_tests/test_bedrock_guardrails.py index 43d088268eb..9390b3275a8 100644 --- a/tests/guardrails_tests/test_bedrock_guardrails.py +++ b/tests/guardrails_tests/test_bedrock_guardrails.py @@ -11,6 +11,34 @@ from litellm.caching import DualCache from unittest.mock import MagicMock, AsyncMock, patch +def test_bedrock_guardrail_uses_active_metadata_bucket_for_team_id(): + router = MagicMock() + router.get_model_list.return_value = [ + {"litellm_params": {"custom_llm_provider": "bedrock"}, "model_info": {}} + ] + request_data = { + "model": "team-alias", + "metadata": {"user_api_key_team_id": "legacy-team"}, + "litellm_metadata": {"user_api_key_team_id": "active-team"}, + } + + with patch("litellm.proxy.proxy_server.llm_router", router): + assert BedrockGuardrail._router_allows_bedrock(request_data) is True + + router.get_model_list.assert_called_once_with(model_name="team-alias", team_id="active-team") + + +def test_bedrock_guardrail_ignores_blocked_deployments(): + router = MagicMock() + router.get_model_list.return_value = [ + {"litellm_params": {"custom_llm_provider": "openai"}, "model_info": {"blocked": True}}, + {"litellm_params": {"custom_llm_provider": "bedrock"}, "model_info": {}}, + ] + + with patch("litellm.proxy.proxy_server.llm_router", router): + assert BedrockGuardrail._router_allows_bedrock({"model": "alias"}) is True + + @pytest.mark.asyncio async def test_bedrock_guardrails_pii_masking(): # Create proper mock objects From 08cfef679192d8607a295e536064769c6ad3e432 Mon Sep 17 00:00:00 2001 From: aiedwardyi <41576951+aiedwardyi@users.noreply.github.com> Date: Mon, 24 Aug 2026 18:17:57 +0900 Subject: [PATCH 05/21] fix: scope Bedrock guardrail routing --- .../guardrail_hooks/bedrock_guardrails.py | 74 +++++++++++++++---- .../test_bedrock_guardrails.py | 49 ++++++++++++ 2 files changed, 107 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index ab6fafbba9d..0c31ad00200 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -31,7 +31,6 @@ from litellm.constants import BEDROCK_APPLY_GUARDRAIL_CHUNK_BUDGET_CHARS from litellm.exceptions import ModifyResponseException from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.core_helpers import ( - get_metadata_variable_name_from_kwargs, redact_nested_match_and_regex_keys, ) from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import bedrock_guardrail_cost @@ -693,16 +692,51 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): if llm_router is None: return None - metadata_key: Final = get_metadata_variable_name_from_kwargs(dict(request_data)) - metadata: Final = request_data.get(metadata_key) - team_id: Final[object | None] = ( - metadata.get("user_api_key_team_id") if isinstance(metadata, Mapping) else None + team_id: Final[object | None] = next( + ( + metadata.get("user_api_key_team_id") + for metadata in (request_data.get("litellm_metadata"), request_data.get("metadata")) + if isinstance(metadata, Mapping) and isinstance(metadata.get("user_api_key_team_id"), str) + ), + None, ) try: - deployments: Final = llm_router.get_model_list( - model_name=model, - team_id=team_id if isinstance(team_id, str) else None, - ) or [] + resolved_team_id: Final = team_id if isinstance(team_id, str) else None + listed_deployments: Final = ( + llm_router.get_model_list( + model_name=model, + team_id=resolved_team_id, + ) + or [] + ) + model_id_deployment: Final = ( + llm_router.get_deployment(model_id=model) + if not listed_deployments and llm_router.has_model_id(model) is True + else None + ) + model_id_deployment_row: Final = ( + model_id_deployment.model_dump(exclude_none=True) + if model_id_deployment is not None and hasattr(model_id_deployment, "model_dump") + else model_id_deployment + ) + candidate_deployments: Final = ( + [model_id_deployment_row] if model_id_deployment_row is not None else listed_deployments + ) + + filter_deployments: Final = getattr(llm_router, "_filter_deployments_by_model_access_groups", None) + filtered_deployments: Final = ( + filter_deployments( + model=model, + healthy_deployments=candidate_deployments, + request_kwargs=dict(request_data), + request_team_id=resolved_team_id, + ) + if callable(filter_deployments) and isinstance(candidate_deployments, list) + else None + ) + deployments: Final = ( + filtered_deployments if isinstance(filtered_deployments, list) else candidate_deployments + ) except Exception: return False active_deployments = [] @@ -713,9 +747,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): else getattr(deployment, "model_info", None) ) blocked = ( - model_info.get("blocked") - if isinstance(model_info, Mapping) - else getattr(model_info, "blocked", None) + model_info.get("blocked") if isinstance(model_info, Mapping) else getattr(model_info, "blocked", None) ) if blocked is not True: active_deployments.append(deployment) @@ -724,10 +756,20 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): providers: list[str] = [] for deployment in active_deployments: - params: object = deployment.get("litellm_params") if isinstance(deployment, Mapping) else None - provider: object = params.get("custom_llm_provider") if isinstance(params, Mapping) else None - if not isinstance(provider, str) and isinstance(params, Mapping): - deployment_model: Final[object | None] = params.get("model") + params: object = ( + deployment.get("litellm_params") + if isinstance(deployment, Mapping) + else getattr(deployment, "litellm_params", None) + ) + provider: object = ( + params.get("custom_llm_provider") + if isinstance(params, Mapping) + else getattr(params, "custom_llm_provider", None) + ) + if not isinstance(provider, str): + deployment_model: Final[object | None] = ( + params.get("model") if isinstance(params, Mapping) else getattr(params, "model", None) + ) provider = ( BedrockGuardrail._resolve_model_provider(deployment_model) if isinstance(deployment_model, str) diff --git a/tests/guardrails_tests/test_bedrock_guardrails.py b/tests/guardrails_tests/test_bedrock_guardrails.py index 9390b3275a8..7e670dfb366 100644 --- a/tests/guardrails_tests/test_bedrock_guardrails.py +++ b/tests/guardrails_tests/test_bedrock_guardrails.py @@ -28,6 +28,55 @@ def test_bedrock_guardrail_uses_active_metadata_bucket_for_team_id(): router.get_model_list.assert_called_once_with(model_name="team-alias", team_id="active-team") +def test_bedrock_guardrail_uses_proxy_team_when_alternate_metadata_is_empty(): + router = MagicMock() + router.get_model_list.return_value = [ + {"litellm_params": {"custom_llm_provider": "bedrock"}, "model_info": {}} + ] + request_data = { + "model": "team-alias", + "metadata": {"user_api_key_team_id": "proxy-team"}, + "litellm_metadata": {}, + } + + with patch("litellm.proxy.proxy_server.llm_router", router): + assert BedrockGuardrail._router_allows_bedrock(request_data) is True + + router.get_model_list.assert_called_once_with(model_name="team-alias", team_id="proxy-team") + + +def test_bedrock_guardrail_resolves_router_model_id(): + router = MagicMock() + router.get_model_list.return_value = [] + router.has_model_id.return_value = True + deployment = MagicMock() + deployment.model_dump.return_value = { + "litellm_params": {"custom_llm_provider": "bedrock"}, + "model_info": {}, + } + router.get_deployment.return_value = deployment + + with patch("litellm.proxy.proxy_server.llm_router", router): + assert BedrockGuardrail._router_allows_bedrock({"model": "deployment-id"}) is True + + router.get_deployment.assert_called_once_with(model_id="deployment-id") + deployment.model_dump.assert_called_once_with(exclude_none=True) + + +def test_bedrock_guardrail_filters_access_group_deployments(): + router = MagicMock() + router.get_model_list.return_value = [ + {"litellm_params": {"custom_llm_provider": "openai"}, "model_info": {}}, + {"litellm_params": {"custom_llm_provider": "bedrock"}, "model_info": {}}, + ] + router._filter_deployments_by_model_access_groups.return_value = [ + {"litellm_params": {"custom_llm_provider": "bedrock"}, "model_info": {}} + ] + + with patch("litellm.proxy.proxy_server.llm_router", router): + assert BedrockGuardrail._router_allows_bedrock({"model": "scoped-alias"}) is True + + def test_bedrock_guardrail_ignores_blocked_deployments(): router = MagicMock() router.get_model_list.return_value = [ From 34e041e9f7de92f35535af21ffe532bd9ca4e492 Mon Sep 17 00:00:00 2001 From: aiedwardyi <41576951+aiedwardyi@users.noreply.github.com> Date: Mon, 24 Aug 2026 22:47:46 +0900 Subject: [PATCH 06/21] fix: address Bedrock guardrail review feedback --- .../guardrail_hooks/bedrock_guardrails.py | 37 +++++- .../test_bedrock_guardrails.py | 78 ------------ .../test_bedrock_guardrails.py | 120 ++++++++++++++++++ 3 files changed, 150 insertions(+), 85 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 0c31ad00200..b0a075c89a0 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -667,16 +667,13 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): return merged_messages - # NOTE: Consider moving these helpers to CustomGuardrail when the filtering - # logic becomes shared across providers. - #### CALL HOOKS - proxy only #### @staticmethod def _resolve_model_provider(model: str) -> str | None: try: _, custom_llm_provider, _, _ = litellm.get_llm_provider(model=model) return custom_llm_provider - except Exception: + except Exception: # noqa: BLE001 # provider resolution has a safe prefix fallback return model.partition("/")[0] @staticmethod @@ -709,9 +706,29 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): ) or [] ) + model_group_aliases: Final = getattr(llm_router, "model_group_alias", None) + team_filtered_deployments: Final = ( + [ + deployment + for deployment in listed_deployments + if ( + ( + deployment.get("model_info") + if isinstance(deployment, Mapping) + else getattr(deployment, "model_info", None) + ) + or {} + ).get("team_id") + in (None, resolved_team_id) + ] + if resolved_team_id is not None + and isinstance(model_group_aliases, Mapping) + and model in model_group_aliases + else listed_deployments + ) model_id_deployment: Final = ( llm_router.get_deployment(model_id=model) - if not listed_deployments and llm_router.has_model_id(model) is True + if not team_filtered_deployments and llm_router.has_model_id(model) is True else None ) model_id_deployment_row: Final = ( @@ -720,7 +737,9 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): else model_id_deployment ) candidate_deployments: Final = ( - [model_id_deployment_row] if model_id_deployment_row is not None else listed_deployments + [model_id_deployment_row] + if model_id_deployment_row is not None + else team_filtered_deployments ) filter_deployments: Final = getattr(llm_router, "_filter_deployments_by_model_access_groups", None) @@ -737,7 +756,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): deployments: Final = ( filtered_deployments if isinstance(filtered_deployments, list) else candidate_deployments ) - except Exception: + except Exception: # noqa: BLE001 # optional router state must not break guardrail auth return False active_deployments = [] for deployment in deployments: @@ -793,6 +812,10 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): if router_allows_bedrock is not None: return api_key if router_allows_bedrock else None + explicit_provider: Final[object | None] = request_data.get("custom_llm_provider") + if isinstance(explicit_provider, str): + return api_key if explicit_provider in ("bedrock", "bedrock_converse") else None + model: Final[object | None] = request_data.get("model") model_provider: Final[str | None] = ( BedrockGuardrail._resolve_model_provider(model) if isinstance(model, str) else None diff --git a/tests/guardrails_tests/test_bedrock_guardrails.py b/tests/guardrails_tests/test_bedrock_guardrails.py index 7e670dfb366..4916d47d31b 100644 --- a/tests/guardrails_tests/test_bedrock_guardrails.py +++ b/tests/guardrails_tests/test_bedrock_guardrails.py @@ -10,84 +10,6 @@ from litellm.proxy._types import UserAPIKeyAuth from litellm.caching import DualCache from unittest.mock import MagicMock, AsyncMock, patch - -def test_bedrock_guardrail_uses_active_metadata_bucket_for_team_id(): - router = MagicMock() - router.get_model_list.return_value = [ - {"litellm_params": {"custom_llm_provider": "bedrock"}, "model_info": {}} - ] - request_data = { - "model": "team-alias", - "metadata": {"user_api_key_team_id": "legacy-team"}, - "litellm_metadata": {"user_api_key_team_id": "active-team"}, - } - - with patch("litellm.proxy.proxy_server.llm_router", router): - assert BedrockGuardrail._router_allows_bedrock(request_data) is True - - router.get_model_list.assert_called_once_with(model_name="team-alias", team_id="active-team") - - -def test_bedrock_guardrail_uses_proxy_team_when_alternate_metadata_is_empty(): - router = MagicMock() - router.get_model_list.return_value = [ - {"litellm_params": {"custom_llm_provider": "bedrock"}, "model_info": {}} - ] - request_data = { - "model": "team-alias", - "metadata": {"user_api_key_team_id": "proxy-team"}, - "litellm_metadata": {}, - } - - with patch("litellm.proxy.proxy_server.llm_router", router): - assert BedrockGuardrail._router_allows_bedrock(request_data) is True - - router.get_model_list.assert_called_once_with(model_name="team-alias", team_id="proxy-team") - - -def test_bedrock_guardrail_resolves_router_model_id(): - router = MagicMock() - router.get_model_list.return_value = [] - router.has_model_id.return_value = True - deployment = MagicMock() - deployment.model_dump.return_value = { - "litellm_params": {"custom_llm_provider": "bedrock"}, - "model_info": {}, - } - router.get_deployment.return_value = deployment - - with patch("litellm.proxy.proxy_server.llm_router", router): - assert BedrockGuardrail._router_allows_bedrock({"model": "deployment-id"}) is True - - router.get_deployment.assert_called_once_with(model_id="deployment-id") - deployment.model_dump.assert_called_once_with(exclude_none=True) - - -def test_bedrock_guardrail_filters_access_group_deployments(): - router = MagicMock() - router.get_model_list.return_value = [ - {"litellm_params": {"custom_llm_provider": "openai"}, "model_info": {}}, - {"litellm_params": {"custom_llm_provider": "bedrock"}, "model_info": {}}, - ] - router._filter_deployments_by_model_access_groups.return_value = [ - {"litellm_params": {"custom_llm_provider": "bedrock"}, "model_info": {}} - ] - - with patch("litellm.proxy.proxy_server.llm_router", router): - assert BedrockGuardrail._router_allows_bedrock({"model": "scoped-alias"}) is True - - -def test_bedrock_guardrail_ignores_blocked_deployments(): - router = MagicMock() - router.get_model_list.return_value = [ - {"litellm_params": {"custom_llm_provider": "openai"}, "model_info": {"blocked": True}}, - {"litellm_params": {"custom_llm_provider": "bedrock"}, "model_info": {}}, - ] - - with patch("litellm.proxy.proxy_server.llm_router", router): - assert BedrockGuardrail._router_allows_bedrock({"model": "alias"}) is True - - @pytest.mark.asyncio async def test_bedrock_guardrails_pii_masking(): # Create proper mock objects diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index bf8b48625e8..13bbb15ef30 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -30,6 +30,126 @@ from litellm.types.proxy.guardrails.guardrail_hooks.bedrock_guardrails import ( from litellm.types.utils import CallTypes, ModelResponse +def test_bedrock_guardrail_uses_active_metadata_bucket_for_team_id(): + router = MagicMock() + router.get_model_list.return_value = [ + {"litellm_params": {"custom_llm_provider": "bedrock"}, "model_info": {}} + ] + request_data = { + "model": "team-alias", + "metadata": {"user_api_key_team_id": "legacy-team"}, + "litellm_metadata": {"user_api_key_team_id": "active-team"}, + } + + with patch("litellm.proxy.proxy_server.llm_router", router): + assert BedrockGuardrail._router_allows_bedrock(request_data) is True + + router.get_model_list.assert_called_once_with(model_name="team-alias", team_id="active-team") + + +def test_bedrock_guardrail_uses_proxy_team_when_alternate_metadata_is_empty(): + router = MagicMock() + router.get_model_list.return_value = [ + {"litellm_params": {"custom_llm_provider": "bedrock"}, "model_info": {}} + ] + request_data = { + "model": "team-alias", + "metadata": {"user_api_key_team_id": "proxy-team"}, + "litellm_metadata": {}, + } + + with patch("litellm.proxy.proxy_server.llm_router", router): + assert BedrockGuardrail._router_allows_bedrock(request_data) is True + + router.get_model_list.assert_called_once_with(model_name="team-alias", team_id="proxy-team") + + +def test_bedrock_guardrail_resolves_router_model_id(): + router = MagicMock() + router.get_model_list.return_value = [] + router.has_model_id.return_value = True + deployment = MagicMock() + deployment.model_dump.return_value = { + "litellm_params": {"custom_llm_provider": "bedrock"}, + "model_info": {}, + } + router.get_deployment.return_value = deployment + + with patch("litellm.proxy.proxy_server.llm_router", router): + assert BedrockGuardrail._router_allows_bedrock({"model": "deployment-id"}) is True + + router.get_deployment.assert_called_once_with(model_id="deployment-id") + deployment.model_dump.assert_called_once_with(exclude_none=True) + + +def test_bedrock_guardrail_filters_access_group_deployments(): + router = MagicMock() + router.get_model_list.return_value = [ + {"litellm_params": {"custom_llm_provider": "openai"}, "model_info": {}}, + {"litellm_params": {"custom_llm_provider": "bedrock"}, "model_info": {}}, + ] + router._filter_deployments_by_model_access_groups.return_value = [ + {"litellm_params": {"custom_llm_provider": "bedrock"}, "model_info": {}} + ] + + with patch("litellm.proxy.proxy_server.llm_router", router): + assert BedrockGuardrail._router_allows_bedrock({"model": "scoped-alias"}) is True + + +def test_bedrock_guardrail_ignores_blocked_deployments(): + router = MagicMock() + router.get_model_list.return_value = [ + {"litellm_params": {"custom_llm_provider": "openai"}, "model_info": {"blocked": True}}, + {"litellm_params": {"custom_llm_provider": "bedrock"}, "model_info": {}}, + ] + + with patch("litellm.proxy.proxy_server.llm_router", router): + assert BedrockGuardrail._router_allows_bedrock({"model": "alias"}) is True + + +def test_bedrock_guardrail_filters_alias_deployments_by_team(): + router = MagicMock() + router.model_group_alias = {"team-alias": "shared-group"} + router.get_model_list.return_value = [ + { + "litellm_params": {"custom_llm_provider": "openai"}, + "model_info": {"team_id": "other-team"}, + }, + { + "litellm_params": {"custom_llm_provider": "bedrock"}, + "model_info": {"team_id": "active-team"}, + }, + ] + + with patch("litellm.proxy.proxy_server.llm_router", router): + assert ( + BedrockGuardrail._router_allows_bedrock( + { + "model": "team-alias", + "litellm_metadata": {"user_api_key_team_id": "active-team"}, + } + ) + is True + ) + + +def test_bedrock_guardrail_honors_explicit_provider_without_router(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "llm_router", None) + + assert ( + BedrockGuardrail._get_bedrock_api_key( + { + "model": "bedrock/anthropic.claude-3-haiku", + "custom_llm_provider": "openai", + "api_key": "openai-key", + } + ) + is None + ) + + @pytest.mark.asyncio async def test__redact_pii_matches_function(): """Test the _redact_pii_matches function directly""" From aa3c21eab944362a2149206839762bd5e57acc63 Mon Sep 17 00:00:00 2001 From: aiedwardyi <41576951+aiedwardyi@users.noreply.github.com> Date: Mon, 24 Aug 2026 23:33:03 +0900 Subject: [PATCH 07/21] fix: close second Bedrock review round --- .../guardrail_hooks/bedrock_guardrails.py | 40 +++++++++++++-- .../test_bedrock_guardrails.py | 51 +++++++++++++++++++ 2 files changed, 87 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index b0a075c89a0..5a5b0307fa9 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -737,9 +737,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): else model_id_deployment ) candidate_deployments: Final = ( - [model_id_deployment_row] - if model_id_deployment_row is not None - else team_filtered_deployments + [model_id_deployment_row] if model_id_deployment_row is not None else team_filtered_deployments ) filter_deployments: Final = getattr(llm_router, "_filter_deployments_by_model_access_groups", None) @@ -771,6 +769,37 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): if blocked is not True: active_deployments.append(deployment) if not active_deployments: + router_settings = getattr(llm_router, "router_general_settings", None) + if getattr(router_settings, "pass_through_all_models", False) is True: + provider = request_data.get("custom_llm_provider") + if not isinstance(provider, str): + provider = BedrockGuardrail._resolve_model_provider(model) if isinstance(model, str) else None + return provider in ("bedrock", "bedrock_converse") if isinstance(provider, str) else None + + default_deployment = getattr(llm_router, "default_deployment", None) + if default_deployment is not None: + default_params: object = ( + default_deployment.get("litellm_params") + if isinstance(default_deployment, Mapping) + else getattr(default_deployment, "litellm_params", None) + ) + provider: object = ( + default_params.get("custom_llm_provider") + if isinstance(default_params, Mapping) + else getattr(default_params, "custom_llm_provider", None) + ) + if not isinstance(provider, str): + default_model: object = ( + default_params.get("model") + if isinstance(default_params, Mapping) + else getattr(default_params, "model", None) + ) + provider = ( + BedrockGuardrail._resolve_model_provider(default_model) + if isinstance(default_model, str) + else None + ) + return provider in ("bedrock", "bedrock_converse") if isinstance(provider, str) else None return False providers: list[str] = [] @@ -808,11 +837,14 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): if not isinstance(api_key, str): return None + explicit_provider: Final[object | None] = request_data.get("custom_llm_provider") + if isinstance(explicit_provider, str) and explicit_provider not in ("bedrock", "bedrock_converse"): + return None + router_allows_bedrock: Final = BedrockGuardrail._router_allows_bedrock(request_data) if router_allows_bedrock is not None: return api_key if router_allows_bedrock else None - explicit_provider: Final[object | None] = request_data.get("custom_llm_provider") if isinstance(explicit_provider, str): return api_key if explicit_provider in ("bedrock", "bedrock_converse") else None diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index 13bbb15ef30..0340f83c9d3 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -82,6 +82,57 @@ def test_bedrock_guardrail_resolves_router_model_id(): deployment.model_dump.assert_called_once_with(exclude_none=True) +def test_bedrock_guardrail_accepts_pass_through_bedrock_provider(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + + router = MagicMock() + router.get_model_list.return_value = [] + router.router_general_settings.pass_through_all_models = True + router.default_deployment = None + monkeypatch.setattr(proxy_server, "llm_router", router) + + request_data = { + "model": "amazon.nova-lite-v1:0", + "custom_llm_provider": "bedrock", + "api_key": "bedrock-key", + } + assert BedrockGuardrail._router_allows_bedrock(request_data) is True + assert BedrockGuardrail._get_bedrock_api_key(request_data) == "bedrock-key" + + +def test_bedrock_guardrail_accepts_bedrock_default_deployment(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + + router = MagicMock() + router.get_model_list.return_value = [] + router.router_general_settings.pass_through_all_models = False + router.default_deployment = {"litellm_params": {"custom_llm_provider": "bedrock"}} + monkeypatch.setattr(proxy_server, "llm_router", router) + + assert BedrockGuardrail._router_allows_bedrock({"model": "unlisted-model"}) is True + + +def test_bedrock_guardrail_explicit_non_bedrock_provider_wins_alias(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + + router = MagicMock() + router.get_model_list.return_value = [ + {"litellm_params": {"custom_llm_provider": "bedrock"}, "model_info": {}} + ] + monkeypatch.setattr(proxy_server, "llm_router", router) + + assert ( + BedrockGuardrail._get_bedrock_api_key( + { + "model": "bedrock-alias", + "custom_llm_provider": "openai", + "api_key": "openai-key", + } + ) + is None + ) + + def test_bedrock_guardrail_filters_access_group_deployments(): router = MagicMock() router.get_model_list.return_value = [ From 9fe37b1a3a8a86e0318e4807283493c6e28ba484 Mon Sep 17 00:00:00 2001 From: aiedwardyi <41576951+aiedwardyi@users.noreply.github.com> Date: Tue, 25 Aug 2026 00:04:00 +0900 Subject: [PATCH 08/21] fix: align Bedrock router eligibility --- .../guardrail_hooks/bedrock_guardrails.py | 248 ++++++++++++++---- .../test_bedrock_guardrails.py | 137 ++++++++++ 2 files changed, 335 insertions(+), 50 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 5a5b0307fa9..5d045e4d64e 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -53,6 +53,9 @@ from litellm.proxy.guardrails.anthropic_sse import ( is_raw_sse_stream, model_response_text, ) +from litellm.router_strategy.tag_based_routing import _get_tags_from_request_kwargs, is_valid_deployment_tag +from litellm.router_utils.common_utils import filter_team_based_models +from litellm.router_utils.cooldown_handlers import _get_cooldown_deployments from litellm.secret_managers.main import get_secret_str from litellm.types.guardrails import BedrockChecksConfigModel, GuardrailEventHooks from litellm.types.llms.openai import AllMessageValues, ChatCompletionUserMessage @@ -676,6 +679,82 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): except Exception: # noqa: BLE001 # provider resolution has a safe prefix fallback return model.partition("/")[0] + @staticmethod + def _filter_router_deployments_by_tags( + router: object, + deployments: list[object], + request_data: Mapping[str, object], + ) -> list[object]: + request_tags: Final = tuple(_get_tags_from_request_kwargs(request_data)) + deployment_tag_filtering: Final = any( + BedrockGuardrail._router_deployment_field(deployment, "enable_tag_filtering") is True + for deployment in deployments + ) + tag_filtering_enabled: Final = ( + request_data.get("enable_tag_filtering") is True + or getattr(router, "enable_tag_filtering", False) is True + or deployment_tag_filtering + ) + if not tag_filtering_enabled: + return deployments + + def _deployment_tags(deployment: object) -> tuple[str, ...]: + params: Final[object | None] = ( + deployment.get("litellm_params") + if isinstance(deployment, Mapping) + else getattr(deployment, "litellm_params", None) + ) + tags: Final[object | None] = ( + params.get("tags") if isinstance(params, Mapping) else getattr(params, "tags", None) + ) + return ( + tuple(tag for tag in tags if isinstance(tag, str)) + if isinstance(tags, Sequence) and not isinstance(tags, str) + else () + ) + + if not request_tags: + default_deployments: Final = [ + deployment for deployment in deployments if "default" in _deployment_tags(deployment) + ] + return default_deployments or deployments + + required_tags: Final = frozenset(tag[1:] for tag in request_tags if tag.startswith("&") and len(tag) > 1) + excluded_tags: Final = frozenset(tag[1:] for tag in request_tags if tag.startswith("!") and len(tag) > 1) + positive_tags: Final = tuple(tag for tag in request_tags if not tag.startswith(("&", "!"))) + allowed_deployments: Final = [ + deployment for deployment in deployments if not excluded_tags.intersection(_deployment_tags(deployment)) + ] + required_deployments: Final = [ + deployment for deployment in allowed_deployments if required_tags.issubset(_deployment_tags(deployment)) + ] + if not positive_tags: + return required_deployments + + match_any: Final[bool] = ( + getattr(router, "tag_filtering_match_any", True) + if isinstance(getattr(router, "tag_filtering_match_any", True), bool) + else True + ) + matched_deployments: Final = [ + deployment + for deployment in required_deployments + if is_valid_deployment_tag(_deployment_tags(deployment), positive_tags, match_any) + ] + if matched_deployments: + return matched_deployments + fallback_default_deployments: Final = [ + deployment for deployment in required_deployments if "default" in _deployment_tags(deployment) + ] + return fallback_default_deployments + + @staticmethod + def _router_deployment_field(deployment: object, field: str) -> object | None: + model_info: Final[object | None] = ( + deployment.get("model_info") if isinstance(deployment, Mapping) else getattr(deployment, "model_info", None) + ) + return model_info.get(field) if isinstance(model_info, Mapping) else getattr(model_info, field, None) + @staticmethod def _router_allows_bedrock(request_data: Mapping[str, object]) -> bool | None: model: Final[object | None] = request_data.get("model") @@ -699,76 +778,145 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): ) try: resolved_team_id: Final = team_id if isinstance(team_id, str) else None - listed_deployments: Final = ( - llm_router.get_model_list( - model_name=model, - team_id=resolved_team_id, - ) - or [] - ) - model_group_aliases: Final = getattr(llm_router, "model_group_alias", None) - team_filtered_deployments: Final = ( - [ - deployment - for deployment in listed_deployments - if ( - ( - deployment.get("model_info") - if isinstance(deployment, Mapping) - else getattr(deployment, "model_info", None) - ) - or {} - ).get("team_id") - in (None, resolved_team_id) - ] - if resolved_team_id is not None - and isinstance(model_group_aliases, Mapping) - and model in model_group_aliases - else listed_deployments - ) model_id_deployment: Final = ( - llm_router.get_deployment(model_id=model) - if not team_filtered_deployments and llm_router.has_model_id(model) is True - else None + llm_router.get_deployment(model_id=model) if llm_router.has_model_id(model) is True else None ) model_id_deployment_row: Final = ( model_id_deployment.model_dump(exclude_none=True) if model_id_deployment is not None and hasattr(model_id_deployment, "model_dump") else model_id_deployment ) - candidate_deployments: Final = ( - [model_id_deployment_row] if model_id_deployment_row is not None else team_filtered_deployments + raw_listed_deployments: Final = ( + [] + if model_id_deployment_row is not None + else ( + llm_router.get_model_list( + model_name=model, + team_id=resolved_team_id, + ) + or [] + ) + ) + model_group_aliases: Final = getattr(llm_router, "model_group_alias", None) + concrete_model_names: Final[object | None] = getattr(llm_router, "model_names", None) + is_concrete_model: Final = ( + isinstance(concrete_model_names, (list, tuple, set, frozenset)) and model in concrete_model_names + ) + is_model_alias: Final = isinstance(model_group_aliases, Mapping) and model in model_group_aliases + if model_id_deployment_row is None and not is_concrete_model and not is_model_alias: + pattern_router: Final[object | None] = getattr(llm_router, "pattern_router", None) + get_pattern_deployments: Final[object | None] = getattr( + pattern_router, "get_deployments_by_pattern", None + ) + global_pattern_deployments: Final = ( + get_pattern_deployments(model=model) if callable(get_pattern_deployments) else None + ) + team_pattern_router: Final[object | None] = ( + getattr(llm_router, "team_pattern_routers", {}).get(resolved_team_id) + if resolved_team_id is not None + and isinstance(getattr(llm_router, "team_pattern_routers", None), Mapping) + else None + ) + get_team_pattern_deployments: Final[object | None] = getattr( + team_pattern_router, "get_deployments_by_pattern", None + ) + team_pattern_deployments: Final = ( + get_team_pattern_deployments(model=model) if callable(get_team_pattern_deployments) else None + ) + selected_listed_deployments: Final = ( + global_pattern_deployments + if isinstance(global_pattern_deployments, list) and global_pattern_deployments + else ( + team_pattern_deployments + if isinstance(team_pattern_deployments, list) and team_pattern_deployments + else raw_listed_deployments + ) + ) + else: + selected_listed_deployments: Final = raw_listed_deployments + candidate_deployments: Final[list[object]] = ( + [model_id_deployment_row] + if model_id_deployment_row is not None + else [deployment for deployment in selected_listed_deployments if isinstance(deployment, Mapping)] + ) + router_matched: Final = bool(candidate_deployments) + team_filtered_result: Final = ( + candidate_deployments + if model_id_deployment_row is not None + else filter_team_based_models( + healthy_deployments=candidate_deployments, + request_kwargs=dict(request_data), + ) + ) + team_filtered_deployments: Final[list[object]] = ( + team_filtered_result if isinstance(team_filtered_result, list) else candidate_deployments ) filter_deployments: Final = getattr(llm_router, "_filter_deployments_by_model_access_groups", None) filtered_deployments: Final = ( filter_deployments( model=model, - healthy_deployments=candidate_deployments, + healthy_deployments=team_filtered_deployments, request_kwargs=dict(request_data), request_team_id=resolved_team_id, ) - if callable(filter_deployments) and isinstance(candidate_deployments, list) + if callable(filter_deployments) + and isinstance(team_filtered_deployments, list) + and model_id_deployment_row is None else None ) - deployments: Final = ( - filtered_deployments if isinstance(filtered_deployments, list) else candidate_deployments + access_filtered_deployments: Final[list[object]] = ( + filtered_deployments if isinstance(filtered_deployments, list) else team_filtered_deployments + ) + health_filter: Final[object | None] = getattr( + llm_router, "_filter_health_check_unhealthy_deployments", None + ) + health_filtered_deployments: Final = ( + health_filter( + healthy_deployments=access_filtered_deployments, + parent_otel_span=None, + ) + if callable(health_filter) + else access_filtered_deployments + ) + healthy_deployments: Final[list[object]] = ( + health_filtered_deployments + if isinstance(health_filtered_deployments, list) + else access_filtered_deployments + ) + cooldown_cache: Final[object | None] = getattr(llm_router, "cooldown_cache", None) + cooldown_lookup: Final[object | None] = getattr(cooldown_cache, "get_active_cooldowns", None) + cooldown_deployments: Final = ( + _get_cooldown_deployments( + litellm_router_instance=llm_router, + parent_otel_span=None, + ) + if callable(cooldown_lookup) and callable(getattr(llm_router, "get_model_ids", None)) + else [] + ) + cooldown_ids: Final[frozenset[str]] = frozenset( + deployment_id for deployment_id in cooldown_deployments if isinstance(deployment_id, str) + ) + cooldown_filtered_deployments: Final = [ + deployment + for deployment in healthy_deployments + if BedrockGuardrail._router_deployment_field(deployment, "id") not in cooldown_ids + ] + unblocked_deployments: Final = [ + deployment + for deployment in cooldown_filtered_deployments + if BedrockGuardrail._router_deployment_field(deployment, "blocked") is not True + ] + deployments: Final = BedrockGuardrail._filter_router_deployments_by_tags( + router=llm_router, + deployments=unblocked_deployments, + request_data=request_data, ) except Exception: # noqa: BLE001 # optional router state must not break guardrail auth return False - active_deployments = [] - for deployment in deployments: - model_info = ( - deployment.get("model_info") - if isinstance(deployment, Mapping) - else getattr(deployment, "model_info", None) - ) - blocked = ( - model_info.get("blocked") if isinstance(model_info, Mapping) else getattr(model_info, "blocked", None) - ) - if blocked is not True: - active_deployments.append(deployment) - if not active_deployments: + if not deployments: + if router_matched: + return False router_settings = getattr(llm_router, "router_general_settings", None) if getattr(router_settings, "pass_through_all_models", False) is True: provider = request_data.get("custom_llm_provider") @@ -803,7 +951,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): return False providers: list[str] = [] - for deployment in active_deployments: + for deployment in deployments: params: object = ( deployment.get("litellm_params") if isinstance(deployment, Mapping) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index 0340f83c9d3..ae11637a0a5 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -112,6 +112,143 @@ def test_bedrock_guardrail_accepts_bedrock_default_deployment(monkeypatch: pytes assert BedrockGuardrail._router_allows_bedrock({"model": "unlisted-model"}) is True +def test_bedrock_guardrail_rejects_blocked_model_with_pass_through(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + + router = MagicMock() + router.get_model_list.return_value = [ + {"litellm_params": {"custom_llm_provider": "openai"}, "model_info": {"blocked": True}} + ] + router.router_general_settings.pass_through_all_models = True + router.default_deployment = None + monkeypatch.setattr(proxy_server, "llm_router", router) + + assert ( + BedrockGuardrail._get_bedrock_api_key( + { + "model": "blocked-alias", + "custom_llm_provider": "bedrock", + "api_key": "bedrock-key", + } + ) + is None + ) + + +def test_bedrock_guardrail_rejects_access_filtered_model_with_pass_through( + monkeypatch: pytest.MonkeyPatch, +): + from litellm.proxy import proxy_server + + router = MagicMock() + router.get_model_list.return_value = [ + {"litellm_params": {"custom_llm_provider": "openai"}, "model_info": {}}, + {"litellm_params": {"custom_llm_provider": "bedrock"}, "model_info": {}}, + ] + router._filter_deployments_by_model_access_groups.return_value = [] + router.router_general_settings.pass_through_all_models = True + router.default_deployment = None + monkeypatch.setattr(proxy_server, "llm_router", router) + + assert ( + BedrockGuardrail._get_bedrock_api_key( + { + "model": "scoped-alias", + "custom_llm_provider": "bedrock", + "api_key": "bedrock-key", + } + ) + is None + ) + + +def test_bedrock_guardrail_resolves_model_id_before_wildcards(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + + router = MagicMock() + router.has_model_id.return_value = True + deployment = MagicMock() + deployment.model_dump.return_value = { + "litellm_params": {"custom_llm_provider": "bedrock"}, + "model_info": {"id": "deployment-id"}, + } + router.get_deployment.return_value = deployment + router.get_model_list.return_value = [ + {"litellm_params": {"custom_llm_provider": "openai"}, "model_info": {}} + ] + monkeypatch.setattr(proxy_server, "llm_router", router) + + assert BedrockGuardrail._router_allows_bedrock({"model": "deployment-id"}) is True + router.get_model_list.assert_not_called() + + +def test_bedrock_guardrail_ignores_cooling_non_bedrock_deployments(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + + router = MagicMock() + router.get_model_list.return_value = [ + { + "litellm_params": {"custom_llm_provider": "openai"}, + "model_info": {"id": "openai-deployment"}, + }, + { + "litellm_params": {"custom_llm_provider": "bedrock"}, + "model_info": {"id": "bedrock-deployment"}, + }, + ] + router.get_model_ids.return_value = ["openai-deployment", "bedrock-deployment"] + router.cooldown_cache.get_active_cooldowns.return_value = [("openai-deployment", 1.0)] + monkeypatch.setattr(proxy_server, "llm_router", router) + + assert BedrockGuardrail._router_allows_bedrock({"model": "shared-alias"}) is True + + +def test_bedrock_guardrail_matches_global_wildcard_precedence(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + + router = MagicMock() + router.model_names = [] + router.model_group_alias = {} + router.get_model_list.return_value = [ + {"litellm_params": {"custom_llm_provider": "bedrock"}, "model_info": {}} + ] + router.pattern_router.get_deployments_by_pattern.return_value = [ + {"litellm_params": {"custom_llm_provider": "bedrock"}, "model_info": {}} + ] + team_pattern_router = MagicMock() + team_pattern_router.get_deployments_by_pattern.return_value = [ + {"litellm_params": {"custom_llm_provider": "openai"}, "model_info": {}} + ] + router.team_pattern_routers = {"team-id": team_pattern_router} + monkeypatch.setattr(proxy_server, "llm_router", router) + + assert ( + BedrockGuardrail._router_allows_bedrock( + {"model": "provider/model", "litellm_metadata": {"user_api_key_team_id": "team-id"}} + ) + is True + ) + + +def test_bedrock_guardrail_matches_request_tag_pool(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + + router = MagicMock() + router.enable_tag_filtering = True + router.get_model_list.return_value = [ + {"litellm_params": {"custom_llm_provider": "openai", "tags": ["slow"]}, "model_info": {}}, + {"litellm_params": {"custom_llm_provider": "bedrock", "tags": ["fast"]}, "model_info": {}}, + ] + monkeypatch.setattr(proxy_server, "llm_router", router) + + assert ( + BedrockGuardrail._router_allows_bedrock( + {"model": "shared-alias", "metadata": {"tags": ["fast"]}} + ) + is True + ) + + def test_bedrock_guardrail_explicit_non_bedrock_provider_wins_alias(monkeypatch: pytest.MonkeyPatch): from litellm.proxy import proxy_server From 46d9a2a67d0c3948406b92d3a00fc332e8155e38 Mon Sep 17 00:00:00 2001 From: aiedwardyi <41576951+aiedwardyi@users.noreply.github.com> Date: Tue, 25 Aug 2026 10:56:39 +0900 Subject: [PATCH 09/21] fix: align Bedrock tag and fallback routing --- .../guardrail_hooks/bedrock_guardrails.py | 112 ++++++++++++++---- .../test_bedrock_guardrails.py | 85 +++++++++++++ 2 files changed, 175 insertions(+), 22 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 5d045e4d64e..2fed75f2dbe 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -31,6 +31,7 @@ from litellm.constants import BEDROCK_APPLY_GUARDRAIL_CHUNK_BUDGET_CHARS from litellm.exceptions import ModifyResponseException from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.core_helpers import ( + get_metadata_variable_name_from_kwargs, redact_nested_match_and_regex_keys, ) from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import bedrock_guardrail_cost @@ -53,7 +54,14 @@ from litellm.proxy.guardrails.anthropic_sse import ( is_raw_sse_stream, model_response_text, ) -from litellm.router_strategy.tag_based_routing import _get_tags_from_request_kwargs, is_valid_deployment_tag +from litellm.router_strategy.tag_based_routing import ( + _chain_tag_filtering_override, + _get_tags_from_request_kwargs, + _match_deployment, + _request_tags_after_router_consumption, + _split_tags, + _strip_routing_prefix, +) from litellm.router_utils.common_utils import filter_team_based_models from litellm.router_utils.cooldown_handlers import _get_cooldown_deployments from litellm.secret_managers.main import get_secret_str @@ -685,16 +693,18 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): deployments: list[object], request_data: Mapping[str, object], ) -> list[object]: - request_tags: Final = tuple(_get_tags_from_request_kwargs(request_data)) - deployment_tag_filtering: Final = any( - BedrockGuardrail._router_deployment_field(deployment, "enable_tag_filtering") is True - for deployment in deployments + model: Final[object] = request_data.get("model") + chain_tag_filtering: Final[object] = ( + _chain_tag_filtering_override(router, model, deployments) + if isinstance(model, str) + else None ) - tag_filtering_enabled: Final = ( - request_data.get("enable_tag_filtering") is True - or getattr(router, "enable_tag_filtering", False) is True - or deployment_tag_filtering + effective_tag_filtering: Final = ( + chain_tag_filtering + if isinstance(chain_tag_filtering, bool) + else getattr(router, "enable_tag_filtering", False) ) + tag_filtering_enabled: Final = request_data.get("enable_tag_filtering") is True or effective_tag_filtering is True if not tag_filtering_enabled: return deployments @@ -713,23 +723,37 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): else () ) - if not request_tags: + metadata_name: Final = get_metadata_variable_name_from_kwargs(request_data) + metadata: Final[object] = request_data.get(metadata_name) + if not isinstance(metadata, Mapping): default_deployments: Final = [ deployment for deployment in deployments if "default" in _deployment_tags(deployment) ] return default_deployments or deployments - required_tags: Final = frozenset(tag[1:] for tag in request_tags if tag.startswith("&") and len(tag) > 1) - excluded_tags: Final = frozenset(tag[1:] for tag in request_tags if tag.startswith("!") and len(tag) > 1) - positive_tags: Final = tuple(tag for tag in request_tags if not tag.startswith(("&", "!"))) + request_tags: Sequence[str] = tuple(_get_tags_from_request_kwargs(request_data)) + if isinstance(model, str): + request_tags = _request_tags_after_router_consumption(metadata, model) or () + routing_prefix: Final[object] = getattr(router, "tag_routing_prefix", "") + resolved_prefix: Final[str] = routing_prefix if isinstance(routing_prefix, str) else "" + rewritten_tags, _ = _strip_routing_prefix(request_tags, resolved_prefix) + required_tags, positive_tags, excluded_tags = _split_tags(rewritten_tags) + required_set: Final = frozenset(required_tags) + excluded_set: Final = frozenset(excluded_tags) allowed_deployments: Final = [ - deployment for deployment in deployments if not excluded_tags.intersection(_deployment_tags(deployment)) + deployment for deployment in deployments if not excluded_set.intersection(_deployment_tags(deployment)) ] - required_deployments: Final = [ - deployment for deployment in allowed_deployments if required_tags.issubset(_deployment_tags(deployment)) + candidate_deployments: Final = [ + deployment for deployment in allowed_deployments if required_set.issubset(_deployment_tags(deployment)) ] - if not positive_tags: - return required_deployments + + user_agent: Final[object] = metadata.get("user_agent") + header_strings: Final = [f"User-Agent: {user_agent}"] if isinstance(user_agent, str) and user_agent else [] + if not positive_tags and not header_strings: + default_deployments: Final = [ + deployment for deployment in candidate_deployments if "default" in _deployment_tags(deployment) + ] + return default_deployments or candidate_deployments match_any: Final[bool] = ( getattr(router, "tag_filtering_match_any", True) @@ -738,13 +762,20 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): ) matched_deployments: Final = [ deployment - for deployment in required_deployments - if is_valid_deployment_tag(_deployment_tags(deployment), positive_tags, match_any) + for deployment in candidate_deployments + if isinstance(deployment, Mapping) + and _match_deployment( + deployment=deployment, + request_tags=positive_tags, + header_strings=header_strings, + match_any=match_any, + ) + is not None ] if matched_deployments: return matched_deployments fallback_default_deployments: Final = [ - deployment for deployment in required_deployments if "default" in _deployment_tags(deployment) + deployment for deployment in candidate_deployments if "default" in _deployment_tags(deployment) ] return fallback_default_deployments @@ -786,9 +817,24 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): if model_id_deployment is not None and hasattr(model_id_deployment, "model_dump") else model_id_deployment ) + specific_deployment_rows: list[object] | None = None + deployment_names: Final[object] = getattr(llm_router, "deployment_names", None) + specific_lookup: Final[object] = getattr(llm_router, "_get_deployment_by_litellm_model", None) + if ( + model_id_deployment_row is None + and isinstance(deployment_names, Sequence) + and not isinstance(deployment_names, (str, bytes)) + and model in deployment_names + and callable(specific_lookup) + ): + specific_result: Final = specific_lookup(model=model) + if isinstance(specific_result, list): + specific_deployment_rows = specific_result raw_listed_deployments: Final = ( [] if model_id_deployment_row is not None + else specific_deployment_rows + if specific_deployment_rows is not None else ( llm_router.get_model_list( model_name=model, @@ -803,7 +849,12 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): isinstance(concrete_model_names, (list, tuple, set, frozenset)) and model in concrete_model_names ) is_model_alias: Final = isinstance(model_group_aliases, Mapping) and model in model_group_aliases - if model_id_deployment_row is None and not is_concrete_model and not is_model_alias: + if ( + model_id_deployment_row is None + and specific_deployment_rows is None + and not is_concrete_model + and not is_model_alias + ): pattern_router: Final[object | None] = getattr(llm_router, "pattern_router", None) get_pattern_deployments: Final[object | None] = getattr( pattern_router, "get_deployments_by_pattern", None @@ -924,6 +975,23 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): provider = BedrockGuardrail._resolve_model_provider(model) if isinstance(model, str) else None return provider in ("bedrock", "bedrock_converse") if isinstance(provider, str) else None + default_fallback_lookup: Final[object] = getattr(llm_router, "_get_first_default_fallback", None) + default_fallback_model: Final[object] = ( + default_fallback_lookup() if callable(default_fallback_lookup) else None + ) + if isinstance(default_fallback_model, str) and default_fallback_model != model: + fallback_deployments: Final = ( + llm_router.get_model_list( + model_name=default_fallback_model, + team_id=resolved_team_id, + ) + or [] + ) + if isinstance(fallback_deployments, list) and fallback_deployments: + fallback_request_data: Final = dict(request_data) + fallback_request_data["model"] = default_fallback_model + return BedrockGuardrail._router_allows_bedrock(fallback_request_data) + default_deployment = getattr(llm_router, "default_deployment", None) if default_deployment is not None: default_params: object = ( diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index ae11637a0a5..e09a8a94f80 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -249,6 +249,91 @@ def test_bedrock_guardrail_matches_request_tag_pool(monkeypatch: pytest.MonkeyPa ) +def test_bedrock_guardrail_matches_regex_tag_pool(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + + router = MagicMock() + router.enable_tag_filtering = True + router.get_model_list.return_value = [ + {"litellm_params": {"custom_llm_provider": "openai"}, "model_info": {}}, + { + "litellm_params": { + "custom_llm_provider": "bedrock", + "tag_regex": [r"^User-Agent: claude-code/"], + }, + "model_info": {}, + }, + ] + monkeypatch.setattr(proxy_server, "llm_router", router) + + assert ( + BedrockGuardrail._router_allows_bedrock( + {"model": "shared-alias", "metadata": {"user_agent": "claude-code/1.0"}} + ) + is True + ) + + +def test_bedrock_guardrail_honors_false_chain_tag_filtering_override(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + + router = MagicMock() + router.enable_tag_filtering = True + deployments = [ + { + "litellm_params": {"custom_llm_provider": "openai", "tags": ["slow"]}, + "model_info": {"enable_tag_filtering": False}, + }, + {"litellm_params": {"custom_llm_provider": "bedrock", "tags": ["fast"]}, "model_info": {}}, + ] + router.get_model_list.return_value = deployments + router._get_all_deployments.return_value = deployments + monkeypatch.setattr(proxy_server, "llm_router", router) + + assert ( + BedrockGuardrail._router_allows_bedrock( + {"model": "shared-alias", "metadata": {"tags": ["fast"]}} + ) + is False + ) + + +def test_bedrock_guardrail_resolves_specific_deployment_name(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + + router = MagicMock() + router.deployment_names = ["bedrock-deployment"] + router._get_deployment_by_litellm_model.return_value = [ + {"litellm_params": {"custom_llm_provider": "bedrock"}, "model_info": {"id": "bedrock-deployment"}} + ] + router.get_model_list.return_value = [ + {"litellm_params": {"custom_llm_provider": "openai"}, "model_info": {}} + ] + monkeypatch.setattr(proxy_server, "llm_router", router) + + assert BedrockGuardrail._router_allows_bedrock({"model": "bedrock-deployment"}) is True + router._get_deployment_by_litellm_model.assert_called_once_with(model="bedrock-deployment") + router.get_model_list.assert_not_called() + + +def test_bedrock_guardrail_follows_default_fallback_group(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + + router = MagicMock() + router._get_first_default_fallback.return_value = "bedrock-fallback" + router.default_deployment = None + + def _get_model_list(model_name: str, team_id: str | None = None) -> list[dict[str, object]]: + if model_name == "bedrock-fallback": + return [{"litellm_params": {"custom_llm_provider": "bedrock"}, "model_info": {}}] + return [] + + router.get_model_list.side_effect = _get_model_list + monkeypatch.setattr(proxy_server, "llm_router", router) + + assert BedrockGuardrail._router_allows_bedrock({"model": "unknown-model"}) is True + + def test_bedrock_guardrail_explicit_non_bedrock_provider_wins_alias(monkeypatch: pytest.MonkeyPatch): from litellm.proxy import proxy_server From c58daa5d1113a5479561a0ab787b73e1b083783e Mon Sep 17 00:00:00 2001 From: aiedwardyi <41576951+aiedwardyi@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:05:08 +0900 Subject: [PATCH 10/21] fix: align Bedrock router selection --- .../guardrail_hooks/bedrock_guardrails.py | 295 +++++++++++------- .../test_bedrock_guardrails.py | 71 +++++ 2 files changed, 245 insertions(+), 121 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 2fed75f2dbe..325431fd9f7 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -62,7 +62,7 @@ from litellm.router_strategy.tag_based_routing import ( _split_tags, _strip_routing_prefix, ) -from litellm.router_utils.common_utils import filter_team_based_models +from litellm.router_utils.common_utils import filter_team_based_models, filter_web_search_deployments from litellm.router_utils.cooldown_handlers import _get_cooldown_deployments from litellm.secret_managers.main import get_secret_str from litellm.types.guardrails import BedrockChecksConfigModel, GuardrailEventHooks @@ -695,16 +695,16 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): ) -> list[object]: model: Final[object] = request_data.get("model") chain_tag_filtering: Final[object] = ( - _chain_tag_filtering_override(router, model, deployments) - if isinstance(model, str) - else None + _chain_tag_filtering_override(router, model, deployments) if isinstance(model, str) else None ) effective_tag_filtering: Final = ( chain_tag_filtering if isinstance(chain_tag_filtering, bool) else getattr(router, "enable_tag_filtering", False) ) - tag_filtering_enabled: Final = request_data.get("enable_tag_filtering") is True or effective_tag_filtering is True + tag_filtering_enabled: Final = ( + request_data.get("enable_tag_filtering") is True or effective_tag_filtering is True + ) if not tag_filtering_enabled: return deployments @@ -749,7 +749,14 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): user_agent: Final[object] = metadata.get("user_agent") header_strings: Final = [f"User-Agent: {user_agent}"] if isinstance(user_agent, str) and user_agent else [] - if not positive_tags and not header_strings: + has_regex_deployments: Final = any( + isinstance(deployment, Mapping) and bool((deployment.get("litellm_params") or {}).get("tag_regex")) + for deployment in candidate_deployments + ) + has_positive_filter: Final = bool(positive_tags) or ( + bool(header_strings) and has_regex_deployments and not required_set + ) + if not has_positive_filter: default_deployments: Final = [ deployment for deployment in candidate_deployments if "default" in _deployment_tags(deployment) ] @@ -809,94 +816,112 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): ) try: resolved_team_id: Final = team_id if isinstance(team_id, str) else None - model_id_deployment: Final = ( - llm_router.get_deployment(model_id=model) if llm_router.has_model_id(model) is True else None - ) - model_id_deployment_row: Final = ( - model_id_deployment.model_dump(exclude_none=True) - if model_id_deployment is not None and hasattr(model_id_deployment, "model_dump") - else model_id_deployment - ) - specific_deployment_rows: list[object] | None = None - deployment_names: Final[object] = getattr(llm_router, "deployment_names", None) - specific_lookup: Final[object] = getattr(llm_router, "_get_deployment_by_litellm_model", None) - if ( - model_id_deployment_row is None - and isinstance(deployment_names, Sequence) - and not isinstance(deployment_names, (str, bytes)) - and model in deployment_names - and callable(specific_lookup) - ): - specific_result: Final = specific_lookup(model=model) - if isinstance(specific_result, list): - specific_deployment_rows = specific_result - raw_listed_deployments: Final = ( - [] - if model_id_deployment_row is not None - else specific_deployment_rows - if specific_deployment_rows is not None - else ( - llm_router.get_model_list( - model_name=model, - team_id=resolved_team_id, - ) - or [] + router_kwargs: Final = dict(request_data) + common_result: tuple[object, object] | None = None + common_lookup: Final[object] = getattr(llm_router, "_common_checks_available_deployment", None) + if callable(common_lookup): + try: + raw_common_result: Final = common_lookup(model=model, request_kwargs=router_kwargs) + except Exception: # noqa: BLE001 # fall back for lightweight router test doubles + raw_common_result = None + if ( + isinstance(raw_common_result, tuple) + and len(raw_common_result) == 2 + and isinstance(raw_common_result[1], (Mapping, list)) + ): + common_result = raw_common_result + + if common_result is not None: + raw_deployments: Final[object] = common_result[1] + model_id_deployment_row: Final[object | None] = ( + raw_deployments if isinstance(raw_deployments, Mapping) else None ) - ) - model_group_aliases: Final = getattr(llm_router, "model_group_alias", None) - concrete_model_names: Final[object | None] = getattr(llm_router, "model_names", None) - is_concrete_model: Final = ( - isinstance(concrete_model_names, (list, tuple, set, frozenset)) and model in concrete_model_names - ) - is_model_alias: Final = isinstance(model_group_aliases, Mapping) and model in model_group_aliases - if ( - model_id_deployment_row is None - and specific_deployment_rows is None - and not is_concrete_model - and not is_model_alias - ): - pattern_router: Final[object | None] = getattr(llm_router, "pattern_router", None) - get_pattern_deployments: Final[object | None] = getattr( - pattern_router, "get_deployments_by_pattern", None - ) - global_pattern_deployments: Final = ( - get_pattern_deployments(model=model) if callable(get_pattern_deployments) else None - ) - team_pattern_router: Final[object | None] = ( - getattr(llm_router, "team_pattern_routers", {}).get(resolved_team_id) - if resolved_team_id is not None - and isinstance(getattr(llm_router, "team_pattern_routers", None), Mapping) - else None - ) - get_team_pattern_deployments: Final[object | None] = getattr( - team_pattern_router, "get_deployments_by_pattern", None - ) - team_pattern_deployments: Final = ( - get_team_pattern_deployments(model=model) if callable(get_team_pattern_deployments) else None - ) - selected_listed_deployments: Final = ( - global_pattern_deployments - if isinstance(global_pattern_deployments, list) and global_pattern_deployments - else ( - team_pattern_deployments - if isinstance(team_pattern_deployments, list) and team_pattern_deployments - else raw_listed_deployments - ) + candidate_deployments: Final[list[object]] = ( + [raw_deployments] + if isinstance(raw_deployments, Mapping) + else [deployment for deployment in raw_deployments if isinstance(deployment, Mapping)] ) + router_matched: Final = bool(candidate_deployments) else: - selected_listed_deployments: Final = raw_listed_deployments - candidate_deployments: Final[list[object]] = ( - [model_id_deployment_row] - if model_id_deployment_row is not None - else [deployment for deployment in selected_listed_deployments if isinstance(deployment, Mapping)] - ) - router_matched: Final = bool(candidate_deployments) + model_id_deployment: Final = ( + llm_router.get_deployment(model_id=model) if llm_router.has_model_id(model) is True else None + ) + model_id_deployment_row: Final = ( + model_id_deployment.model_dump(exclude_none=True) + if model_id_deployment is not None and hasattr(model_id_deployment, "model_dump") + else model_id_deployment + ) + specific_deployment_rows: list[object] | None = None + deployment_names: Final[object] = getattr(llm_router, "deployment_names", None) + specific_lookup: Final[object] = getattr(llm_router, "_get_deployment_by_litellm_model", None) + model_group_aliases: Final = getattr(llm_router, "model_group_alias", None) + concrete_model_names: Final[object | None] = getattr(llm_router, "model_names", None) + is_concrete_model: Final = ( + isinstance(concrete_model_names, (list, tuple, set, frozenset)) and model in concrete_model_names + ) + is_model_alias: Final = isinstance(model_group_aliases, Mapping) and model in model_group_aliases + raw_listed_deployments = ( + [] + if model_id_deployment_row is not None + else ( + llm_router.get_model_list(model_name=model, team_id=resolved_team_id) or [] + if is_concrete_model or is_model_alias + else [] + ) + ) + if model_id_deployment_row is None and not is_concrete_model and not is_model_alias: + pattern_router: Final[object | None] = getattr(llm_router, "pattern_router", None) + get_pattern_deployments: Final[object | None] = getattr( + pattern_router, "get_deployments_by_pattern", None + ) + global_pattern_deployments: Final = ( + get_pattern_deployments(model=model) if callable(get_pattern_deployments) else None + ) + team_pattern_router: Final[object | None] = ( + getattr(llm_router, "team_pattern_routers", {}).get(resolved_team_id) + if resolved_team_id is not None + and isinstance(getattr(llm_router, "team_pattern_routers", None), Mapping) + else None + ) + get_team_pattern_deployments: Final[object | None] = getattr( + team_pattern_router, "get_deployments_by_pattern", None + ) + team_pattern_deployments: Final = ( + get_team_pattern_deployments(model=model) if callable(get_team_pattern_deployments) else None + ) + if isinstance(global_pattern_deployments, list) and global_pattern_deployments: + raw_listed_deployments = global_pattern_deployments + elif isinstance(team_pattern_deployments, list) and team_pattern_deployments: + raw_listed_deployments = team_pattern_deployments + else: + default_deployment = getattr(llm_router, "default_deployment", None) + if isinstance(default_deployment, Mapping): + raw_listed_deployments = [default_deployment] + elif ( + isinstance(deployment_names, Sequence) + and not isinstance(deployment_names, (str, bytes)) + and model in deployment_names + and callable(specific_lookup) + ): + specific_result: Final = specific_lookup(model=model) + specific_deployment_rows = specific_result if isinstance(specific_result, list) else [] + raw_listed_deployments = specific_deployment_rows + else: + raw_listed_deployments = ( + llm_router.get_model_list(model_name=model, team_id=resolved_team_id) or [] + ) + candidate_deployments = ( + [model_id_deployment_row] + if model_id_deployment_row is not None + else [deployment for deployment in raw_listed_deployments if isinstance(deployment, Mapping)] + ) + router_matched = bool(candidate_deployments) team_filtered_result: Final = ( candidate_deployments if model_id_deployment_row is not None else filter_team_based_models( healthy_deployments=candidate_deployments, - request_kwargs=dict(request_data), + request_kwargs=router_kwargs, ) ) team_filtered_deployments: Final[list[object]] = ( @@ -927,7 +952,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): healthy_deployments=access_filtered_deployments, parent_otel_span=None, ) - if callable(health_filter) + if (common_result is None or model_id_deployment_row is None) and callable(health_filter) else access_filtered_deployments ) healthy_deployments: Final[list[object]] = ( @@ -942,27 +967,54 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): litellm_router_instance=llm_router, parent_otel_span=None, ) - if callable(cooldown_lookup) and callable(getattr(llm_router, "get_model_ids", None)) + if (common_result is None or model_id_deployment_row is None) + and callable(cooldown_lookup) + and callable(getattr(llm_router, "get_model_ids", None)) else [] ) cooldown_ids: Final[frozenset[str]] = frozenset( deployment_id for deployment_id in cooldown_deployments if isinstance(deployment_id, str) ) - cooldown_filtered_deployments: Final = [ - deployment - for deployment in healthy_deployments - if BedrockGuardrail._router_deployment_field(deployment, "id") not in cooldown_ids - ] - unblocked_deployments: Final = [ - deployment - for deployment in cooldown_filtered_deployments - if BedrockGuardrail._router_deployment_field(deployment, "blocked") is not True - ] - deployments: Final = BedrockGuardrail._filter_router_deployments_by_tags( - router=llm_router, - deployments=unblocked_deployments, - request_data=request_data, - ) + if common_result is not None and model_id_deployment_row is not None: + deployments = candidate_deployments + else: + cooldown_filtered_deployments: Final = [ + deployment + for deployment in healthy_deployments + if BedrockGuardrail._router_deployment_field(deployment, "id") not in cooldown_ids + ] + unblocked_deployments: Final = [ + deployment + for deployment in cooldown_filtered_deployments + if BedrockGuardrail._router_deployment_field(deployment, "blocked") is not True + ] + deployments = BedrockGuardrail._filter_router_deployments_by_tags( + router=llm_router, + deployments=unblocked_deployments, + request_data=request_data, + ) + if common_result is not None and model_id_deployment_row is None: + web_search_deployments: Final = filter_web_search_deployments( + healthy_deployments=deployments, + request_kwargs=router_kwargs, + ) + deployments = web_search_deployments if isinstance(web_search_deployments, list) else deployments + plugin_filter: Final[object] = getattr(llm_router, "_filter_by_routing_plugin_candidates", None) + if callable(plugin_filter): + plugin_deployments: Final = plugin_filter( + healthy_deployments=deployments, + request_kwargs=router_kwargs, + ) + if isinstance(plugin_deployments, list): + deployments = plugin_deployments + deployments = litellm.utils._get_order_filtered_deployments( + deployments, + target_order=router_kwargs.pop("_target_order", None), + ) + deployments = litellm.utils._get_excluded_filtered_deployments( + deployments, + excluded_deployment_ids=router_kwargs.pop("_excluded_deployment_ids", None), + ) except Exception: # noqa: BLE001 # optional router state must not break guardrail auth return False if not deployments: @@ -975,25 +1027,8 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): provider = BedrockGuardrail._resolve_model_provider(model) if isinstance(model, str) else None return provider in ("bedrock", "bedrock_converse") if isinstance(provider, str) else None - default_fallback_lookup: Final[object] = getattr(llm_router, "_get_first_default_fallback", None) - default_fallback_model: Final[object] = ( - default_fallback_lookup() if callable(default_fallback_lookup) else None - ) - if isinstance(default_fallback_model, str) and default_fallback_model != model: - fallback_deployments: Final = ( - llm_router.get_model_list( - model_name=default_fallback_model, - team_id=resolved_team_id, - ) - or [] - ) - if isinstance(fallback_deployments, list) and fallback_deployments: - fallback_request_data: Final = dict(request_data) - fallback_request_data["model"] = default_fallback_model - return BedrockGuardrail._router_allows_bedrock(fallback_request_data) - default_deployment = getattr(llm_router, "default_deployment", None) - if default_deployment is not None: + if isinstance(default_deployment, Mapping): default_params: object = ( default_deployment.get("litellm_params") if isinstance(default_deployment, Mapping) @@ -1016,6 +1051,24 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): else None ) return provider in ("bedrock", "bedrock_converse") if isinstance(provider, str) else None + + default_fallback_lookup: Final[object] = getattr(llm_router, "_get_first_default_fallback", None) + default_fallback_model: Final[object] = ( + default_fallback_lookup() if callable(default_fallback_lookup) else None + ) + if isinstance(default_fallback_model, str) and default_fallback_model != model: + fallback_deployments: Final = ( + llm_router.get_model_list( + model_name=default_fallback_model, + team_id=resolved_team_id, + ) + or [] + ) + if isinstance(fallback_deployments, list) and fallback_deployments: + fallback_request_data: Final = dict(request_data) + fallback_request_data["model"] = default_fallback_model + return BedrockGuardrail._router_allows_bedrock(fallback_request_data) + return False providers: list[str] = [] diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index e09a8a94f80..5249fc29a6e 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -316,6 +316,77 @@ def test_bedrock_guardrail_resolves_specific_deployment_name(monkeypatch: pytest router.get_model_list.assert_not_called() +def test_bedrock_guardrail_ignores_user_agent_without_regex_route(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + + router = MagicMock() + router.enable_tag_filtering = True + router.get_model_list.return_value = [ + {"litellm_params": {"custom_llm_provider": "bedrock"}, "model_info": {}} + ] + monkeypatch.setattr(proxy_server, "llm_router", router) + + assert BedrockGuardrail._router_allows_bedrock( + {"model": "shared-alias", "metadata": {"user_agent": "client/1.0"}} + ) is True + + +def test_bedrock_guardrail_applies_router_post_filters(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + + router = MagicMock() + deployments = [ + { + "litellm_params": {"custom_llm_provider": "openai", "order": 1}, + "model_info": {"id": "openai"}, + }, + { + "litellm_params": {"custom_llm_provider": "bedrock", "order": 2}, + "model_info": {"id": "bedrock"}, + }, + ] + router._common_checks_available_deployment.return_value = ("shared-alias", deployments) + router._filter_health_check_unhealthy_deployments.return_value = deployments + router.routing_plugins = [] + router.default_deployment = None + monkeypatch.setattr(proxy_server, "llm_router", router) + + assert BedrockGuardrail._router_allows_bedrock({"model": "shared-alias"}) is False + assert ( + BedrockGuardrail._router_allows_bedrock( + {"model": "shared-alias", "_excluded_deployment_ids": ["openai"]} + ) + is True + ) + + +def test_bedrock_guardrail_applies_web_search_filter(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + + router = MagicMock() + deployments = [ + { + "litellm_params": {"custom_llm_provider": "openai"}, + "model_info": {"id": "openai", "supports_web_search": True}, + }, + { + "litellm_params": {"custom_llm_provider": "bedrock"}, + "model_info": {"id": "bedrock", "supports_web_search": False}, + }, + ] + router._common_checks_available_deployment.return_value = ("shared-alias", deployments) + router._filter_health_check_unhealthy_deployments.return_value = deployments + router.routing_plugins = [] + monkeypatch.setattr(proxy_server, "llm_router", router) + + assert ( + BedrockGuardrail._router_allows_bedrock( + {"model": "shared-alias", "tools": [{"type": "web_search"}]} + ) + is False + ) + + def test_bedrock_guardrail_follows_default_fallback_group(monkeypatch: pytest.MonkeyPatch): from litellm.proxy import proxy_server From 250f30a211ee930ad432a0411a47a48da3b86a31 Mon Sep 17 00:00:00 2001 From: aiedwardyi <41576951+aiedwardyi@users.noreply.github.com> Date: Wed, 26 Aug 2026 11:09:15 +0900 Subject: [PATCH 11/21] fix: align Bedrock guardrail eligibility --- .../guardrail_hooks/bedrock_guardrails.py | 210 +++++++++++++++--- 1 file changed, 185 insertions(+), 25 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 325431fd9f7..2e0a2eb68ec 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -119,6 +119,7 @@ _BEDROCK_INVOKE_GUARDRAIL_CHECKS_PATH: Final = "/guardrail-checks/invoke" # more text blocks is split across multiple messages so ALL content is scanned -- # never truncated (truncation would let a user hide content past the limit). _BEDROCK_CHECKS_MAX_CONTENT_BLOCKS: Final = 10 +_ROUTER_COOLDOWNS_UNSET: Final = object() _BEDROCK_CHECKS_KNOWN_KEYS: Final = frozenset({"contentFilter", "promptAttack", "sensitiveInformation"}) # Keys in a sensitiveInformation result that pinpoint the PII location. They are # stripped before the response is handed to standard logging / telemetry so the @@ -692,8 +693,9 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): router: object, deployments: list[object], request_data: Mapping[str, object], + model: str | None = None, ) -> list[object]: - model: Final[object] = request_data.get("model") + model: Final[object] = model or request_data.get("model") chain_tag_filtering: Final[object] = ( _chain_tag_filtering_override(router, model, deployments) if isinstance(model, str) else None ) @@ -784,7 +786,15 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): fallback_default_deployments: Final = [ deployment for deployment in candidate_deployments if "default" in _deployment_tags(deployment) ] - return fallback_default_deployments + if fallback_default_deployments: + return fallback_default_deployments + if any( + isinstance(deployment, Mapping) + and (deployment.get("model_info") or {}).get("allow_fail_open") is True + for deployment in allowed_deployments + ): + return candidate_deployments or allowed_deployments + return [] @staticmethod def _router_deployment_field(deployment: object, field: str) -> object | None: @@ -794,7 +804,11 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): return model_info.get(field) if isinstance(model_info, Mapping) else getattr(model_info, field, None) @staticmethod - def _router_allows_bedrock(request_data: Mapping[str, object]) -> bool | None: + def _router_allows_bedrock( + request_data: Mapping[str, object], + *, + cooldown_deployments: Sequence[str] | None | object = _ROUTER_COOLDOWNS_UNSET, + ) -> bool | None: model: Final[object | None] = request_data.get("model") if not isinstance(model, str): return False @@ -817,11 +831,19 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): try: resolved_team_id: Final = team_id if isinstance(team_id, str) else None router_kwargs: Final = dict(request_data) + effective_model: str = model + for metadata in (request_data.get("litellm_metadata"), request_data.get("metadata")): + if not isinstance(metadata, Mapping): + continue + routing_decision: Final[object] = metadata.get("routing_decision") + if isinstance(routing_decision, Mapping) and isinstance(routing_decision.get("routed_model"), str): + effective_model = routing_decision["routed_model"] + break common_result: tuple[object, object] | None = None common_lookup: Final[object] = getattr(llm_router, "_common_checks_available_deployment", None) if callable(common_lookup): try: - raw_common_result: Final = common_lookup(model=model, request_kwargs=router_kwargs) + raw_common_result: Final = common_lookup(model=effective_model, request_kwargs=router_kwargs) except Exception: # noqa: BLE001 # fall back for lightweight router test doubles raw_common_result = None if ( @@ -830,6 +852,8 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): and isinstance(raw_common_result[1], (Mapping, list)) ): common_result = raw_common_result + if isinstance(raw_common_result[0], str): + effective_model = raw_common_result[0] if common_result is not None: raw_deployments: Final[object] = common_result[1] @@ -844,7 +868,9 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): router_matched: Final = bool(candidate_deployments) else: model_id_deployment: Final = ( - llm_router.get_deployment(model_id=model) if llm_router.has_model_id(model) is True else None + llm_router.get_deployment(model_id=effective_model) + if llm_router.has_model_id(effective_model) is True + else None ) model_id_deployment_row: Final = ( model_id_deployment.model_dump(exclude_none=True) @@ -857,14 +883,17 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): model_group_aliases: Final = getattr(llm_router, "model_group_alias", None) concrete_model_names: Final[object | None] = getattr(llm_router, "model_names", None) is_concrete_model: Final = ( - isinstance(concrete_model_names, (list, tuple, set, frozenset)) and model in concrete_model_names + isinstance(concrete_model_names, (list, tuple, set, frozenset)) + and effective_model in concrete_model_names + ) + is_model_alias: Final = ( + isinstance(model_group_aliases, Mapping) and effective_model in model_group_aliases ) - is_model_alias: Final = isinstance(model_group_aliases, Mapping) and model in model_group_aliases raw_listed_deployments = ( [] if model_id_deployment_row is not None else ( - llm_router.get_model_list(model_name=model, team_id=resolved_team_id) or [] + llm_router.get_model_list(model_name=effective_model, team_id=resolved_team_id) or [] if is_concrete_model or is_model_alias else [] ) @@ -875,7 +904,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): pattern_router, "get_deployments_by_pattern", None ) global_pattern_deployments: Final = ( - get_pattern_deployments(model=model) if callable(get_pattern_deployments) else None + get_pattern_deployments(model=effective_model) if callable(get_pattern_deployments) else None ) team_pattern_router: Final[object | None] = ( getattr(llm_router, "team_pattern_routers", {}).get(resolved_team_id) @@ -887,7 +916,9 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): team_pattern_router, "get_deployments_by_pattern", None ) team_pattern_deployments: Final = ( - get_team_pattern_deployments(model=model) if callable(get_team_pattern_deployments) else None + get_team_pattern_deployments(model=effective_model) + if callable(get_team_pattern_deployments) + else None ) if isinstance(global_pattern_deployments, list) and global_pattern_deployments: raw_listed_deployments = global_pattern_deployments @@ -900,15 +931,15 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): elif ( isinstance(deployment_names, Sequence) and not isinstance(deployment_names, (str, bytes)) - and model in deployment_names + and effective_model in deployment_names and callable(specific_lookup) ): - specific_result: Final = specific_lookup(model=model) + specific_result: Final = specific_lookup(model=effective_model) specific_deployment_rows = specific_result if isinstance(specific_result, list) else [] raw_listed_deployments = specific_deployment_rows else: raw_listed_deployments = ( - llm_router.get_model_list(model_name=model, team_id=resolved_team_id) or [] + llm_router.get_model_list(model_name=effective_model, team_id=resolved_team_id) or [] ) candidate_deployments = ( [model_id_deployment_row] @@ -931,7 +962,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): filter_deployments: Final = getattr(llm_router, "_filter_deployments_by_model_access_groups", None) filtered_deployments: Final = ( filter_deployments( - model=model, + model=effective_model, healthy_deployments=team_filtered_deployments, request_kwargs=dict(request_data), request_team_id=resolved_team_id, @@ -960,20 +991,43 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): if isinstance(health_filtered_deployments, list) else access_filtered_deployments ) + pre_call_filter: Final[object] = getattr(llm_router, "_pre_call_checks", None) + request_messages: Final[object] = request_data.get("messages") + request_input: Final[object] = request_data.get("input") + if ( + model_id_deployment_row is None + and getattr(llm_router, "enable_pre_call_checks", False) is True + and (isinstance(request_messages, list) or isinstance(request_input, (str, list))) + and callable(pre_call_filter) + ): + pre_call_deployments: Final = pre_call_filter( + model=effective_model, + healthy_deployments=healthy_deployments, + messages=request_messages if isinstance(request_messages, list) else None, + input=request_input if isinstance(request_input, (str, list)) else None, + request_kwargs=router_kwargs, + ) + if isinstance(pre_call_deployments, list): + healthy_deployments = pre_call_deployments cooldown_cache: Final[object | None] = getattr(llm_router, "cooldown_cache", None) cooldown_lookup: Final[object | None] = getattr(cooldown_cache, "get_active_cooldowns", None) - cooldown_deployments: Final = ( + resolved_cooldown_deployments: Final = ( _get_cooldown_deployments( litellm_router_instance=llm_router, parent_otel_span=None, ) - if (common_result is None or model_id_deployment_row is None) + if cooldown_deployments is _ROUTER_COOLDOWNS_UNSET + and (common_result is None or model_id_deployment_row is None) and callable(cooldown_lookup) and callable(getattr(llm_router, "get_model_ids", None)) + else cooldown_deployments + if cooldown_deployments is not _ROUTER_COOLDOWNS_UNSET else [] ) cooldown_ids: Final[frozenset[str]] = frozenset( - deployment_id for deployment_id in cooldown_deployments if isinstance(deployment_id, str) + deployment_id + for deployment_id in (resolved_cooldown_deployments or []) + if isinstance(deployment_id, str) ) if common_result is not None and model_id_deployment_row is not None: deployments = candidate_deployments @@ -992,6 +1046,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): router=llm_router, deployments=unblocked_deployments, request_data=request_data, + model=effective_model, ) if common_result is not None and model_id_deployment_row is None: web_search_deployments: Final = filter_web_search_deployments( @@ -1007,14 +1062,14 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): ) if isinstance(plugin_deployments, list): deployments = plugin_deployments - deployments = litellm.utils._get_order_filtered_deployments( - deployments, - target_order=router_kwargs.pop("_target_order", None), - ) deployments = litellm.utils._get_excluded_filtered_deployments( deployments, excluded_deployment_ids=router_kwargs.pop("_excluded_deployment_ids", None), ) + deployments = litellm.utils._get_order_filtered_deployments( + deployments, + target_order=router_kwargs.pop("_target_order", None), + ) except Exception: # noqa: BLE001 # optional router state must not break guardrail auth return False if not deployments: @@ -1024,7 +1079,11 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): if getattr(router_settings, "pass_through_all_models", False) is True: provider = request_data.get("custom_llm_provider") if not isinstance(provider, str): - provider = BedrockGuardrail._resolve_model_provider(model) if isinstance(model, str) else None + provider = ( + BedrockGuardrail._resolve_model_provider(effective_model) + if isinstance(effective_model, str) + else None + ) return provider in ("bedrock", "bedrock_converse") if isinstance(provider, str) else None default_deployment = getattr(llm_router, "default_deployment", None) @@ -1056,7 +1115,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): default_fallback_model: Final[object] = ( default_fallback_lookup() if callable(default_fallback_lookup) else None ) - if isinstance(default_fallback_model, str) and default_fallback_model != model: + if isinstance(default_fallback_model, str) and default_fallback_model != effective_model: fallback_deployments: Final = ( llm_router.get_model_list( model_name=default_fallback_model, @@ -1097,6 +1156,107 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): providers.append(provider) return bool(providers) and all(provider in ("bedrock", "bedrock_converse") for provider in providers) + @staticmethod + async def _async_get_bedrock_api_key(request_data: Mapping[str, object] | None) -> str | None: + if not request_data: + return None + + api_key: Final[object | None] = request_data.get("api_key") + if not isinstance(api_key, str): + return None + + explicit_provider: Final[object | None] = request_data.get("custom_llm_provider") + if isinstance(explicit_provider, str) and explicit_provider not in ("bedrock", "bedrock_converse"): + return None + + try: + from litellm.proxy.proxy_server import llm_router + except ImportError: + llm_router = None + + if llm_router is not None: + async_lookup: Final[object] = getattr(llm_router, "async_get_healthy_deployments", None) + if callable(async_lookup): + model: Final[object | None] = request_data.get("model") + if isinstance(model, str): + effective_model = model + for metadata in (request_data.get("litellm_metadata"), request_data.get("metadata")): + if not isinstance(metadata, Mapping): + continue + routing_decision: Final[object] = metadata.get("routing_decision") + if isinstance(routing_decision, Mapping) and isinstance( + routing_decision.get("routed_model"), str + ): + effective_model = routing_decision["routed_model"] + break + try: + healthy_deployments: Final = await async_lookup( + model=effective_model, + request_kwargs=dict(request_data), + messages=request_data.get("messages") + if isinstance(request_data.get("messages"), list) + else None, + input=request_data.get("input") + if isinstance(request_data.get("input"), (str, list)) + else None, + specific_deployment=request_data.get("specific_deployment") is True, + ) + deployments: Final[list[object]] = ( + [healthy_deployments] + if isinstance(healthy_deployments, Mapping) + else healthy_deployments + if isinstance(healthy_deployments, list) + else [] + ) + if deployments: + providers: list[str] = [] + for deployment in deployments: + params: object = ( + deployment.get("litellm_params") + if isinstance(deployment, Mapping) + else getattr(deployment, "litellm_params", None) + ) + provider: object = ( + params.get("custom_llm_provider") + if isinstance(params, Mapping) + else getattr(params, "custom_llm_provider", None) + ) + if not isinstance(provider, str): + deployment_model: object = ( + params.get("model") + if isinstance(params, Mapping) + else getattr(params, "model", None) + ) + provider = ( + BedrockGuardrail._resolve_model_provider(deployment_model) + if isinstance(deployment_model, str) + else None + ) + if not isinstance(provider, str): + return None + providers.append(provider) + return api_key if all( + provider in ("bedrock", "bedrock_converse") for provider in providers + ) else None + except Exception: # noqa: BLE001 # fall back to the sync compatibility path + pass + + router_allows_bedrock: Final = BedrockGuardrail._router_allows_bedrock( + request_data, + cooldown_deployments=[], + ) + if router_allows_bedrock is not None: + return api_key if router_allows_bedrock else None + + if isinstance(explicit_provider, str): + return api_key if explicit_provider in ("bedrock", "bedrock_converse") else None + + model: Final[object | None] = request_data.get("model") + model_provider: Final[str | None] = ( + BedrockGuardrail._resolve_model_provider(model) if isinstance(model, str) else None + ) + return api_key if model_provider in ("bedrock", "bedrock_converse") else None + @staticmethod def _get_bedrock_api_key(request_data: Mapping[str, object] | None) -> str | None: if not request_data: @@ -1295,7 +1455,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): bedrock_request_data: Final[dict] = dict( self.convert_to_bedrock_format(source=source, messages=messages, response=response) ) - api_key: Final = self._get_bedrock_api_key(request_data) + api_key: Final = await self._async_get_bedrock_api_key(request_data) if request_data: dynamic_request_body_params = self.get_guardrail_dynamic_request_body_params(request_data=request_data) bedrock_request_data.update( @@ -2280,7 +2440,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): credentials, aws_region_name = self._load_credentials() body: Final[dict[str, Any]] = {"messages": checks_messages, "checks": self.checks} - api_key: Final = self._get_bedrock_api_key(request_data) + api_key: Final = await self._async_get_bedrock_api_key(request_data) prepared_request: Final = self._prepare_request( credentials=credentials, From e673f4b48ede77bfb9b04fc2a1d17609cc91e41a Mon Sep 17 00:00:00 2001 From: aiedwardyi <41576951+aiedwardyi@users.noreply.github.com> Date: Wed, 26 Aug 2026 11:43:33 +0900 Subject: [PATCH 12/21] fix: harden Bedrock pre-call routing --- .../guardrail_hooks/bedrock_guardrails.py | 99 +++++++++++----- .../test_bedrock_guardrails.py | 109 ++++++++++++++++++ 2 files changed, 179 insertions(+), 29 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 2e0a2eb68ec..82706c04c1d 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -704,8 +704,13 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): if isinstance(chain_tag_filtering, bool) else getattr(router, "enable_tag_filtering", False) ) + router_settings_override: Final[object] = request_data.get("router_settings_override") + trusted_request_tag_filtering: Final[bool] = ( + isinstance(router_settings_override, Mapping) + and router_settings_override.get("enable_tag_filtering") is True + ) tag_filtering_enabled: Final = ( - request_data.get("enable_tag_filtering") is True or effective_tag_filtering is True + trusted_request_tag_filtering or effective_tag_filtering is True ) if not tag_filtering_enabled: return deployments @@ -796,6 +801,21 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): return candidate_deployments or allowed_deployments return [] + @staticmethod + def _get_trusted_router_request_kwargs(request_data: Mapping[str, object]) -> dict[str, object]: + router_kwargs: Final = dict(request_data) + router_kwargs.pop("enable_tag_filtering", None) + router_settings_override: Final[object] = request_data.get("router_settings_override") + if isinstance(router_settings_override, Mapping) and router_settings_override.get("enable_tag_filtering") is True: + router_kwargs["enable_tag_filtering"] = True + for metadata_name in ("metadata", "litellm_metadata"): + metadata: Final[object] = router_kwargs.get(metadata_name) + if isinstance(metadata, Mapping) and "routing_decision" in metadata: + router_kwargs[metadata_name] = { + key: value for key, value in metadata.items() if key != "routing_decision" + } + return router_kwargs + @staticmethod def _router_deployment_field(deployment: object, field: str) -> object | None: model_info: Final[object | None] = ( @@ -808,6 +828,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): request_data: Mapping[str, object], *, cooldown_deployments: Sequence[str] | None | object = _ROUTER_COOLDOWNS_UNSET, + apply_tag_filtering: bool = True, ) -> bool | None: model: Final[object | None] = request_data.get("model") if not isinstance(model, str): @@ -830,15 +851,8 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): ) try: resolved_team_id: Final = team_id if isinstance(team_id, str) else None - router_kwargs: Final = dict(request_data) + router_kwargs: Final = BedrockGuardrail._get_trusted_router_request_kwargs(request_data) effective_model: str = model - for metadata in (request_data.get("litellm_metadata"), request_data.get("metadata")): - if not isinstance(metadata, Mapping): - continue - routing_decision: Final[object] = metadata.get("routing_decision") - if isinstance(routing_decision, Mapping) and isinstance(routing_decision.get("routed_model"), str): - effective_model = routing_decision["routed_model"] - break common_result: tuple[object, object] | None = None common_lookup: Final[object] = getattr(llm_router, "_common_checks_available_deployment", None) if callable(common_lookup): @@ -1042,11 +1056,15 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): for deployment in cooldown_filtered_deployments if BedrockGuardrail._router_deployment_field(deployment, "blocked") is not True ] - deployments = BedrockGuardrail._filter_router_deployments_by_tags( - router=llm_router, - deployments=unblocked_deployments, - request_data=request_data, - model=effective_model, + deployments = ( + BedrockGuardrail._filter_router_deployments_by_tags( + router=llm_router, + deployments=unblocked_deployments, + request_data=request_data, + model=effective_model, + ) + if apply_tag_filtering + else unblocked_deployments ) if common_result is not None and model_id_deployment_row is None: web_search_deployments: Final = filter_web_search_deployments( @@ -1126,7 +1144,10 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): if isinstance(fallback_deployments, list) and fallback_deployments: fallback_request_data: Final = dict(request_data) fallback_request_data["model"] = default_fallback_model - return BedrockGuardrail._router_allows_bedrock(fallback_request_data) + return BedrockGuardrail._router_allows_bedrock( + fallback_request_data, + apply_tag_filtering=apply_tag_filtering, + ) return False @@ -1175,29 +1196,49 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): llm_router = None if llm_router is not None: + router_request_kwargs: Final = BedrockGuardrail._get_trusted_router_request_kwargs(request_data) + routing_strategy: object | None = getattr(llm_router, "routing_strategy", None) + routing_context: Final[object] = getattr(llm_router, "_get_routing_context", None) + if callable(routing_context): + try: + context_result: Final = routing_context( + model=request_data.get("model"), request_kwargs=router_request_kwargs + ) + if isinstance(context_result, tuple) and context_result: + routing_strategy = context_result[0] + except Exception: # noqa: BLE001 # fall back to the router default + pass + if hasattr(routing_strategy, "value"): + routing_strategy = routing_strategy.value + if isinstance(routing_strategy, str) and routing_strategy not in { + "usage-based-routing-v2", + "simple-shuffle", + "cost-based-routing", + "latency-based-routing", + "least-busy", + }: + router_allows_bedrock: Final = BedrockGuardrail._router_allows_bedrock( + request_data, + cooldown_deployments=[], + apply_tag_filtering=False, + ) + if router_allows_bedrock is not None: + return api_key if router_allows_bedrock else None + async_lookup: Final[object] = getattr(llm_router, "async_get_healthy_deployments", None) if callable(async_lookup): model: Final[object | None] = request_data.get("model") if isinstance(model, str): effective_model = model - for metadata in (request_data.get("litellm_metadata"), request_data.get("metadata")): - if not isinstance(metadata, Mapping): - continue - routing_decision: Final[object] = metadata.get("routing_decision") - if isinstance(routing_decision, Mapping) and isinstance( - routing_decision.get("routed_model"), str - ): - effective_model = routing_decision["routed_model"] - break try: healthy_deployments: Final = await async_lookup( model=effective_model, - request_kwargs=dict(request_data), - messages=request_data.get("messages") - if isinstance(request_data.get("messages"), list) + request_kwargs=router_request_kwargs, + messages=router_request_kwargs.get("messages") + if isinstance(router_request_kwargs.get("messages"), list) else None, - input=request_data.get("input") - if isinstance(request_data.get("input"), (str, list)) + input=router_request_kwargs.get("input") + if isinstance(router_request_kwargs.get("input"), (str, list)) else None, specific_deployment=request_data.get("specific_deployment") is True, ) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index 5249fc29a6e..e14d1b99cff 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -249,6 +249,115 @@ def test_bedrock_guardrail_matches_request_tag_pool(monkeypatch: pytest.MonkeyPa ) +def test_bedrock_guardrail_ignores_client_routing_decision(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + + router = MagicMock() + router.get_model_list.return_value = [ + {"litellm_params": {"custom_llm_provider": "openai"}, "model_info": {}}, + ] + monkeypatch.setattr(proxy_server, "llm_router", router) + + assert ( + BedrockGuardrail._get_bedrock_api_key( + { + "model": "shared-alias", + "api_key": "bedrock-key", + "metadata": {"routing_decision": {"routed_model": "bedrock-alias"}}, + } + ) + is None + ) + + +def test_bedrock_guardrail_ignores_client_tag_filtering_override(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + + router = MagicMock() + router.enable_tag_filtering = False + router.get_model_list.return_value = [ + {"litellm_params": {"custom_llm_provider": "openai", "tags": ["slow"]}, "model_info": {}}, + {"litellm_params": {"custom_llm_provider": "bedrock", "tags": ["fast"]}, "model_info": {}}, + ] + monkeypatch.setattr(proxy_server, "llm_router", router) + + assert ( + BedrockGuardrail._get_bedrock_api_key( + { + "model": "shared-alias", + "api_key": "bedrock-key", + "enable_tag_filtering": True, + "metadata": {"tags": ["fast"]}, + } + ) + is None + ) + + +def test_bedrock_guardrail_uses_trusted_tag_filtering_override(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + + router = MagicMock() + router.enable_tag_filtering = False + router.get_model_list.return_value = [ + {"litellm_params": {"custom_llm_provider": "openai", "tags": ["slow"]}, "model_info": {}}, + {"litellm_params": {"custom_llm_provider": "bedrock", "tags": ["fast"]}, "model_info": {}}, + ] + monkeypatch.setattr(proxy_server, "llm_router", router) + + assert ( + BedrockGuardrail._get_bedrock_api_key( + { + "model": "shared-alias", + "api_key": "bedrock-key", + "router_settings_override": {"enable_tag_filtering": True}, + "metadata": {"tags": ["fast"]}, + } + ) + == "bedrock-key" + ) + + +@pytest.mark.asyncio +async def test_bedrock_guardrail_async_sync_strategy_uses_unfiltered_pool(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + + router = MagicMock() + router.routing_strategy = "usage-based-routing" + router.model_names = ["shared-alias"] + router.model_group_alias = {} + router.has_model_id.return_value = False + router.get_model_list.return_value = [ + {"litellm_params": {"custom_llm_provider": "openai", "tags": ["slow"]}, "model_info": {}}, + {"litellm_params": {"custom_llm_provider": "bedrock", "tags": ["fast"]}, "model_info": {}}, + ] + router._filter_health_check_unhealthy_deployments.side_effect = lambda healthy_deployments, **_: healthy_deployments + router._filter_deployments_by_model_access_groups.side_effect = ( + lambda **kwargs: kwargs["healthy_deployments"] + ) + router.get_model_ids.return_value = [] + router.cooldown_cache.get_active_cooldowns.return_value = [] + router.pattern_router = None + router.default_deployment = None + router.router_general_settings.pass_through_all_models = False + router.async_get_healthy_deployments = AsyncMock( + return_value=[{"litellm_params": {"custom_llm_provider": "bedrock"}, "model_info": {}}] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + + assert ( + await BedrockGuardrail._async_get_bedrock_api_key( + { + "model": "shared-alias", + "api_key": "bedrock-key", + "metadata": {"tags": ["fast"]}, + } + ) + is None + ) + router.async_get_healthy_deployments.assert_not_awaited() + + def test_bedrock_guardrail_matches_regex_tag_pool(monkeypatch: pytest.MonkeyPatch): from litellm.proxy import proxy_server From 7b24291f24a344d9825dcf4c005a35bf2931dfff Mon Sep 17 00:00:00 2001 From: aiedwardyi <41576951+aiedwardyi@users.noreply.github.com> Date: Wed, 26 Aug 2026 11:57:47 +0900 Subject: [PATCH 13/21] fix: align Bedrock tag eligibility --- .../guardrail_hooks/bedrock_guardrails.py | 12 +--------- .../test_bedrock_guardrails.py | 24 ------------------- 2 files changed, 1 insertion(+), 35 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 82706c04c1d..e329875fb47 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -704,14 +704,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): if isinstance(chain_tag_filtering, bool) else getattr(router, "enable_tag_filtering", False) ) - router_settings_override: Final[object] = request_data.get("router_settings_override") - trusted_request_tag_filtering: Final[bool] = ( - isinstance(router_settings_override, Mapping) - and router_settings_override.get("enable_tag_filtering") is True - ) - tag_filtering_enabled: Final = ( - trusted_request_tag_filtering or effective_tag_filtering is True - ) + tag_filtering_enabled: Final = effective_tag_filtering is True if not tag_filtering_enabled: return deployments @@ -805,9 +798,6 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): def _get_trusted_router_request_kwargs(request_data: Mapping[str, object]) -> dict[str, object]: router_kwargs: Final = dict(request_data) router_kwargs.pop("enable_tag_filtering", None) - router_settings_override: Final[object] = request_data.get("router_settings_override") - if isinstance(router_settings_override, Mapping) and router_settings_override.get("enable_tag_filtering") is True: - router_kwargs["enable_tag_filtering"] = True for metadata_name in ("metadata", "litellm_metadata"): metadata: Final[object] = router_kwargs.get(metadata_name) if isinstance(metadata, Mapping) and "routing_decision" in metadata: diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index e14d1b99cff..30a899ab9ef 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -294,30 +294,6 @@ def test_bedrock_guardrail_ignores_client_tag_filtering_override(monkeypatch: py ) -def test_bedrock_guardrail_uses_trusted_tag_filtering_override(monkeypatch: pytest.MonkeyPatch): - from litellm.proxy import proxy_server - - router = MagicMock() - router.enable_tag_filtering = False - router.get_model_list.return_value = [ - {"litellm_params": {"custom_llm_provider": "openai", "tags": ["slow"]}, "model_info": {}}, - {"litellm_params": {"custom_llm_provider": "bedrock", "tags": ["fast"]}, "model_info": {}}, - ] - monkeypatch.setattr(proxy_server, "llm_router", router) - - assert ( - BedrockGuardrail._get_bedrock_api_key( - { - "model": "shared-alias", - "api_key": "bedrock-key", - "router_settings_override": {"enable_tag_filtering": True}, - "metadata": {"tags": ["fast"]}, - } - ) - == "bedrock-key" - ) - - @pytest.mark.asyncio async def test_bedrock_guardrail_async_sync_strategy_uses_unfiltered_pool(monkeypatch: pytest.MonkeyPatch): from litellm.proxy import proxy_server From 34ae7695901ed2162a908c639e6ebde7f4bf0d3b Mon Sep 17 00:00:00 2001 From: aiedwardyi <41576951+aiedwardyi@users.noreply.github.com> Date: Wed, 26 Aug 2026 12:32:11 +0900 Subject: [PATCH 14/21] fix: align Bedrock router checks --- .../guardrail_hooks/bedrock_guardrails.py | 109 ++++++++---------- 1 file changed, 47 insertions(+), 62 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index e329875fb47..56cfb87dc6f 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -781,11 +781,6 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): ] if matched_deployments: return matched_deployments - fallback_default_deployments: Final = [ - deployment for deployment in candidate_deployments if "default" in _deployment_tags(deployment) - ] - if fallback_default_deployments: - return fallback_default_deployments if any( isinstance(deployment, Mapping) and (deployment.get("model_info") or {}).get("allow_fail_open") is True @@ -813,6 +808,29 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): ) return model_info.get(field) if isinstance(model_info, Mapping) else getattr(model_info, field, None) + @staticmethod + def _router_deployment_provider(deployment: object) -> str | None: + params: Final[object] = ( + deployment.get("litellm_params") + if isinstance(deployment, Mapping) + else getattr(deployment, "litellm_params", None) + ) + provider: Final[object] = ( + params.get("custom_llm_provider") + if isinstance(params, Mapping) + else getattr(params, "custom_llm_provider", None) + ) + if isinstance(provider, str): + return provider + deployment_model: Final[object] = ( + params.get("model") if isinstance(params, Mapping) else getattr(params, "model", None) + ) + return ( + BedrockGuardrail._resolve_model_provider(deployment_model) + if isinstance(deployment_model, str) + else None + ) + @staticmethod def _router_allows_bedrock( request_data: Mapping[str, object], @@ -824,6 +842,11 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): if not isinstance(model, str): return False + selected_deployment: Final[object | None] = request_data.get("deployment") + if selected_deployment is not None: + selected_provider: Final[str | None] = BedrockGuardrail._router_deployment_provider(selected_deployment) + return selected_provider in ("bedrock", "bedrock_converse") if selected_provider is not None else False + try: from litellm.proxy.proxy_server import llm_router except ImportError: @@ -847,7 +870,11 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): common_lookup: Final[object] = getattr(llm_router, "_common_checks_available_deployment", None) if callable(common_lookup): try: - raw_common_result: Final = common_lookup(model=effective_model, request_kwargs=router_kwargs) + raw_common_result: Final = common_lookup( + model=effective_model, + request_kwargs=router_kwargs, + specific_deployment=request_data.get("specific_deployment") is True, + ) except Exception: # noqa: BLE001 # fall back for lightweight router test doubles raw_common_result = None if ( @@ -1070,14 +1097,14 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): ) if isinstance(plugin_deployments, list): deployments = plugin_deployments - deployments = litellm.utils._get_excluded_filtered_deployments( - deployments, - excluded_deployment_ids=router_kwargs.pop("_excluded_deployment_ids", None), - ) deployments = litellm.utils._get_order_filtered_deployments( deployments, target_order=router_kwargs.pop("_target_order", None), ) + deployments = litellm.utils._get_excluded_filtered_deployments( + deployments, + excluded_deployment_ids=router_kwargs.pop("_excluded_deployment_ids", None), + ) except Exception: # noqa: BLE001 # optional router state must not break guardrail auth return False if not deployments: @@ -1143,26 +1170,8 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): providers: list[str] = [] for deployment in deployments: - params: object = ( - deployment.get("litellm_params") - if isinstance(deployment, Mapping) - else getattr(deployment, "litellm_params", None) - ) - provider: object = ( - params.get("custom_llm_provider") - if isinstance(params, Mapping) - else getattr(params, "custom_llm_provider", None) - ) - if not isinstance(provider, str): - deployment_model: Final[object | None] = ( - params.get("model") if isinstance(params, Mapping) else getattr(params, "model", None) - ) - provider = ( - BedrockGuardrail._resolve_model_provider(deployment_model) - if isinstance(deployment_model, str) - else None - ) - if not isinstance(provider, str): + provider: Final[str | None] = BedrockGuardrail._router_deployment_provider(deployment) + if provider is None: return False providers.append(provider) return bool(providers) and all(provider in ("bedrock", "bedrock_converse") for provider in providers) @@ -1188,16 +1197,6 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): if llm_router is not None: router_request_kwargs: Final = BedrockGuardrail._get_trusted_router_request_kwargs(request_data) routing_strategy: object | None = getattr(llm_router, "routing_strategy", None) - routing_context: Final[object] = getattr(llm_router, "_get_routing_context", None) - if callable(routing_context): - try: - context_result: Final = routing_context( - model=request_data.get("model"), request_kwargs=router_request_kwargs - ) - if isinstance(context_result, tuple) and context_result: - routing_strategy = context_result[0] - except Exception: # noqa: BLE001 # fall back to the router default - pass if hasattr(routing_strategy, "value"): routing_strategy = routing_strategy.value if isinstance(routing_strategy, str) and routing_strategy not in { @@ -1242,30 +1241,16 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): if deployments: providers: list[str] = [] for deployment in deployments: - params: object = ( - deployment.get("litellm_params") - if isinstance(deployment, Mapping) - else getattr(deployment, "litellm_params", None) - ) - provider: object = ( - params.get("custom_llm_provider") - if isinstance(params, Mapping) - else getattr(params, "custom_llm_provider", None) - ) - if not isinstance(provider, str): - deployment_model: object = ( - params.get("model") - if isinstance(params, Mapping) - else getattr(params, "model", None) - ) - provider = ( - BedrockGuardrail._resolve_model_provider(deployment_model) - if isinstance(deployment_model, str) - else None - ) - if not isinstance(provider, str): + provider: Final[str | None] = BedrockGuardrail._router_deployment_provider(deployment) + if provider is None: return None providers.append(provider) + router_allows_bedrock: Final = BedrockGuardrail._router_allows_bedrock( + request_data, + cooldown_deployments=[], + ) + if router_allows_bedrock is not None: + return api_key if router_allows_bedrock else None return api_key if all( provider in ("bedrock", "bedrock_converse") for provider in providers ) else None From 336003cc3b9ae12e2f2b6c25ddeb54f9e752a5ef Mon Sep 17 00:00:00 2001 From: aiedwardyi <41576951+aiedwardyi@users.noreply.github.com> Date: Wed, 26 Aug 2026 12:38:57 +0900 Subject: [PATCH 15/21] chore: format Bedrock guardrail --- .../guardrail_hooks/bedrock_guardrails.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 56cfb87dc6f..18a259b1eb3 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -782,8 +782,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): if matched_deployments: return matched_deployments if any( - isinstance(deployment, Mapping) - and (deployment.get("model_info") or {}).get("allow_fail_open") is True + isinstance(deployment, Mapping) and (deployment.get("model_info") or {}).get("allow_fail_open") is True for deployment in allowed_deployments ): return candidate_deployments or allowed_deployments @@ -825,11 +824,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): deployment_model: Final[object] = ( params.get("model") if isinstance(params, Mapping) else getattr(params, "model", None) ) - return ( - BedrockGuardrail._resolve_model_provider(deployment_model) - if isinstance(deployment_model, str) - else None - ) + return BedrockGuardrail._resolve_model_provider(deployment_model) if isinstance(deployment_model, str) else None @staticmethod def _router_allows_bedrock( @@ -1251,9 +1246,11 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): ) if router_allows_bedrock is not None: return api_key if router_allows_bedrock else None - return api_key if all( - provider in ("bedrock", "bedrock_converse") for provider in providers - ) else None + return ( + api_key + if all(provider in ("bedrock", "bedrock_converse") for provider in providers) + else None + ) except Exception: # noqa: BLE001 # fall back to the sync compatibility path pass From 455879c4e5ce89a2774b2a4ee75dfeb685ac2e7c Mon Sep 17 00:00:00 2001 From: aiedwardyi <41576951+aiedwardyi@users.noreply.github.com> Date: Wed, 26 Aug 2026 12:40:59 +0900 Subject: [PATCH 16/21] fix: trust router-owned deployment state --- litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 18a259b1eb3..bcd0ef738c7 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -838,7 +838,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): return False selected_deployment: Final[object | None] = request_data.get("deployment") - if selected_deployment is not None: + if selected_deployment is not None and not isinstance(selected_deployment, Mapping): selected_provider: Final[str | None] = BedrockGuardrail._router_deployment_provider(selected_deployment) return selected_provider in ("bedrock", "bedrock_converse") if selected_provider is not None else False From 6183bc25a184873bcd50d14f9c71e7af29cc6ba1 Mon Sep 17 00:00:00 2001 From: aiedwardyi <41576951+aiedwardyi@users.noreply.github.com> Date: Wed, 26 Aug 2026 12:58:46 +0900 Subject: [PATCH 17/21] fix: honor Bedrock tag routing constraints --- .../guardrail_hooks/bedrock_guardrails.py | 12 +++- .../test_bedrock_guardrails.py | 55 +++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index bcd0ef738c7..5f5d2d231b1 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -699,8 +699,16 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): chain_tag_filtering: Final[object] = ( _chain_tag_filtering_override(router, model, deployments) if isinstance(model, str) else None ) + router_settings_override: Final[object] = request_data.get("router_settings_override") + override_tag_filtering: Final[object] = ( + router_settings_override.get("enable_tag_filtering") + if isinstance(router_settings_override, Mapping) + else None + ) effective_tag_filtering: Final = ( - chain_tag_filtering + True + if override_tag_filtering is True + else chain_tag_filtering if isinstance(chain_tag_filtering, bool) else getattr(router, "enable_tag_filtering", False) ) @@ -757,6 +765,8 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): bool(header_strings) and has_regex_deployments and not required_set ) if not has_positive_filter: + if required_set or excluded_set: + return candidate_deployments default_deployments: Final = [ deployment for deployment in candidate_deployments if "default" in _deployment_tags(deployment) ] diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index 30a899ab9ef..d70f7bbd94b 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -294,6 +294,61 @@ def test_bedrock_guardrail_ignores_client_tag_filtering_override(monkeypatch: py ) +def test_bedrock_guardrail_honors_router_settings_tag_filtering_override(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + + router = MagicMock() + router.enable_tag_filtering = False + router.get_model_list.return_value = [ + { + "litellm_params": {"custom_llm_provider": "openai", "tags": ["slow"]}, + "model_info": {}, + }, + { + "litellm_params": {"custom_llm_provider": "bedrock", "tags": ["fast"]}, + "model_info": {}, + }, + ] + monkeypatch.setattr(proxy_server, "llm_router", router) + + assert ( + BedrockGuardrail._get_bedrock_api_key( + { + "model": "shared-alias", + "api_key": "bedrock-key", + "router_settings_override": {"enable_tag_filtering": True}, + "metadata": {"tags": ["fast"]}, + } + ) + == "bedrock-key" + ) + + +def test_bedrock_guardrail_keeps_all_required_tag_matches(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + + router = MagicMock() + router.enable_tag_filtering = True + router.get_model_list.return_value = [ + { + "litellm_params": {"custom_llm_provider": "bedrock", "tags": ["required", "default"]}, + "model_info": {}, + }, + { + "litellm_params": {"custom_llm_provider": "openai", "tags": ["required"]}, + "model_info": {}, + }, + ] + monkeypatch.setattr(proxy_server, "llm_router", router) + + assert ( + BedrockGuardrail._router_allows_bedrock( + {"model": "shared-alias", "metadata": {"tags": ["&required"]}} + ) + is False + ) + + @pytest.mark.asyncio async def test_bedrock_guardrail_async_sync_strategy_uses_unfiltered_pool(monkeypatch: pytest.MonkeyPatch): from litellm.proxy import proxy_server From 053a26a57ae3eee1b984a98179162f3b19da7243 Mon Sep 17 00:00:00 2001 From: aiedwardyi <41576951+aiedwardyi@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:18:59 +0900 Subject: [PATCH 18/21] fix: align Bedrock tag fallback --- .../guardrail_hooks/bedrock_guardrails.py | 57 ++++++++++---- .../test_bedrock_guardrails.py | 77 +++++++++++++++++++ 2 files changed, 120 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 5f5d2d231b1..4af516abae2 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -57,10 +57,12 @@ from litellm.proxy.guardrails.anthropic_sse import ( from litellm.router_strategy.tag_based_routing import ( _chain_tag_filtering_override, _get_tags_from_request_kwargs, + _inherited_constraint_sets, _match_deployment, _request_tags_after_router_consumption, _split_tags, _strip_routing_prefix, + _unknown_required_tag_hides_an_answer, ) from litellm.router_utils.common_utils import filter_team_based_models, filter_web_search_deployments from litellm.router_utils.cooldown_handlers import _get_cooldown_deployments @@ -744,10 +746,13 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): request_tags = _request_tags_after_router_consumption(metadata, model) or () routing_prefix: Final[object] = getattr(router, "tag_routing_prefix", "") resolved_prefix: Final[str] = routing_prefix if isinstance(routing_prefix, str) else "" - rewritten_tags, _ = _strip_routing_prefix(request_tags, resolved_prefix) + rewritten_tags, routing_confirmed = _strip_routing_prefix(request_tags, resolved_prefix) required_tags, positive_tags, excluded_tags = _split_tags(rewritten_tags) required_set: Final = frozenset(required_tags) excluded_set: Final = frozenset(excluded_tags) + inherited_required_set, inherited_excluded_set = _inherited_constraint_sets( + metadata.get("inherited_tags"), resolved_prefix + ) allowed_deployments: Final = [ deployment for deployment in deployments if not excluded_set.intersection(_deployment_tags(deployment)) ] @@ -764,9 +769,41 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): has_positive_filter: Final = bool(positive_tags) or ( bool(header_strings) and has_regex_deployments and not required_set ) + + def _fail_open_deployments() -> list[object]: + if _unknown_required_tag_hides_an_answer( + deployments, + excluded_set, + required_set, + routing_confirmed, + ): + return [] + if not any( + isinstance(deployment, Mapping) + and (deployment.get("model_info") or {}).get("allow_fail_open") is True + for deployment in deployments + ): + return [] + trusted_excluded: Final = ( + frozenset() if inherited_excluded_set is None else inherited_excluded_set & excluded_set + ) + trusted_required: Final = ( + frozenset() if inherited_required_set is None else inherited_required_set & required_set + ) + trusted_deployments: Final = [ + deployment + for deployment in deployments + if not trusted_excluded.intersection(_deployment_tags(deployment)) + and trusted_required.issubset(_deployment_tags(deployment)) + ] + default_deployments: Final = [ + deployment for deployment in trusted_deployments if "default" in _deployment_tags(deployment) + ] + return default_deployments or trusted_deployments + if not has_positive_filter: if required_set or excluded_set: - return candidate_deployments + return candidate_deployments or _fail_open_deployments() default_deployments: Final = [ deployment for deployment in candidate_deployments if "default" in _deployment_tags(deployment) ] @@ -791,12 +828,10 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): ] if matched_deployments: return matched_deployments - if any( - isinstance(deployment, Mapping) and (deployment.get("model_info") or {}).get("allow_fail_open") is True - for deployment in allowed_deployments - ): - return candidate_deployments or allowed_deployments - return [] + default_deployments: Final = [ + deployment for deployment in candidate_deployments if "default" in _deployment_tags(deployment) + ] + return default_deployments or _fail_open_deployments() @staticmethod def _get_trusted_router_request_kwargs(request_data: Mapping[str, object]) -> dict[str, object]: @@ -1250,12 +1285,6 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): if provider is None: return None providers.append(provider) - router_allows_bedrock: Final = BedrockGuardrail._router_allows_bedrock( - request_data, - cooldown_deployments=[], - ) - if router_allows_bedrock is not None: - return api_key if router_allows_bedrock else None return ( api_key if all(provider in ("bedrock", "bedrock_converse") for provider in providers) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index d70f7bbd94b..080c32ed317 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -349,6 +349,61 @@ def test_bedrock_guardrail_keeps_all_required_tag_matches(monkeypatch: pytest.Mo ) +def test_bedrock_guardrail_mirrors_router_fail_open_default(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + + router = MagicMock() + router.enable_tag_filtering = True + router.get_model_list.return_value = [ + { + "litellm_params": {"custom_llm_provider": "bedrock", "tags": ["caller-only"]}, + "model_info": {"allow_fail_open": True}, + }, + { + "litellm_params": {"custom_llm_provider": "openai", "tags": ["default"]}, + "model_info": {}, + }, + ] + router._get_all_deployments.return_value = router.get_model_list.return_value + monkeypatch.setattr(proxy_server, "llm_router", router) + + assert ( + BedrockGuardrail._router_allows_bedrock( + { + "model": "shared-alias", + "metadata": {"tags": ["&caller-only", "unmatched"], "inherited_tags": []}, + } + ) + is False + ) + + +def test_bedrock_guardrail_preserves_default_for_unknown_tag(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + + router = MagicMock() + router.enable_tag_filtering = True + router.get_model_list.return_value = [ + { + "litellm_params": {"custom_llm_provider": "openai", "tags": ["other"]}, + "model_info": {}, + }, + { + "litellm_params": {"custom_llm_provider": "bedrock", "tags": ["default"]}, + "model_info": {}, + }, + ] + router._get_all_deployments.return_value = router.get_model_list.return_value + monkeypatch.setattr(proxy_server, "llm_router", router) + + assert ( + BedrockGuardrail._router_allows_bedrock( + {"model": "shared-alias", "metadata": {"tags": ["unknown"]}} + ) + is True + ) + + @pytest.mark.asyncio async def test_bedrock_guardrail_async_sync_strategy_uses_unfiltered_pool(monkeypatch: pytest.MonkeyPatch): from litellm.proxy import proxy_server @@ -389,6 +444,28 @@ async def test_bedrock_guardrail_async_sync_strategy_uses_unfiltered_pool(monkey router.async_get_healthy_deployments.assert_not_awaited() +@pytest.mark.asyncio +async def test_bedrock_guardrail_async_uses_callback_filtered_pool(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + + router = MagicMock() + router.routing_strategy = "usage-based-routing-v2" + router.async_get_healthy_deployments = AsyncMock( + return_value=[{"litellm_params": {"custom_llm_provider": "bedrock"}, "model_info": {}}] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + + with patch.object(BedrockGuardrail, "_router_allows_bedrock", return_value=False) as router_allows: + assert ( + await BedrockGuardrail._async_get_bedrock_api_key( + {"model": "shared-alias", "api_key": "bedrock-key"} + ) + == "bedrock-key" + ) + + router_allows.assert_not_called() + + def test_bedrock_guardrail_matches_regex_tag_pool(monkeypatch: pytest.MonkeyPatch): from litellm.proxy import proxy_server From 241c168071a90d240860edf130dc3dde90297079 Mon Sep 17 00:00:00 2001 From: aiedwardyi <41576951+aiedwardyi@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:49:44 +0900 Subject: [PATCH 19/21] fix: close routing review gaps --- .../guardrail_hooks/bedrock_guardrails.py | 80 +++++++++++++++++-- .../test_bedrock_guardrails.py | 73 +++++++++++++++++ 2 files changed, 145 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 4af516abae2..cc25b6ee56f 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -839,10 +839,16 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): router_kwargs.pop("enable_tag_filtering", None) for metadata_name in ("metadata", "litellm_metadata"): metadata: Final[object] = router_kwargs.get(metadata_name) - if isinstance(metadata, Mapping) and "routing_decision" in metadata: + if isinstance(metadata, Mapping): router_kwargs[metadata_name] = { key: value for key, value in metadata.items() if key != "routing_decision" } + router_settings_override: Final[object] = router_kwargs.get("router_settings_override") + if ( + isinstance(router_settings_override, Mapping) + and router_settings_override.get("enable_tag_filtering") is True + ): + router_kwargs["enable_tag_filtering"] = True return router_kwargs @staticmethod @@ -871,6 +877,37 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): ) return BedrockGuardrail._resolve_model_provider(deployment_model) if isinstance(deployment_model, str) else None + @staticmethod + def _router_deployments_for_provider_check(deployments: Sequence[object]) -> list[object]: + if not deployments: + return [] + + def _weight(deployment: object, weight_by: str) -> object | None: + params: Final[object] = ( + deployment.get("litellm_params") + if isinstance(deployment, Mapping) + else getattr(deployment, "litellm_params", None) + ) + return ( + params.get(weight_by) + if isinstance(params, Mapping) + else getattr(params, weight_by, None) + ) + + for weight_by in ("weight", "rpm", "tpm"): + first_weight: Final[object | None] = _weight(deployments[0], weight_by) + if first_weight is None: + continue + try: + weights: Final[list[object]] = [ + 0 if (value := _weight(deployment, weight_by)) is None else value for deployment in deployments + ] + if sum(weights) > 0: + return [deployment for deployment, weight in zip(deployments, weights) if weight > 0] + except (TypeError, ValueError): + return list(deployments) + return list(deployments) + @staticmethod def _router_allows_bedrock( request_data: Mapping[str, object], @@ -1259,25 +1296,52 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): model: Final[object | None] = request_data.get("model") if isinstance(model, str): effective_model = model + effective_messages: object | None = ( + router_request_kwargs.get("messages") + if isinstance(router_request_kwargs.get("messages"), list) + else None + ) + effective_input: object | None = ( + router_request_kwargs.get("input") + if isinstance(router_request_kwargs.get("input"), (str, list)) + else None + ) try: + pre_routing_lookup: Final[object] = getattr(llm_router, "async_pre_routing_hook", None) + if callable(pre_routing_lookup): + pre_routing_result = pre_routing_lookup( + model=model, + request_kwargs=router_request_kwargs, + messages=effective_messages, + input=effective_input, + specific_deployment=request_data.get("specific_deployment") is True, + ) + if asyncio.iscoroutine(pre_routing_result): + pre_routing_result = await pre_routing_result + routed_model: Final[object] = getattr(pre_routing_result, "model", None) + if isinstance(routed_model, str): + effective_model = routed_model + routed_messages: Final[object] = getattr(pre_routing_result, "messages", None) + effective_messages = routed_messages if isinstance(routed_messages, list) else None + routed_params: Final[object] = getattr(pre_routing_result, "litellm_params", None) + if isinstance(routed_params, Mapping): + router_request_kwargs.update(routed_params) healthy_deployments: Final = await async_lookup( model=effective_model, request_kwargs=router_request_kwargs, - messages=router_request_kwargs.get("messages") - if isinstance(router_request_kwargs.get("messages"), list) - else None, - input=router_request_kwargs.get("input") - if isinstance(router_request_kwargs.get("input"), (str, list)) - else None, + messages=effective_messages, + input=effective_input, specific_deployment=request_data.get("specific_deployment") is True, ) - deployments: Final[list[object]] = ( + deployments: list[object] = ( [healthy_deployments] if isinstance(healthy_deployments, Mapping) else healthy_deployments if isinstance(healthy_deployments, list) else [] ) + if routing_strategy == "simple-shuffle": + deployments = BedrockGuardrail._router_deployments_for_provider_check(deployments) if deployments: providers: list[str] = [] for deployment in deployments: diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index 080c32ed317..a058cd0dacf 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -466,6 +466,79 @@ async def test_bedrock_guardrail_async_uses_callback_filtered_pool(monkeypatch: router_allows.assert_not_called() +@pytest.mark.asyncio +async def test_bedrock_guardrail_async_uses_pre_routed_model(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + + router = MagicMock() + router.routing_strategy = "usage-based-routing-v2" + router.async_pre_routing_hook = AsyncMock( + return_value=MagicMock(model="bedrock-model", messages=[{"role": "user", "content": "hi"}]) + ) + router.async_get_healthy_deployments = AsyncMock( + return_value=[{"litellm_params": {"custom_llm_provider": "bedrock"}, "model_info": {}}] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + + assert ( + await BedrockGuardrail._async_get_bedrock_api_key( + { + "model": "router-alias", + "api_key": "bedrock-key", + "messages": [{"role": "user", "content": "hi"}], + } + ) + == "bedrock-key" + ) + assert router.async_get_healthy_deployments.await_args.kwargs["model"] == "bedrock-model" + + +@pytest.mark.asyncio +async def test_bedrock_guardrail_async_honors_nested_tag_override(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + + router = MagicMock() + router.routing_strategy = "usage-based-routing-v2" + router.async_get_healthy_deployments = AsyncMock( + return_value=[{"litellm_params": {"custom_llm_provider": "bedrock"}, "model_info": {}}] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + + assert ( + await BedrockGuardrail._async_get_bedrock_api_key( + { + "model": "router-alias", + "api_key": "bedrock-key", + "router_settings_override": {"enable_tag_filtering": True}, + } + ) + == "bedrock-key" + ) + assert router.async_get_healthy_deployments.await_args.kwargs["request_kwargs"]["enable_tag_filtering"] is True + + +@pytest.mark.asyncio +async def test_bedrock_guardrail_async_ignores_zero_weight_provider(monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + + router = MagicMock() + router.routing_strategy = "simple-shuffle" + router.async_get_healthy_deployments = AsyncMock( + return_value=[ + {"litellm_params": {"custom_llm_provider": "openai", "weight": 0}, "model_info": {}}, + {"litellm_params": {"custom_llm_provider": "bedrock", "weight": 1}, "model_info": {}}, + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + + assert ( + await BedrockGuardrail._async_get_bedrock_api_key( + {"model": "router-alias", "api_key": "bedrock-key"} + ) + == "bedrock-key" + ) + + def test_bedrock_guardrail_matches_regex_tag_pool(monkeypatch: pytest.MonkeyPatch): from litellm.proxy import proxy_server From 642bdc134f6a9908700c9c4fdceeda5d0c2c7a17 Mon Sep 17 00:00:00 2001 From: aiedwardyi <41576951+aiedwardyi@users.noreply.github.com> Date: Wed, 26 Aug 2026 18:35:23 +0900 Subject: [PATCH 20/21] style: format bedrock guardrail hook --- .../guardrails/guardrail_hooks/bedrock_guardrails.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index cc25b6ee56f..03e03a5d536 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -779,8 +779,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): ): return [] if not any( - isinstance(deployment, Mapping) - and (deployment.get("model_info") or {}).get("allow_fail_open") is True + isinstance(deployment, Mapping) and (deployment.get("model_info") or {}).get("allow_fail_open") is True for deployment in deployments ): return [] @@ -888,11 +887,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): if isinstance(deployment, Mapping) else getattr(deployment, "litellm_params", None) ) - return ( - params.get(weight_by) - if isinstance(params, Mapping) - else getattr(params, weight_by, None) - ) + return params.get(weight_by) if isinstance(params, Mapping) else getattr(params, weight_by, None) for weight_by in ("weight", "rpm", "tpm"): first_weight: Final[object | None] = _weight(deployments[0], weight_by) From 695fa583d46bdca56db62e0ded0760a6498ee85e Mon Sep 17 00:00:00 2001 From: aiedwardyi <41576951+aiedwardyi@users.noreply.github.com> Date: Fri, 28 Aug 2026 12:56:24 +0900 Subject: [PATCH 21/21] fix: split Bedrock router checks under the strict-rule budget Extracts candidate resolution, the eligibility chain and the empty-deployment verdict out of _router_allows_bedrock, and the async router lookup out of _async_get_bedrock_api_key, clearing the two added C901s and the S110. Test fixtures gave deployments differing orders and no model_info.id, so the router's own order and team filters decided both cases before the assertion did. --- .../guardrail_hooks/bedrock_guardrails.py | 818 ++++++++++-------- .../test_bedrock_guardrails.py | 13 +- 2 files changed, 467 insertions(+), 364 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 03e03a5d536..7d46af13852 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -122,6 +122,18 @@ _BEDROCK_INVOKE_GUARDRAIL_CHECKS_PATH: Final = "/guardrail-checks/invoke" # never truncated (truncation would let a user hide content past the limit). _BEDROCK_CHECKS_MAX_CONTENT_BLOCKS: Final = 10 _ROUTER_COOLDOWNS_UNSET: Final = object() + + +class _RouterCandidates(NamedTuple): + """What the router would consider for a request, before the eligibility filters.""" + + effective_model: str + common_result: tuple[object, object] | None + model_id_deployment_row: object | None + candidate_deployments: Sequence[object] + router_matched: bool + + _BEDROCK_CHECKS_KNOWN_KEYS: Final = frozenset({"contentFilter", "promptAttack", "sensitiveInformation"}) # Keys in a sensitiveInformation result that pinpoint the PII location. They are # stripped before the response is handed to standard logging / telemetry so the @@ -903,6 +915,347 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): return list(deployments) return list(deployments) + @staticmethod + def _router_candidate_deployments( + llm_router: object, + request_data: Mapping[str, object], + router_kwargs: dict[str, object], # mutable-ok: the router pops its own routing keys off these kwargs + model: str, + resolved_team_id: str | None, + ) -> _RouterCandidates: + """Deployments the router would consider, before any of the eligibility filters.""" + effective_model: str = model + common_result: tuple[object, object] | None = None + common_lookup: Final[object] = getattr(llm_router, "_common_checks_available_deployment", None) + if callable(common_lookup): + try: + raw_common_result: Final = common_lookup( + model=effective_model, + request_kwargs=router_kwargs, + specific_deployment=request_data.get("specific_deployment") is True, + ) + except Exception: # noqa: BLE001 # fall back for lightweight router test doubles + raw_common_result = None + if ( + isinstance(raw_common_result, tuple) + and len(raw_common_result) == 2 + and isinstance(raw_common_result[1], (Mapping, list)) + ): + common_result = raw_common_result + if isinstance(raw_common_result[0], str): + effective_model = raw_common_result[0] + + if common_result is not None: + raw_deployments: Final[object] = common_result[1] + model_id_deployment_row: Final[object | None] = ( + raw_deployments if isinstance(raw_deployments, Mapping) else None + ) + candidate_deployments: Final[list[object]] = ( + [raw_deployments] + if isinstance(raw_deployments, Mapping) + else [deployment for deployment in raw_deployments if isinstance(deployment, Mapping)] + ) + router_matched: Final = bool(candidate_deployments) + else: + model_id_deployment: Final = ( + llm_router.get_deployment(model_id=effective_model) + if llm_router.has_model_id(effective_model) is True + else None + ) + model_id_deployment_row: Final = ( + model_id_deployment.model_dump(exclude_none=True) + if model_id_deployment is not None and hasattr(model_id_deployment, "model_dump") + else model_id_deployment + ) + specific_deployment_rows: list[object] | None = None + deployment_names: Final[object] = getattr(llm_router, "deployment_names", None) + specific_lookup: Final[object] = getattr(llm_router, "_get_deployment_by_litellm_model", None) + model_group_aliases: Final = getattr(llm_router, "model_group_alias", None) + concrete_model_names: Final[object | None] = getattr(llm_router, "model_names", None) + is_concrete_model: Final = ( + isinstance(concrete_model_names, (list, tuple, set, frozenset)) + and effective_model in concrete_model_names + ) + is_model_alias: Final = isinstance(model_group_aliases, Mapping) and effective_model in model_group_aliases + raw_listed_deployments = ( + [] + if model_id_deployment_row is not None + else ( + llm_router.get_model_list(model_name=effective_model, team_id=resolved_team_id) or [] + if is_concrete_model or is_model_alias + else [] + ) + ) + if model_id_deployment_row is None and not is_concrete_model and not is_model_alias: + pattern_router: Final[object | None] = getattr(llm_router, "pattern_router", None) + get_pattern_deployments: Final[object | None] = getattr( + pattern_router, "get_deployments_by_pattern", None + ) + global_pattern_deployments: Final = ( + get_pattern_deployments(model=effective_model) if callable(get_pattern_deployments) else None + ) + team_pattern_router: Final[object | None] = ( + getattr(llm_router, "team_pattern_routers", {}).get(resolved_team_id) + if resolved_team_id is not None + and isinstance(getattr(llm_router, "team_pattern_routers", None), Mapping) + else None + ) + get_team_pattern_deployments: Final[object | None] = getattr( + team_pattern_router, "get_deployments_by_pattern", None + ) + team_pattern_deployments: Final = ( + get_team_pattern_deployments(model=effective_model) + if callable(get_team_pattern_deployments) + else None + ) + if isinstance(global_pattern_deployments, list) and global_pattern_deployments: + raw_listed_deployments = global_pattern_deployments + elif isinstance(team_pattern_deployments, list) and team_pattern_deployments: + raw_listed_deployments = team_pattern_deployments + else: + default_deployment = getattr(llm_router, "default_deployment", None) + if isinstance(default_deployment, Mapping): + raw_listed_deployments = [default_deployment] + elif ( + isinstance(deployment_names, Sequence) + and not isinstance(deployment_names, (str, bytes)) + and effective_model in deployment_names + and callable(specific_lookup) + ): + specific_result: Final = specific_lookup(model=effective_model) + specific_deployment_rows = specific_result if isinstance(specific_result, list) else [] + raw_listed_deployments = specific_deployment_rows + else: + raw_listed_deployments = ( + llm_router.get_model_list(model_name=effective_model, team_id=resolved_team_id) or [] + ) + candidate_deployments = ( + [model_id_deployment_row] + if model_id_deployment_row is not None + else [deployment for deployment in raw_listed_deployments if isinstance(deployment, Mapping)] + ) + router_matched = bool(candidate_deployments) + return _RouterCandidates( + effective_model, + common_result, + model_id_deployment_row, + candidate_deployments, + router_matched, + ) + + @staticmethod + def _filter_router_deployments( + llm_router: object, + request_data: Mapping[str, object], + router_kwargs: dict[str, object], # mutable-ok: the router pops its own routing keys off these kwargs + *, + effective_model: str, + resolved_team_id: str | None, + common_result: tuple[object, object] | None, + model_id_deployment_row: object | None, + candidate_deployments: Sequence[object], + cooldown_deployments: Sequence[str] | None | object, + apply_tag_filtering: bool, + ) -> Sequence[object]: + """The router's own eligibility chain, in the router's order. + + Order filtering runs before the weighted-failover exclusion, matching Router + so a guardrail verdict cannot disagree with the deployment actually picked. + """ + team_filtered_result: Final = ( + candidate_deployments + if model_id_deployment_row is not None + else filter_team_based_models( + healthy_deployments=candidate_deployments, + request_kwargs=router_kwargs, + ) + ) + team_filtered_deployments: Final[list[object]] = ( + team_filtered_result if isinstance(team_filtered_result, list) else candidate_deployments + ) + + filter_deployments: Final = getattr(llm_router, "_filter_deployments_by_model_access_groups", None) + filtered_deployments: Final = ( + filter_deployments( + model=effective_model, + healthy_deployments=team_filtered_deployments, + request_kwargs=dict(request_data), + request_team_id=resolved_team_id, + ) + if callable(filter_deployments) + and isinstance(team_filtered_deployments, list) + and model_id_deployment_row is None + else None + ) + access_filtered_deployments: Final[list[object]] = ( + filtered_deployments if isinstance(filtered_deployments, list) else team_filtered_deployments + ) + health_filter: Final[object | None] = getattr(llm_router, "_filter_health_check_unhealthy_deployments", None) + health_filtered_deployments: Final = ( + health_filter( + healthy_deployments=access_filtered_deployments, + parent_otel_span=None, + ) + if (common_result is None or model_id_deployment_row is None) and callable(health_filter) + else access_filtered_deployments + ) + healthy_deployments: Final[list[object]] = ( + health_filtered_deployments + if isinstance(health_filtered_deployments, list) + else access_filtered_deployments + ) + pre_call_filter: Final[object] = getattr(llm_router, "_pre_call_checks", None) + request_messages: Final[object] = request_data.get("messages") + request_input: Final[object] = request_data.get("input") + if ( + model_id_deployment_row is None + and getattr(llm_router, "enable_pre_call_checks", False) is True + and (isinstance(request_messages, list) or isinstance(request_input, (str, list))) + and callable(pre_call_filter) + ): + pre_call_deployments: Final = pre_call_filter( + model=effective_model, + healthy_deployments=healthy_deployments, + messages=request_messages if isinstance(request_messages, list) else None, + input=request_input if isinstance(request_input, (str, list)) else None, + request_kwargs=router_kwargs, + ) + if isinstance(pre_call_deployments, list): + healthy_deployments = pre_call_deployments + cooldown_cache: Final[object | None] = getattr(llm_router, "cooldown_cache", None) + cooldown_lookup: Final[object | None] = getattr(cooldown_cache, "get_active_cooldowns", None) + resolved_cooldown_deployments: Final = ( + _get_cooldown_deployments( + litellm_router_instance=llm_router, + parent_otel_span=None, + ) + if cooldown_deployments is _ROUTER_COOLDOWNS_UNSET + and (common_result is None or model_id_deployment_row is None) + and callable(cooldown_lookup) + and callable(getattr(llm_router, "get_model_ids", None)) + else cooldown_deployments + if cooldown_deployments is not _ROUTER_COOLDOWNS_UNSET + else [] + ) + cooldown_ids: Final[frozenset[str]] = frozenset( + deployment_id for deployment_id in (resolved_cooldown_deployments or []) if isinstance(deployment_id, str) + ) + if common_result is not None and model_id_deployment_row is not None: + deployments = candidate_deployments + else: + cooldown_filtered_deployments: Final = [ + deployment + for deployment in healthy_deployments + if BedrockGuardrail._router_deployment_field(deployment, "id") not in cooldown_ids + ] + unblocked_deployments: Final = [ + deployment + for deployment in cooldown_filtered_deployments + if BedrockGuardrail._router_deployment_field(deployment, "blocked") is not True + ] + deployments = ( + BedrockGuardrail._filter_router_deployments_by_tags( + router=llm_router, + deployments=unblocked_deployments, + request_data=request_data, + model=effective_model, + ) + if apply_tag_filtering + else unblocked_deployments + ) + if common_result is not None and model_id_deployment_row is None: + web_search_deployments: Final = filter_web_search_deployments( + healthy_deployments=deployments, + request_kwargs=router_kwargs, + ) + deployments = web_search_deployments if isinstance(web_search_deployments, list) else deployments + plugin_filter: Final[object] = getattr(llm_router, "_filter_by_routing_plugin_candidates", None) + if callable(plugin_filter): + plugin_deployments: Final = plugin_filter( + healthy_deployments=deployments, + request_kwargs=router_kwargs, + ) + if isinstance(plugin_deployments, list): + deployments = plugin_deployments + deployments = litellm.utils._get_order_filtered_deployments( + deployments, + target_order=router_kwargs.pop("_target_order", None), + ) + deployments = litellm.utils._get_excluded_filtered_deployments( + deployments, + excluded_deployment_ids=router_kwargs.pop("_excluded_deployment_ids", None), + ) + return deployments + + @staticmethod + def _router_verdict_without_deployments( + llm_router: object, + request_data: Mapping[str, object], + *, + effective_model: str, + resolved_team_id: str | None, + router_matched: bool, + apply_tag_filtering: bool, + ) -> bool | None: + """Verdict when the filters left nothing: pass-through, default deployment, or fallback.""" + if router_matched: + return False + router_settings: Final[object] = getattr(llm_router, "router_general_settings", None) + if getattr(router_settings, "pass_through_all_models", False) is True: + requested_provider: Final[object] = request_data.get("custom_llm_provider") + passthrough_provider: Final[object] = ( + requested_provider + if isinstance(requested_provider, str) + else BedrockGuardrail._resolve_model_provider(effective_model) + ) + return ( + passthrough_provider in ("bedrock", "bedrock_converse") + if isinstance(passthrough_provider, str) + else None + ) + + default_deployment: Final[object] = getattr(llm_router, "default_deployment", None) + if isinstance(default_deployment, Mapping): + default_params: Final[object] = default_deployment.get("litellm_params") + configured_provider: Final[object] = ( + default_params.get("custom_llm_provider") + if isinstance(default_params, Mapping) + else getattr(default_params, "custom_llm_provider", None) + ) + default_model: Final[object] = ( + default_params.get("model") + if isinstance(default_params, Mapping) + else getattr(default_params, "model", None) + ) + default_provider: Final[object] = ( + configured_provider + if isinstance(configured_provider, str) + else BedrockGuardrail._resolve_model_provider(default_model) + if isinstance(default_model, str) + else None + ) + return default_provider in ("bedrock", "bedrock_converse") if isinstance(default_provider, str) else None + + default_fallback_lookup: Final[object] = getattr(llm_router, "_get_first_default_fallback", None) + default_fallback_model: Final[object] = default_fallback_lookup() if callable(default_fallback_lookup) else None + if isinstance(default_fallback_model, str) and default_fallback_model != effective_model: + fallback_deployments: Final = ( + llm_router.get_model_list( + model_name=default_fallback_model, + team_id=resolved_team_id, + ) + or [] + ) + if isinstance(fallback_deployments, list) and fallback_deployments: + fallback_request_data: Final = dict(request_data) + fallback_request_data["model"] = default_fallback_model + return BedrockGuardrail._router_allows_bedrock( + fallback_request_data, + apply_tag_filtering=apply_tag_filtering, + ) + + return False + @staticmethod def _router_allows_bedrock( request_data: Mapping[str, object], @@ -937,308 +1290,32 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): try: resolved_team_id: Final = team_id if isinstance(team_id, str) else None router_kwargs: Final = BedrockGuardrail._get_trusted_router_request_kwargs(request_data) - effective_model: str = model - common_result: tuple[object, object] | None = None - common_lookup: Final[object] = getattr(llm_router, "_common_checks_available_deployment", None) - if callable(common_lookup): - try: - raw_common_result: Final = common_lookup( - model=effective_model, - request_kwargs=router_kwargs, - specific_deployment=request_data.get("specific_deployment") is True, - ) - except Exception: # noqa: BLE001 # fall back for lightweight router test doubles - raw_common_result = None - if ( - isinstance(raw_common_result, tuple) - and len(raw_common_result) == 2 - and isinstance(raw_common_result[1], (Mapping, list)) - ): - common_result = raw_common_result - if isinstance(raw_common_result[0], str): - effective_model = raw_common_result[0] - - if common_result is not None: - raw_deployments: Final[object] = common_result[1] - model_id_deployment_row: Final[object | None] = ( - raw_deployments if isinstance(raw_deployments, Mapping) else None - ) - candidate_deployments: Final[list[object]] = ( - [raw_deployments] - if isinstance(raw_deployments, Mapping) - else [deployment for deployment in raw_deployments if isinstance(deployment, Mapping)] - ) - router_matched: Final = bool(candidate_deployments) - else: - model_id_deployment: Final = ( - llm_router.get_deployment(model_id=effective_model) - if llm_router.has_model_id(effective_model) is True - else None - ) - model_id_deployment_row: Final = ( - model_id_deployment.model_dump(exclude_none=True) - if model_id_deployment is not None and hasattr(model_id_deployment, "model_dump") - else model_id_deployment - ) - specific_deployment_rows: list[object] | None = None - deployment_names: Final[object] = getattr(llm_router, "deployment_names", None) - specific_lookup: Final[object] = getattr(llm_router, "_get_deployment_by_litellm_model", None) - model_group_aliases: Final = getattr(llm_router, "model_group_alias", None) - concrete_model_names: Final[object | None] = getattr(llm_router, "model_names", None) - is_concrete_model: Final = ( - isinstance(concrete_model_names, (list, tuple, set, frozenset)) - and effective_model in concrete_model_names - ) - is_model_alias: Final = ( - isinstance(model_group_aliases, Mapping) and effective_model in model_group_aliases - ) - raw_listed_deployments = ( - [] - if model_id_deployment_row is not None - else ( - llm_router.get_model_list(model_name=effective_model, team_id=resolved_team_id) or [] - if is_concrete_model or is_model_alias - else [] - ) - ) - if model_id_deployment_row is None and not is_concrete_model and not is_model_alias: - pattern_router: Final[object | None] = getattr(llm_router, "pattern_router", None) - get_pattern_deployments: Final[object | None] = getattr( - pattern_router, "get_deployments_by_pattern", None - ) - global_pattern_deployments: Final = ( - get_pattern_deployments(model=effective_model) if callable(get_pattern_deployments) else None - ) - team_pattern_router: Final[object | None] = ( - getattr(llm_router, "team_pattern_routers", {}).get(resolved_team_id) - if resolved_team_id is not None - and isinstance(getattr(llm_router, "team_pattern_routers", None), Mapping) - else None - ) - get_team_pattern_deployments: Final[object | None] = getattr( - team_pattern_router, "get_deployments_by_pattern", None - ) - team_pattern_deployments: Final = ( - get_team_pattern_deployments(model=effective_model) - if callable(get_team_pattern_deployments) - else None - ) - if isinstance(global_pattern_deployments, list) and global_pattern_deployments: - raw_listed_deployments = global_pattern_deployments - elif isinstance(team_pattern_deployments, list) and team_pattern_deployments: - raw_listed_deployments = team_pattern_deployments - else: - default_deployment = getattr(llm_router, "default_deployment", None) - if isinstance(default_deployment, Mapping): - raw_listed_deployments = [default_deployment] - elif ( - isinstance(deployment_names, Sequence) - and not isinstance(deployment_names, (str, bytes)) - and effective_model in deployment_names - and callable(specific_lookup) - ): - specific_result: Final = specific_lookup(model=effective_model) - specific_deployment_rows = specific_result if isinstance(specific_result, list) else [] - raw_listed_deployments = specific_deployment_rows - else: - raw_listed_deployments = ( - llm_router.get_model_list(model_name=effective_model, team_id=resolved_team_id) or [] - ) - candidate_deployments = ( - [model_id_deployment_row] - if model_id_deployment_row is not None - else [deployment for deployment in raw_listed_deployments if isinstance(deployment, Mapping)] - ) - router_matched = bool(candidate_deployments) - team_filtered_result: Final = ( - candidate_deployments - if model_id_deployment_row is not None - else filter_team_based_models( - healthy_deployments=candidate_deployments, - request_kwargs=router_kwargs, - ) + candidates: Final = BedrockGuardrail._router_candidate_deployments( + llm_router, request_data, router_kwargs, model, resolved_team_id ) - team_filtered_deployments: Final[list[object]] = ( - team_filtered_result if isinstance(team_filtered_result, list) else candidate_deployments + deployments: Final = BedrockGuardrail._filter_router_deployments( + llm_router, + request_data, + router_kwargs, + effective_model=candidates.effective_model, + resolved_team_id=resolved_team_id, + common_result=candidates.common_result, + model_id_deployment_row=candidates.model_id_deployment_row, + candidate_deployments=candidates.candidate_deployments, + cooldown_deployments=cooldown_deployments, + apply_tag_filtering=apply_tag_filtering, ) - - filter_deployments: Final = getattr(llm_router, "_filter_deployments_by_model_access_groups", None) - filtered_deployments: Final = ( - filter_deployments( - model=effective_model, - healthy_deployments=team_filtered_deployments, - request_kwargs=dict(request_data), - request_team_id=resolved_team_id, - ) - if callable(filter_deployments) - and isinstance(team_filtered_deployments, list) - and model_id_deployment_row is None - else None - ) - access_filtered_deployments: Final[list[object]] = ( - filtered_deployments if isinstance(filtered_deployments, list) else team_filtered_deployments - ) - health_filter: Final[object | None] = getattr( - llm_router, "_filter_health_check_unhealthy_deployments", None - ) - health_filtered_deployments: Final = ( - health_filter( - healthy_deployments=access_filtered_deployments, - parent_otel_span=None, - ) - if (common_result is None or model_id_deployment_row is None) and callable(health_filter) - else access_filtered_deployments - ) - healthy_deployments: Final[list[object]] = ( - health_filtered_deployments - if isinstance(health_filtered_deployments, list) - else access_filtered_deployments - ) - pre_call_filter: Final[object] = getattr(llm_router, "_pre_call_checks", None) - request_messages: Final[object] = request_data.get("messages") - request_input: Final[object] = request_data.get("input") - if ( - model_id_deployment_row is None - and getattr(llm_router, "enable_pre_call_checks", False) is True - and (isinstance(request_messages, list) or isinstance(request_input, (str, list))) - and callable(pre_call_filter) - ): - pre_call_deployments: Final = pre_call_filter( - model=effective_model, - healthy_deployments=healthy_deployments, - messages=request_messages if isinstance(request_messages, list) else None, - input=request_input if isinstance(request_input, (str, list)) else None, - request_kwargs=router_kwargs, - ) - if isinstance(pre_call_deployments, list): - healthy_deployments = pre_call_deployments - cooldown_cache: Final[object | None] = getattr(llm_router, "cooldown_cache", None) - cooldown_lookup: Final[object | None] = getattr(cooldown_cache, "get_active_cooldowns", None) - resolved_cooldown_deployments: Final = ( - _get_cooldown_deployments( - litellm_router_instance=llm_router, - parent_otel_span=None, - ) - if cooldown_deployments is _ROUTER_COOLDOWNS_UNSET - and (common_result is None or model_id_deployment_row is None) - and callable(cooldown_lookup) - and callable(getattr(llm_router, "get_model_ids", None)) - else cooldown_deployments - if cooldown_deployments is not _ROUTER_COOLDOWNS_UNSET - else [] - ) - cooldown_ids: Final[frozenset[str]] = frozenset( - deployment_id - for deployment_id in (resolved_cooldown_deployments or []) - if isinstance(deployment_id, str) - ) - if common_result is not None and model_id_deployment_row is not None: - deployments = candidate_deployments - else: - cooldown_filtered_deployments: Final = [ - deployment - for deployment in healthy_deployments - if BedrockGuardrail._router_deployment_field(deployment, "id") not in cooldown_ids - ] - unblocked_deployments: Final = [ - deployment - for deployment in cooldown_filtered_deployments - if BedrockGuardrail._router_deployment_field(deployment, "blocked") is not True - ] - deployments = ( - BedrockGuardrail._filter_router_deployments_by_tags( - router=llm_router, - deployments=unblocked_deployments, - request_data=request_data, - model=effective_model, - ) - if apply_tag_filtering - else unblocked_deployments - ) - if common_result is not None and model_id_deployment_row is None: - web_search_deployments: Final = filter_web_search_deployments( - healthy_deployments=deployments, - request_kwargs=router_kwargs, - ) - deployments = web_search_deployments if isinstance(web_search_deployments, list) else deployments - plugin_filter: Final[object] = getattr(llm_router, "_filter_by_routing_plugin_candidates", None) - if callable(plugin_filter): - plugin_deployments: Final = plugin_filter( - healthy_deployments=deployments, - request_kwargs=router_kwargs, - ) - if isinstance(plugin_deployments, list): - deployments = plugin_deployments - deployments = litellm.utils._get_order_filtered_deployments( - deployments, - target_order=router_kwargs.pop("_target_order", None), - ) - deployments = litellm.utils._get_excluded_filtered_deployments( - deployments, - excluded_deployment_ids=router_kwargs.pop("_excluded_deployment_ids", None), - ) except Exception: # noqa: BLE001 # optional router state must not break guardrail auth return False if not deployments: - if router_matched: - return False - router_settings = getattr(llm_router, "router_general_settings", None) - if getattr(router_settings, "pass_through_all_models", False) is True: - provider = request_data.get("custom_llm_provider") - if not isinstance(provider, str): - provider = ( - BedrockGuardrail._resolve_model_provider(effective_model) - if isinstance(effective_model, str) - else None - ) - return provider in ("bedrock", "bedrock_converse") if isinstance(provider, str) else None - - default_deployment = getattr(llm_router, "default_deployment", None) - if isinstance(default_deployment, Mapping): - default_params: object = ( - default_deployment.get("litellm_params") - if isinstance(default_deployment, Mapping) - else getattr(default_deployment, "litellm_params", None) - ) - provider: object = ( - default_params.get("custom_llm_provider") - if isinstance(default_params, Mapping) - else getattr(default_params, "custom_llm_provider", None) - ) - if not isinstance(provider, str): - default_model: object = ( - default_params.get("model") - if isinstance(default_params, Mapping) - else getattr(default_params, "model", None) - ) - provider = ( - BedrockGuardrail._resolve_model_provider(default_model) - if isinstance(default_model, str) - else None - ) - return provider in ("bedrock", "bedrock_converse") if isinstance(provider, str) else None - - default_fallback_lookup: Final[object] = getattr(llm_router, "_get_first_default_fallback", None) - default_fallback_model: Final[object] = ( - default_fallback_lookup() if callable(default_fallback_lookup) else None + return BedrockGuardrail._router_verdict_without_deployments( + llm_router, + request_data, + effective_model=candidates.effective_model, + resolved_team_id=resolved_team_id, + router_matched=candidates.router_matched, + apply_tag_filtering=apply_tag_filtering, ) - if isinstance(default_fallback_model, str) and default_fallback_model != effective_model: - fallback_deployments: Final = ( - llm_router.get_model_list( - model_name=default_fallback_model, - team_id=resolved_team_id, - ) - or [] - ) - if isinstance(fallback_deployments, list) and fallback_deployments: - fallback_request_data: Final = dict(request_data) - fallback_request_data["model"] = default_fallback_model - return BedrockGuardrail._router_allows_bedrock( - fallback_request_data, - apply_tag_filtering=apply_tag_filtering, - ) - - return False providers: list[str] = [] for deployment in deployments: @@ -1248,6 +1325,82 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): providers.append(provider) return bool(providers) and all(provider in ("bedrock", "bedrock_converse") for provider in providers) + @staticmethod + async def _async_router_bedrock_verdict( + llm_router: object, + request_data: Mapping[str, object], + router_request_kwargs: dict[str, object], # mutable-ok: the pre-routing hook writes resolved params back here + routing_strategy: object | None, + ) -> bool | None: + """Whether the router's async healthy-deployment set is all-Bedrock. + + None means no verdict, so the caller falls through to the sync path. + """ + async_lookup: Final[object] = getattr(llm_router, "async_get_healthy_deployments", None) + if not callable(async_lookup): + return None + model: Final[object | None] = request_data.get("model") + if not isinstance(model, str): + return None + + effective_model = model + effective_messages: object | None = ( + router_request_kwargs.get("messages") if isinstance(router_request_kwargs.get("messages"), list) else None + ) + effective_input: object | None = ( + router_request_kwargs.get("input") if isinstance(router_request_kwargs.get("input"), (str, list)) else None + ) + try: + pre_routing_lookup: Final[object] = getattr(llm_router, "async_pre_routing_hook", None) + if callable(pre_routing_lookup): + pre_routing_result = pre_routing_lookup( + model=model, + request_kwargs=router_request_kwargs, + messages=effective_messages, + input=effective_input, + specific_deployment=request_data.get("specific_deployment") is True, + ) + if asyncio.iscoroutine(pre_routing_result): + pre_routing_result = await pre_routing_result + routed_model: Final[object] = getattr(pre_routing_result, "model", None) + if isinstance(routed_model, str): + effective_model = routed_model + routed_messages: Final[object] = getattr(pre_routing_result, "messages", None) + effective_messages = routed_messages if isinstance(routed_messages, list) else None + routed_params: Final[object] = getattr(pre_routing_result, "litellm_params", None) + if isinstance(routed_params, Mapping): + router_request_kwargs.update(routed_params) + healthy_deployments: Final = await async_lookup( + model=effective_model, + request_kwargs=router_request_kwargs, + messages=effective_messages, + input=effective_input, + specific_deployment=request_data.get("specific_deployment") is True, + ) + except Exception as exc: # noqa: BLE001 # fall back to the sync compatibility path + verbose_proxy_logger.debug("Bedrock guardrail: async router lookup failed, using the sync path: %s", exc) + return None + + deployments: list[object] = ( + [healthy_deployments] + if isinstance(healthy_deployments, Mapping) + else healthy_deployments + if isinstance(healthy_deployments, list) + else [] + ) + if routing_strategy == "simple-shuffle": + deployments = BedrockGuardrail._router_deployments_for_provider_check(deployments) + if not deployments: + return None + + providers: list[str] = [] + for deployment in deployments: + provider: Final[str | None] = BedrockGuardrail._router_deployment_provider(deployment) + if provider is None: + return False + providers.append(provider) + return all(provider in ("bedrock", "bedrock_converse") for provider in providers) + @staticmethod async def _async_get_bedrock_api_key(request_data: Mapping[str, object] | None) -> str | None: if not request_data: @@ -1286,71 +1439,14 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): if router_allows_bedrock is not None: return api_key if router_allows_bedrock else None - async_lookup: Final[object] = getattr(llm_router, "async_get_healthy_deployments", None) - if callable(async_lookup): - model: Final[object | None] = request_data.get("model") - if isinstance(model, str): - effective_model = model - effective_messages: object | None = ( - router_request_kwargs.get("messages") - if isinstance(router_request_kwargs.get("messages"), list) - else None - ) - effective_input: object | None = ( - router_request_kwargs.get("input") - if isinstance(router_request_kwargs.get("input"), (str, list)) - else None - ) - try: - pre_routing_lookup: Final[object] = getattr(llm_router, "async_pre_routing_hook", None) - if callable(pre_routing_lookup): - pre_routing_result = pre_routing_lookup( - model=model, - request_kwargs=router_request_kwargs, - messages=effective_messages, - input=effective_input, - specific_deployment=request_data.get("specific_deployment") is True, - ) - if asyncio.iscoroutine(pre_routing_result): - pre_routing_result = await pre_routing_result - routed_model: Final[object] = getattr(pre_routing_result, "model", None) - if isinstance(routed_model, str): - effective_model = routed_model - routed_messages: Final[object] = getattr(pre_routing_result, "messages", None) - effective_messages = routed_messages if isinstance(routed_messages, list) else None - routed_params: Final[object] = getattr(pre_routing_result, "litellm_params", None) - if isinstance(routed_params, Mapping): - router_request_kwargs.update(routed_params) - healthy_deployments: Final = await async_lookup( - model=effective_model, - request_kwargs=router_request_kwargs, - messages=effective_messages, - input=effective_input, - specific_deployment=request_data.get("specific_deployment") is True, - ) - deployments: list[object] = ( - [healthy_deployments] - if isinstance(healthy_deployments, Mapping) - else healthy_deployments - if isinstance(healthy_deployments, list) - else [] - ) - if routing_strategy == "simple-shuffle": - deployments = BedrockGuardrail._router_deployments_for_provider_check(deployments) - if deployments: - providers: list[str] = [] - for deployment in deployments: - provider: Final[str | None] = BedrockGuardrail._router_deployment_provider(deployment) - if provider is None: - return None - providers.append(provider) - return ( - api_key - if all(provider in ("bedrock", "bedrock_converse") for provider in providers) - else None - ) - except Exception: # noqa: BLE001 # fall back to the sync compatibility path - pass + async_verdict: Final = await BedrockGuardrail._async_router_bedrock_verdict( + llm_router, + request_data, + router_request_kwargs, + routing_strategy, + ) + if async_verdict is not None: + return api_key if async_verdict else None router_allows_bedrock: Final = BedrockGuardrail._router_allows_bedrock( request_data, diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index a058cd0dacf..815c58bef81 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -625,13 +625,15 @@ def test_bedrock_guardrail_applies_router_post_filters(monkeypatch: pytest.Monke from litellm.proxy import proxy_server router = MagicMock() + # Equal order: the min-order filter runs before exclusion, so an openai row ordered + # ahead of bedrock would decide the verdict on its own and never exercise exclusion. deployments = [ { "litellm_params": {"custom_llm_provider": "openai", "order": 1}, "model_info": {"id": "openai"}, }, { - "litellm_params": {"custom_llm_provider": "bedrock", "order": 2}, + "litellm_params": {"custom_llm_provider": "bedrock", "order": 1}, "model_info": {"id": "bedrock"}, }, ] @@ -648,6 +650,9 @@ def test_bedrock_guardrail_applies_router_post_filters(monkeypatch: pytest.Monke ) is True ) + assert ( + BedrockGuardrail._router_allows_bedrock({"model": "shared-alias", "_target_order": 2}) is False + ) def test_bedrock_guardrail_applies_web_search_filter(monkeypatch: pytest.MonkeyPatch): @@ -744,14 +749,16 @@ def test_bedrock_guardrail_ignores_blocked_deployments(): def test_bedrock_guardrail_filters_alias_deployments_by_team(): router = MagicMock() router.model_group_alias = {"team-alias": "shared-group"} + # filter_team_based_models drops by model_info.id, so a row without one takes every + # other id-less row down with it. router.get_model_list.return_value = [ { "litellm_params": {"custom_llm_provider": "openai"}, - "model_info": {"team_id": "other-team"}, + "model_info": {"id": "openai", "team_id": "other-team"}, }, { "litellm_params": {"custom_llm_provider": "bedrock"}, - "model_info": {"team_id": "active-team"}, + "model_info": {"id": "bedrock", "team_id": "active-team"}, }, ]