From b346eefd1f230b38f32e37ecb7a4ee8f73effb8b Mon Sep 17 00:00:00 2001 From: yucheng Date: Fri, 18 Sep 2026 08:33:46 +0000 Subject: [PATCH] fix(proxy): rerun requested model guardrail merge on fallback instead of carrying a raw list Structured guardrail entries (dicts) are unhashable, so merging a carried list through _merge_guardrails_with_existing raised TypeError on the fallback path. Resolve the requested model alias through _check_and_merge_model_level_guardrails instead and add a regression test for structured request guardrails Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/common_request_processing.py | 31 +++++++------------ litellm/proxy/utils.py | 8 +++-- .../proxy/test_common_request_processing.py | 20 ++++++++++++ 3 files changed, 36 insertions(+), 23 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 607ce3441e9..174b9ceff93 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -94,11 +94,7 @@ from litellm.proxy.common_utils.sse_keepalive import ( from litellm.proxy.dd_span_tagger import DDSpanTagger from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _arm_auto_router_compression from litellm.proxy.route_llm_request import route_request -from litellm.proxy.utils import ( - ProxyLogging, - _check_and_merge_model_level_guardrails, - _merge_guardrails_with_existing, -) +from litellm.proxy.utils import ProxyLogging, _check_and_merge_model_level_guardrails from litellm.router import Router from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict from litellm.router_utils.common_utils import resolve_model_group_alias @@ -1938,7 +1934,7 @@ class ProxyBaseLLMRequestProcessing: user_api_base: str | None = None, model: str | None = None, llm_router: Router | None = None, - requested_model_guardrails: list | None = None, + rate_limited_model: str | None = None, ) -> tuple[dict, LiteLLMLoggingObj]: start_time: Final = datetime.now() # start before calling guardrail hooks @@ -2102,12 +2098,15 @@ class ProxyBaseLLMRequestProcessing: # model_info when allow_client_pricing_override is set, so a caller # could otherwise spoof an unguarded model_info.id while requesting # a guarded alias and bypass guardrails (veria-ai HIGH on #29654). + merged_for_requested: Final = ( + self.data + if rate_limited_model is None + else _check_and_merge_model_level_guardrails( + data=self.data, llm_router=llm_router, trust_client_model_info=False, model_alias=rate_limited_model + ) + ) self.data = _check_and_merge_model_level_guardrails( - data=( - self.data - if requested_model_guardrails is None - else _merge_guardrails_with_existing(data=self.data, model_level_guardrails=requested_model_guardrails) - ), + data=merged_for_requested, llm_router=llm_router, trust_client_model_info=False, ) @@ -2217,8 +2216,6 @@ class ProxyBaseLLMRequestProcessing: original_model, fallback_models, ) - requested_model_guardrails: Final = self._request_guardrails(rate_limited_data) - try: for fallback_model in fallback_models: if fallback_model == original_model: @@ -2241,7 +2238,7 @@ class ProxyBaseLLMRequestProcessing: model=fallback_model, route_type=route_type, llm_router=llm_router, - requested_model_guardrails=requested_model_guardrails, + rate_limited_model=original_model, ) except ProxyRateLimitError: continue @@ -2252,12 +2249,6 @@ class ProxyBaseLLMRequestProcessing: self.data = rate_limited_data raise original_exc - @staticmethod - def _request_guardrails(data: dict) -> list | None: - metadata: Final = data.get("metadata") - guardrails: Final = metadata.get("guardrails") if isinstance(metadata, dict) else None - return guardrails if isinstance(guardrails, list) else None - @staticmethod def _configured_fallbacks(llm_router: Router, user_api_key_dict: UserAPIKeyAuth) -> list | None: key_router_settings: Final = user_api_key_dict.router_settings diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 950ac5e9906..ad0c0a82d49 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -7497,6 +7497,7 @@ def _check_and_merge_model_level_guardrails( data: dict, llm_router: Router | None, trust_client_model_info: bool = True, + model_alias: str | None = None, ) -> dict: """ Check if the model has guardrails defined and merge them with existing guardrails in the request data. @@ -7504,6 +7505,7 @@ def _check_and_merge_model_level_guardrails( Args: data: The request data dict llm_router: The LLM router instance to get deployment info from + model_alias: Resolve guardrails for this model group instead of data["model"] trust_client_model_info: If False, ignore metadata.model_info.id and resolve guardrails by alias-union only. Set to False on the pre_call path because add_litellm_data_to_request preserves @@ -7548,13 +7550,13 @@ def _check_and_merge_model_level_guardrails( # set on ANY eligible deployment still fires (#29652; addresses # veria-ai HIGH on the single-deployment fallback that would skip # non-first deployments). - model_alias: Final = data.get("model") - if not isinstance(model_alias, str) or not model_alias: + alias: Final = model_alias if model_alias is not None else data.get("model") + if not isinstance(alias, str) or not alias: return data # Pass team_id so team-scoped public model names resolve the same way # route_request resolves them; otherwise team-scoped deployments are # invisible to this lookup and their guardrails are silently dropped. - deployments: Final = llm_router.get_model_list(model_name=model_alias, team_id=team_id) or [] + deployments: Final = llm_router.get_model_list(model_name=alias, team_id=team_id) or [] seen: Final[set] = set() union: Final[list] = [] for dep in deployments: diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index f440711ac29..135ce019813 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -6970,6 +6970,26 @@ class TestPreCallWithFallbacksOnLocalRateLimit: assert data["messages"] == [{"role": "user", "content": "my ssn is [REDACTED-SSN]"}] assert rig[3] == [primary_model, primary_model, fallback_model] + @pytest.mark.asyncio + async def test_fallback_keeps_structured_request_guardrails(self, monkeypatch: pytest.MonkeyPatch): + primary_model = "gpt-4.1" + fallback_model = "gpt-4.1-mini" + structured_guardrail = {"pii-guard": {"extra_body": {"threshold": 0.5}}} + key = self._otel_key(model_rpm_limit={primary_model: 1}) + rig = self._v3_limiter_rig(monkeypatch, key, [{primary_model: [fallback_model]}]) + request = { + "model": primary_model, + "messages": [{"role": "user", "content": "hi"}], + "guardrails": [structured_guardrail], + } + + await self._pre_call(dict(request), key, rig) + _, (data, _) = await self._pre_call(dict(request), key, rig) + + assert data["model"] == fallback_model + assert data["metadata"]["guardrails"] == [structured_guardrail] + assert rig[3] == [primary_model, primary_model, fallback_model] + class _RecordingSuccessLogger(CustomLogger): def __init__(self):