From 8c72342ad58ed4731022c1f2d6401b620426a6ee Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:25:34 -0700 Subject: [PATCH 1/4] fix(guardrails): resync event_hook and accept raw dicts in in-memory guardrail updates --- basedpyright-code-budget.json | 18 ++--- litellm/integrations/custom_guardrail.py | 73 ++++++++++++++----- .../guardrail_hooks/azure/prompt_shield.py | 40 ++++------ .../guardrail_hooks/bedrock_guardrails.py | 5 +- .../guardrail_hooks/lakera_ai_v2.py | 25 +++---- .../model_armor/model_armor.py | 2 +- .../guardrails/guardrail_hooks/presidio.py | 47 ++++++------ .../guardrail_hooks/qualifire/qualifire.py | 9 ++- .../guardrail_hooks/tool_permission.py | 18 ++--- .../zscaler_ai_guard/zscaler_ai_guard.py | 7 +- .../proxy/guardrails/guardrail_registry.py | 16 ++-- ruff-strict-budget.json | 2 +- .../integrations/test_custom_guardrail.py | 64 ++++++++++++++++ .../guardrail_hooks/test_presidio.py | 37 +++++++++- .../guardrails/test_guardrail_registry.py | 59 +++++++++++++++ type-discipline-budget.json | 8 +- 16 files changed, 308 insertions(+), 122 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index df52069e71f..4e82e0752ca 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -1,6 +1,6 @@ { "reportAny": { - "limit": 14076 + "limit": 14075 }, "reportArgumentType": { "limit": 2216 @@ -9,7 +9,7 @@ "limit": 319 }, "reportAttributeAccessIssue": { - "limit": 480 + "limit": 479 }, "reportCallIssue": { "limit": 112 @@ -57,7 +57,7 @@ "limit": 5601 }, "reportMissingTypeArgument": { - "limit": 15306 + "limit": 15303 }, "reportMissingTypeStubs": { "limit": 40 @@ -99,22 +99,22 @@ "limit": 0 }, "reportUnknownArgumentType": { - "limit": 44364 + "limit": 44360 }, "reportUnknownLambdaType": { "limit": 109 }, "reportUnknownMemberType": { - "limit": 38350 + "limit": 38346 }, "reportUnknownParameterType": { - "limit": 19626 + "limit": 19623 }, "reportUnknownVariableType": { - "limit": 29890 + "limit": 29881 }, "reportUnnecessaryCast": { - "limit": 111 + "limit": 110 }, "reportUnnecessaryComparison": { "limit": 695 @@ -123,7 +123,7 @@ "limit": 5 }, "reportUnnecessaryIsInstance": { - "limit": 826 + "limit": 825 }, "reportUntypedBaseClass": { "limit": 0 diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index e87ac9521ae..8754d116537 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -2,10 +2,12 @@ import contextvars import hashlib import os import secrets -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from datetime import datetime from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, get_args +from pydantic import TypeAdapter + from litellm._logging import verbose_logger from litellm.caching import DualCache from litellm.integrations.custom_logger import CustomLogger @@ -121,6 +123,18 @@ def _strict_guardrail_modes_enabled() -> bool: return True if parsed is None else parsed +def updated_litellm_param(litellm_params: "LitellmParams | Mapping[str, object]", key: str) -> object: + if isinstance(litellm_params, Mapping): + return litellm_params.get(key) + value: Final[object] = getattr(litellm_params, key, None) + return value + + +GUARDRAIL_MODE_ADAPTER: Final[TypeAdapter[GuardrailEventHooks | list[GuardrailEventHooks] | Mode]] = TypeAdapter( + GuardrailEventHooks | list[GuardrailEventHooks] | Mode +) + + def get_session_id_from_request_data(request_data: dict[str, Any]) -> str | None: """Extract session_id from request data (litellm_session_id or metadata).""" session_id = request_data.get("litellm_session_id") @@ -214,18 +228,7 @@ class CustomGuardrail(CustomLogger): self.only_scan_new_messages: bool = only_scan_new_messages if supported_event_hooks: - ## validate event_hook is in supported_event_hooks - try: - self._validate_event_hook(event_hook, supported_event_hooks) - except ValueError as validation_error: - if _strict_guardrail_modes_enabled(): - raise - verbose_logger.warning( - "%s. LITELLM_STRICT_GUARDRAIL_MODES=false; continuing " - "with unsupported event_hook. Set the env var to true " - "(default) to enforce validation and fail at startup.", - validation_error, - ) + self._validate_or_warn_event_hook(event_hook, supported_event_hooks) super().__init__(**kwargs) def render_violation_message(self, default: str, context: Mapping[str, object] | None = None) -> str: @@ -588,12 +591,12 @@ class CustomGuardrail(CustomLogger): def _validate_event_hook( self, - event_hook: GuardrailEventHooks | list[GuardrailEventHooks] | Mode | None, - supported_event_hooks: list[GuardrailEventHooks], + event_hook: GuardrailEventHooks | Sequence[GuardrailEventHooks] | Mode | None, + supported_event_hooks: Sequence[GuardrailEventHooks], ) -> None: def _validate_event_hook_list_is_in_supported_event_hooks( - event_hook: list[GuardrailEventHooks] | list[str], - supported_event_hooks: list[GuardrailEventHooks], + event_hook: Sequence[GuardrailEventHooks] | Sequence[str], + supported_event_hooks: Sequence[GuardrailEventHooks], ) -> None: for hook in event_hook: if isinstance(hook, str): @@ -622,6 +625,23 @@ class CustomGuardrail(CustomLogger): if event_hook not in supported_event_hooks: raise ValueError(f"Event hook {event_hook} is not in the supported event hooks {supported_event_hooks}") + def _validate_or_warn_event_hook( + self, + event_hook: GuardrailEventHooks | Sequence[GuardrailEventHooks] | Mode | None, + supported_event_hooks: Sequence[GuardrailEventHooks], + ) -> None: + try: + self._validate_event_hook(event_hook, supported_event_hooks) + except ValueError as validation_error: + if _strict_guardrail_modes_enabled(): + raise + verbose_logger.warning( + "%s. LITELLM_STRICT_GUARDRAIL_MODES=false; continuing " + "with unsupported event_hook. Set the env var to true " + "(default) to enforce validation and fail at startup.", + validation_error, + ) + @staticmethod def _get_admin_metadata(data: dict) -> dict: """Return merged admin-configured key and team metadata from the request data. @@ -1271,12 +1291,25 @@ class CustomGuardrail(CustomLogger): # Mask the content return content_string[:start_index] + mask_string + content_string[end_index:] - def update_in_memory_litellm_params(self, litellm_params: LitellmParams) -> None: + def update_in_memory_litellm_params(self, litellm_params: "LitellmParams | Mapping[str, object]") -> None: """ - Update the guardrails litellm params in memory + Update the guardrails litellm params in memory, accepting either a + LitellmParams object or the raw params mapping stored in the DB, and + resync ``event_hook`` when the update carries a new ``mode``. The new + mode is validated against ``supported_event_hooks`` before any state + is mutated, so a rejected update leaves the guardrail untouched. """ - for key, value in vars(litellm_params).items(): + updated_params: Final[Mapping[str, object]] = ( + litellm_params if isinstance(litellm_params, Mapping) else vars(litellm_params) + ) + raw_mode: Final = updated_params.get("mode") + new_event_hook: Final = None if raw_mode is None else GUARDRAIL_MODE_ADAPTER.validate_python(raw_mode) + if new_event_hook is not None and self.supported_event_hooks: + self._validate_or_warn_event_hook(new_event_hook, self.supported_event_hooks) + for key, value in updated_params.items(): setattr(self, key, value) + if new_event_hook is not None: + self.event_hook = new_event_hook def get_guardrails_messages_for_call_type( self, call_type: CallTypes, data: dict | None = None diff --git a/litellm/proxy/guardrails/guardrail_hooks/azure/prompt_shield.py b/litellm/proxy/guardrails/guardrail_hooks/azure/prompt_shield.py index 6e29d44662e..58bebfbdb6e 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/azure/prompt_shield.py +++ b/litellm/proxy/guardrails/guardrail_hooks/azure/prompt_shield.py @@ -14,6 +14,7 @@ from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, + updated_litellm_param, ) from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import ( AZURE_PROMPT_SHIELD_TEXT_RECORD_UNIT, @@ -61,15 +62,6 @@ def _resolved_secret_value(value: object) -> object: return value -def _updated_param(litellm_params: "LitellmParams | dict", key: str) -> object: # mutable-ok: DB dict - """Read one param from a Mapping or a pydantic object, including pydantic - extras (cost_tier / price_per_1000_text_records live there), which the base - class ``vars()`` loop never sees.""" - if isinstance(litellm_params, Mapping): - return litellm_params.get(key) - return getattr(litellm_params, key, None) - - def _resolved_cost_tier(raw: object) -> str | None: """Normalize the configured cost_tier to 'free' / 'paid' / None.""" value: Final = _resolved_secret_value(raw) @@ -270,29 +262,27 @@ class AzureContentSafetyPromptShieldGuardrail(AzureGuardrailBase, CustomGuardrai verbose_proxy_logger.warning("Azure Prompt Shield: No user prompt found") return None - def update_in_memory_litellm_params(self, litellm_params: "LitellmParams | dict") -> None: # mutable-ok: DB dict + def update_in_memory_litellm_params(self, litellm_params: "LitellmParams | Mapping[str, object]") -> None: """Apply updated params in place, re-resolving billing and credentials. - Pricing is read via ``_updated_param`` (the values are pydantic extras, and - the immediate PUT sync hands this method the raw DB dict). Pricing and any - ``os.environ/`` credential references are validated and resolved BEFORE any - state is mutated, so an invalid update leaves the running guardrail - untouched and a raw reference never overwrites a resolved credential. + Pricing is read via ``updated_litellm_param`` (the values are pydantic + extras, and the immediate PUT sync hands this method the raw DB dict). + Pricing and any ``os.environ/`` credential references are validated and + resolved BEFORE any state is mutated, so an invalid update leaves the + running guardrail untouched and a raw reference never overwrites a + resolved credential. Both input shapes flow through the base update so + the event_hook resync applies to each. """ - cost_tier: Final = _resolved_cost_tier(_updated_param(litellm_params, "cost_tier")) - price: Final = _resolved_price(_updated_param(litellm_params, "price_per_1000_text_records"), cost_tier) + cost_tier: Final = _resolved_cost_tier(updated_litellm_param(litellm_params, "cost_tier")) + price: Final = _resolved_price(updated_litellm_param(litellm_params, "price_per_1000_text_records"), cost_tier) resolved_credentials: dict[str, object] = {} # mutable-ok: staged before mutation for cred_key in ("api_key", "api_base"): - cred_value = _updated_param(litellm_params, cred_key) + cred_value = updated_litellm_param(litellm_params, cred_key) if isinstance(cred_value, str) and cred_value.startswith("os.environ/"): resolved_credentials[cred_key] = _resolved_secret_value(cred_value) - if isinstance(litellm_params, Mapping): - for key, value in litellm_params.items(): - setattr(self, key, resolved_credentials.get(key, value)) - else: - super().update_in_memory_litellm_params(litellm_params) - for cred_key, cred_value in resolved_credentials.items(): - setattr(self, cred_key, cred_value) + super().update_in_memory_litellm_params(litellm_params) + for cred_key, cred_value in resolved_credentials.items(): + setattr(self, cred_key, cred_value) self.cost_tier = cost_tier self.price_per_1000_text_records = price diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 30526d30dc5..17ca36ea6a4 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -317,9 +317,10 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): self.streaming_sampling_rate = streaming_params.streaming_sampling_rate self.streaming_end_of_stream_only = streaming_params.streaming_end_of_stream_only - def update_in_memory_litellm_params(self, litellm_params: LitellmParams) -> None: + def update_in_memory_litellm_params(self, litellm_params: "LitellmParams | Mapping[str, object]") -> None: super().update_in_memory_litellm_params(litellm_params) - self._set_streaming_params(BedrockGuardrailStreamingParams.from_extras(litellm_params.model_extra)) + extras: Final = litellm_params if isinstance(litellm_params, Mapping) else litellm_params.model_extra + self._set_streaming_params(BedrockGuardrailStreamingParams.from_extras(extras)) def _streams_incrementally(self) -> bool: return not self.streaming_buffer_until_moderated and not self.mask_response_content diff --git a/litellm/proxy/guardrails/guardrail_hooks/lakera_ai_v2.py b/litellm/proxy/guardrails/guardrail_hooks/lakera_ai_v2.py index 2f98a9afbd8..9e1382feb21 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/lakera_ai_v2.py +++ b/litellm/proxy/guardrails/guardrail_hooks/lakera_ai_v2.py @@ -13,6 +13,7 @@ from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( DEFAULT_ADVISORY_MESSAGE, CustomGuardrail, + updated_litellm_param, ) from litellm.llms.base_llm.guardrail_translation.utils import ( effective_skip_system_message_for_guardrail, @@ -304,7 +305,7 @@ class LakeraAIGuardrail(CustomGuardrail): breakdown=self.breakdown, ) - def update_in_memory_litellm_params(self, litellm_params: LitellmParams) -> None: + def update_in_memory_litellm_params(self, litellm_params: "LitellmParams | Mapping[str, object]") -> None: """ The base implementation blindly ``setattr``s every field on ``litellm_params`` (including ``on_flagged``/``advisory_system_message``/``payload``/``breakdown``) @@ -313,24 +314,18 @@ class LakeraAIGuardrail(CustomGuardrail): on_flagged combinations __init__ rejects. Validate the prospective post-update state *before* mutating, so a rejected update leaves the live instance untouched instead of raising after it's already been corrupted. - - The base setattr also writes ``litellm_params.mode`` onto a new ``self.mode`` - attribute rather than the ``self.event_hook`` dispatch actually reads - (LitellmParams has no field literally named ``event_hook``), so without the - explicit sync below a hot reload that changes mode would pass validation but - keep dispatching on the stale event_hook. """ - new_event_hook: Final = litellm_params.mode or self.event_hook - prospective_payload: Final = litellm_params.payload - prospective_breakdown: Final = litellm_params.breakdown + raw_on_flagged: Final = updated_litellm_param(litellm_params, "on_flagged") + raw_advisory: Final = updated_litellm_param(litellm_params, "advisory_system_message") + raw_payload: Final = updated_litellm_param(litellm_params, "payload") + raw_breakdown: Final = updated_litellm_param(litellm_params, "breakdown") self._validate_advisory_config( - on_flagged=litellm_params.on_flagged or self.on_flagged, - advisory_system_message=litellm_params.advisory_system_message, - payload=self.payload if prospective_payload is None else prospective_payload, - breakdown=self.breakdown if prospective_breakdown is None else prospective_breakdown, + on_flagged=raw_on_flagged if isinstance(raw_on_flagged, str) and raw_on_flagged else self.on_flagged, + advisory_system_message=raw_advisory if isinstance(raw_advisory, str) else None, + payload=raw_payload if isinstance(raw_payload, bool) else self.payload, + breakdown=raw_breakdown if isinstance(raw_breakdown, bool) else self.breakdown, ) super().update_in_memory_litellm_params(litellm_params=litellm_params) - self.event_hook = new_event_hook def _validate_advisory_config( self, diff --git a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py index d187b5b12e9..c96334fb2f9 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py +++ b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py @@ -185,7 +185,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): if self.optional_params.get("fail_on_error", True): raise e from None - def update_in_memory_litellm_params(self, litellm_params: LitellmParams) -> None: + def update_in_memory_litellm_params(self, litellm_params: "LitellmParams | Mapping[str, object]") -> None: super().update_in_memory_litellm_params(litellm_params) self.sanitize_error_detail = self.sanitize_error_detail is not False diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index da51a905ae3..da2776eda7d 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -11,7 +11,7 @@ import asyncio import json import threading -from collections.abc import AsyncGenerator, AsyncIterable, Awaitable, Sequence +from collections.abc import AsyncGenerator, AsyncIterable, Awaitable, Mapping, Sequence from contextlib import asynccontextmanager from datetime import datetime from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, TypedDict, cast @@ -35,6 +35,7 @@ if TYPE_CHECKING: from litellm.caching.caching import DualCache from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException from litellm.integrations.custom_guardrail import ( + GUARDRAIL_MODE_ADAPTER, CustomGuardrail, log_guardrail_information, ) @@ -530,17 +531,17 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): return created @staticmethod - def _coerce_analyze_chunk_size(value: int | None) -> int: + def _coerce_analyze_chunk_size(value: object) -> int: """ Validate a configured chunk size, falling back to the default. - Non-positive values would either bypass chunking entirely or degenerate - it into per-character splits (silently disabling detection), so they are - replaced by the default; values below 4 bytes are floored to 4 and the - splitter always emits at least one character per chunk, so the chunked - path can never re-enter itself. + Non-positive or non-integer values would either bypass chunking entirely + or degenerate it into per-character splits (silently disabling + detection), so they are replaced by the default; values below 4 bytes + are floored to 4 and the splitter always emits at least one character + per chunk, so the chunked path can never re-enter itself. """ - if not value or value <= 0: + if not isinstance(value, int) or value <= 0: return DEFAULT_PRESIDIO_ANALYZE_CHUNK_SIZE_BYTES return max(value, 4) @@ -1628,20 +1629,24 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): inputs["texts"] = new_texts return inputs - def update_in_memory_litellm_params(self, litellm_params: LitellmParams) -> None: + def update_in_memory_litellm_params(self, litellm_params: "LitellmParams | Mapping[str, object]") -> None: """ Update the guardrails litellm params in memory """ super().update_in_memory_litellm_params(litellm_params) - if litellm_params.pii_entities_config: - self.pii_entities_config = litellm_params.pii_entities_config - if litellm_params.presidio_score_thresholds: - self.presidio_score_thresholds = litellm_params.presidio_score_thresholds - if litellm_params.presidio_entities_deny_list: - self.presidio_entities_deny_list = litellm_params.presidio_entities_deny_list - if litellm_params.presidio_analyze_chunk_size_bytes is not None: - # Same validation as __init__: a non-positive value from a guardrail - # update must not silently disable detection via degenerate chunking. - self.presidio_analyze_chunk_size_bytes = self._coerce_analyze_chunk_size( - litellm_params.presidio_analyze_chunk_size_bytes - ) + self.presidio_analyze_chunk_size_bytes = self._coerce_analyze_chunk_size(self.presidio_analyze_chunk_size_bytes) + self._resync_output_stage_event_hook() + + def _resync_output_stage_event_hook(self) -> None: + if self.event_hook == GuardrailEventHooks.logging_only: + return + if self.apply_to_output: + self.event_hook = GuardrailEventHooks.post_call + return + if not self.output_parse_pii: + return + current_hook: Final = self.event_hook + if isinstance(current_hook, str) and current_hook != "post_call": + self.event_hook = GUARDRAIL_MODE_ADAPTER.validate_python((current_hook, GuardrailEventHooks.post_call)) + elif isinstance(current_hook, list) and "post_call" not in current_hook: + self.event_hook = GUARDRAIL_MODE_ADAPTER.validate_python((*current_hook, GuardrailEventHooks.post_call)) diff --git a/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py b/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py index f834426d619..cc1234da4d9 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py +++ b/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py @@ -7,6 +7,7 @@ import json import os +from collections.abc import Mapping from typing import Any, Final, Literal from fastapi import HTTPException @@ -15,6 +16,7 @@ from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, + updated_litellm_param, ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.custom_httpx.http_handler import ( @@ -111,7 +113,7 @@ class QualifireGuardrail(CustomGuardrail): "only 'block' and 'monitor' are supported." ) - def update_in_memory_litellm_params(self, litellm_params: LitellmParams) -> None: + def update_in_memory_litellm_params(self, litellm_params: "LitellmParams | Mapping[str, object]") -> None: """ The base implementation blindly ``setattr``s every field on ``litellm_params`` (including ``on_flagged``) onto this live instance with no revalidation, so an @@ -121,7 +123,10 @@ class QualifireGuardrail(CustomGuardrail): the live instance untouched instead of raising after it's already been corrupted. Mirrors LakeraAIGuardrail's own override of this same method. """ - prospective_on_flagged: Final = litellm_params.on_flagged or self.on_flagged + raw_on_flagged: Final = updated_litellm_param(litellm_params, "on_flagged") + prospective_on_flagged: Final = ( + raw_on_flagged if isinstance(raw_on_flagged, str) and raw_on_flagged else self.on_flagged + ) self._validate_on_flagged(prospective_on_flagged) super().update_in_memory_litellm_params(litellm_params=litellm_params) diff --git a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py index a8b33109900..88e24db207a 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py +++ b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py @@ -159,7 +159,7 @@ class ToolPermissionGuardrail(CustomGuardrail): self._compiled_rule_targets = compiled_targets self._compiled_rule_patterns = compiled_patterns - def update_in_memory_litellm_params(self, litellm_params: LitellmParams | dict) -> None: + def update_in_memory_litellm_params(self, litellm_params: "LitellmParams | Mapping[str, object]") -> None: """Apply updated params in place, rebuilding the compiled rule state. The base implementation only ``setattr``s raw fields, which would leave @@ -169,17 +169,11 @@ class ToolPermissionGuardrail(CustomGuardrail): immediate in-memory sync take effect, mirroring the PresidioGuardrail override of this method. """ - # ``litellm_params`` may arrive as the raw DB dict (the proxy ``cast()``s - # it to ``LitellmParams`` without converting), so handle both shapes. The - # base ``setattr`` loop is model-only, so apply the dict case here. previous_rules: Final = self.rules - if isinstance(litellm_params, dict): - params = litellm_params - for key, value in params.items(): - setattr(self, key, value) - else: - super().update_in_memory_litellm_params(litellm_params) - params = vars(litellm_params) + params: Final[Mapping[str, object]] = ( + litellm_params if isinstance(litellm_params, Mapping) else vars(litellm_params) + ) + super().update_in_memory_litellm_params(litellm_params) # The generic update above sets ``self.rules`` from the incoming value # (None on a partial update that omits rules), but never rebuilds the @@ -187,7 +181,7 @@ class ToolPermissionGuardrail(CustomGuardrail): # the previous ruleset so a partial update doesn't silently wipe it. An # explicit empty list still clears the rules. rules: Final = params.get("rules") - if rules is not None: + if isinstance(rules, list): try: self._load_rules(rules) except Exception: diff --git a/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py b/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py index 1aefa38ecf8..2928ea5d068 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py +++ b/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py @@ -4,6 +4,7 @@ # # +-------------------------------------------------------------+ import os +from collections.abc import Mapping from typing import TYPE_CHECKING, Final, Literal, Optional from fastapi import HTTPException @@ -12,6 +13,7 @@ from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, + updated_litellm_param, ) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, @@ -102,9 +104,10 @@ class ZscalerAIGuard(CustomGuardrail): return timeout - def update_in_memory_litellm_params(self, litellm_params: "LitellmParams") -> None: + def update_in_memory_litellm_params(self, litellm_params: "LitellmParams | Mapping[str, object]") -> None: super().update_in_memory_litellm_params(litellm_params) - self.timeout = self._resolve_timeout(litellm_params.timeout) + raw_timeout: Final = updated_litellm_param(litellm_params, "timeout") + self.timeout = self._resolve_timeout(raw_timeout if isinstance(raw_timeout, (int, float)) else None) @staticmethod def _resolve_metadata_value(request_data: dict | None, key: str) -> str | None: diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index dc13c09dd38..3abd952da8d 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -6,7 +6,7 @@ import os from collections.abc import Callable, Iterator, Mapping from datetime import datetime, timezone from itertools import chain, count -from typing import TYPE_CHECKING, Final, Literal, Optional, Protocol, cast +from typing import TYPE_CHECKING, Final, Literal, Optional, Protocol from pydantic import ValidationError @@ -624,17 +624,19 @@ class InMemoryGuardrailHandler: """ Update a guardrail in memory - - updates the guardrail in memory - updates the guardrail params in litellm.callback_manager + - stores the guardrail in memory only after the callback update + succeeds, so a failed update stays visible as a diff to the + per-worker DB poller and gets retried instead of going stale """ + custom_guardrail_callback: Final = self.guardrail_id_to_custom_guardrail.get(guardrail_id) + updated_litellm_params: Final = guardrail.get("litellm_params") + if custom_guardrail_callback and updated_litellm_params: + custom_guardrail_callback.update_in_memory_litellm_params(litellm_params=updated_litellm_params) + self.IN_MEMORY_GUARDRAILS[guardrail_id] = guardrail self._sources[guardrail_id] = source - custom_guardrail_callback: Final = self.guardrail_id_to_custom_guardrail.get(guardrail_id) - if custom_guardrail_callback: - updated_litellm_params: Final = cast(LitellmParams, guardrail.get("litellm_params", {})) - custom_guardrail_callback.update_in_memory_litellm_params(litellm_params=updated_litellm_params) - def delete_in_memory_guardrail(self, guardrail_id: str) -> None: """ Delete a guardrail in memory and remove from litellm callbacks. diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index 9b1cc977a64..f43e8c6e93e 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -231,7 +231,7 @@ "limit": 5 }, "TID251": { - "limit": 1073 + "limit": 1072 }, "TRY002": { "limit": 524 diff --git a/tests/test_litellm/integrations/test_custom_guardrail.py b/tests/test_litellm/integrations/test_custom_guardrail.py index d978eb48c12..c84fcd5ac11 100644 --- a/tests/test_litellm/integrations/test_custom_guardrail.py +++ b/tests/test_litellm/integrations/test_custom_guardrail.py @@ -9,6 +9,7 @@ from litellm.integrations.custom_guardrail import ( log_guardrail_information, ) from litellm.proxy._types import CallTypes, UserAPIKeyAuth +from litellm.types.guardrails import GuardrailEventHooks, LitellmParams from litellm.types.utils import GenericGuardrailAPIInputs, GuardrailTracingDetail @@ -2237,3 +2238,66 @@ class TestRecordsOwnGuardrailInformation: ) assert _guardrail_entries(request_data) == [] + + +class TestUpdateInMemoryLitellmParams: + """A PUT /guardrails update reaches the live callback through + update_in_memory_litellm_params: it must accept both a LitellmParams object + and the raw DB dict, and resync self.event_hook (which dispatch reads) from + the incoming mode instead of only writing a dead self.mode attribute (LIT-6591).""" + + def _guardrail(self) -> CustomGuardrail: + return CustomGuardrail( + guardrail_name="update-test", + event_hook=GuardrailEventHooks.pre_call, + default_on=True, + supported_event_hooks=[GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call], + ) + + def test_mode_change_resyncs_event_hook_dispatch(self): + guardrail = self._guardrail() + + guardrail.update_in_memory_litellm_params( + LitellmParams(guardrail="update-test", mode="post_call", default_on=True) + ) + + assert guardrail.event_hook is GuardrailEventHooks.post_call + assert guardrail.should_run_guardrail(data={}, event_type=GuardrailEventHooks.post_call) is True + assert guardrail.should_run_guardrail(data={}, event_type=GuardrailEventHooks.pre_call) is False + + def test_raw_db_dict_copies_params_and_resyncs_event_hook(self): + guardrail = self._guardrail() + + guardrail.update_in_memory_litellm_params( + { + "guardrail": "update-test", + "mode": "post_call", + "api_base": "https://guardrail.example.com", + "default_on": True, + } + ) + + assert guardrail.event_hook is GuardrailEventHooks.post_call + assert getattr(guardrail, "api_base", None) == "https://guardrail.example.com" + assert guardrail.should_run_guardrail(data={}, event_type=GuardrailEventHooks.post_call) is True + + def test_strict_mode_rejects_unsupported_mode_without_mutating(self, monkeypatch): + monkeypatch.delenv("LITELLM_STRICT_GUARDRAIL_MODES", raising=False) + guardrail = self._guardrail() + + with pytest.raises(ValueError, match="not in the supported event hooks"): + guardrail.update_in_memory_litellm_params( + {"mode": "during_call", "api_base": "https://guardrail.example.com"} + ) + + assert guardrail.event_hook is GuardrailEventHooks.pre_call + assert getattr(guardrail, "api_base", None) is None + + def test_non_strict_mode_warns_and_applies_unsupported_mode(self, monkeypatch): + monkeypatch.setenv("LITELLM_STRICT_GUARDRAIL_MODES", "false") + guardrail = self._guardrail() + + guardrail.update_in_memory_litellm_params({"mode": "during_call", "api_base": "https://guardrail.example.com"}) + + assert guardrail.event_hook is GuardrailEventHooks.during_call + assert getattr(guardrail, "api_base", None) == "https://guardrail.example.com" diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py index 4ee6741ee02..519df980d22 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py @@ -17,7 +17,7 @@ from litellm.proxy.guardrails.guardrail_hooks.presidio import ( _OPTIONAL_PresidioPIIMasking, ) from litellm.exceptions import GuardrailRaisedException -from litellm.types.guardrails import LitellmParams, PiiAction, PiiEntityType +from litellm.types.guardrails import GuardrailEventHooks, LitellmParams, PiiAction, PiiEntityType from litellm.types.utils import Choices, Message, ModelResponse from litellm.exceptions import BlockedPiiEntityError @@ -3167,6 +3167,41 @@ def test_update_in_memory_coerces_invalid_chunk_size(): assert guardrail.presidio_analyze_chunk_size_bytes == DEFAULT_PRESIDIO_ANALYZE_CHUNK_SIZE_BYTES +def test_update_in_memory_output_callback_keeps_forced_post_call(): + """The registry-tracked callback for filter_scope='output' is initialized with a + forced post_call hook regardless of the configured mode; a mode-changing update + must not move it off the response stage (LIT-6591).""" + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + apply_to_output=True, + event_hook=GuardrailEventHooks.post_call.value, + ) + + guardrail.update_in_memory_litellm_params({"guardrail": "presidio", "mode": "pre_call", "default_on": True}) + + assert guardrail.event_hook is GuardrailEventHooks.post_call + assert guardrail.should_run_guardrail(data={}, event_type=GuardrailEventHooks.post_call) is True + assert guardrail.should_run_guardrail(data={}, event_type=GuardrailEventHooks.pre_call) is False + + +def test_update_in_memory_output_parse_pii_keeps_post_call_expansion(): + """A guardrail with output_parse_pii must keep running on post_call to unmask the + response after a mode-changing update, mirroring the constructor's expansion.""" + guardrail = _OPTIONAL_PresidioPIIMasking( + mock_testing=True, + output_parse_pii=True, + event_hook="pre_call", + ) + + guardrail.update_in_memory_litellm_params( + LitellmParams(guardrail="presidio", mode="during_call", output_parse_pii=True, default_on=True) + ) + + assert guardrail.should_run_guardrail(data={}, event_type=GuardrailEventHooks.during_call) is True + assert guardrail.should_run_guardrail(data={}, event_type=GuardrailEventHooks.post_call) is True + assert guardrail.should_run_guardrail(data={}, event_type=GuardrailEventHooks.pre_call) is False + + def test_split_text_handles_chunk_size_below_char_width(): chunks = _OPTIONAL_PresidioPIIMasking._split_text_for_analysis( text="\U0001f642\U0001f642", chunk_size_bytes=3, overlap_chars=8 diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py index 2c0735970d3..015d530257b 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py @@ -179,6 +179,65 @@ def test_update_in_memory_guardrail(): assert handler.guardrail_id_to_custom_guardrail["123"].event_hook is GuardrailEventHooks.pre_call +def test_update_in_memory_guardrail_raw_db_dict_resyncs_event_hook(): + """PUT /guardrails hands this method the raw DB row, whose litellm_params is a + plain dict; the update must still apply and move dispatch to the new mode + instead of raising inside vars() and leaving the worker stale (LIT-6591).""" + handler = InMemoryGuardrailHandler() + handler.guardrail_id_to_custom_guardrail["123"] = CustomGuardrail( + guardrail_name="test-guardrail", + default_on=True, + event_hook=GuardrailEventHooks.pre_call, + supported_event_hooks=[GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call], + ) + + updated_row = { + "guardrail_id": "123", + "guardrail_name": "test-guardrail", + "litellm_params": {"guardrail": "test-guardrail", "mode": "post_call", "default_on": True}, + } + handler.update_in_memory_guardrail("123", updated_row) + + callback = handler.guardrail_id_to_custom_guardrail["123"] + assert callback.event_hook is GuardrailEventHooks.post_call + assert callback.should_run_guardrail(data={}, event_type=GuardrailEventHooks.post_call) is True + assert callback.should_run_guardrail(data={}, event_type=GuardrailEventHooks.pre_call) is False + assert handler.IN_MEMORY_GUARDRAILS["123"] == updated_row + + +def test_update_in_memory_guardrail_failed_callback_update_stays_visible_to_poller(monkeypatch): + """When the callback update raises, IN_MEMORY_GUARDRAILS must keep the old row: + storing the new row first would make the per-worker DB poller see no diff and + never re-initialize, leaving the PUT-serving worker stale until restart.""" + monkeypatch.delenv("LITELLM_STRICT_GUARDRAIL_MODES", raising=False) + handler = InMemoryGuardrailHandler() + stale_row = Guardrail( + guardrail_id="123", + guardrail_name="test-guardrail", + litellm_params=LitellmParams(guardrail="test-guardrail", mode="pre_call", default_on=True), + ) + handler.IN_MEMORY_GUARDRAILS["123"] = stale_row + handler.guardrail_id_to_custom_guardrail["123"] = CustomGuardrail( + guardrail_name="test-guardrail", + default_on=True, + event_hook=GuardrailEventHooks.pre_call, + supported_event_hooks=[GuardrailEventHooks.pre_call], + ) + + with pytest.raises(ValueError, match="not in the supported event hooks"): + handler.update_in_memory_guardrail( + "123", + { + "guardrail_id": "123", + "guardrail_name": "test-guardrail", + "litellm_params": {"guardrail": "test-guardrail", "mode": "post_call", "default_on": True}, + }, + ) + + assert handler.IN_MEMORY_GUARDRAILS["123"] == stale_row + assert handler.guardrail_id_to_custom_guardrail["123"].event_hook is GuardrailEventHooks.pre_call + + def _make_guardrail(guardrail_id: str, name: str = "g") -> Guardrail: return Guardrail( guardrail_id=guardrail_id, diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 3d2e97d55a5..73ea794f8a8 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,9 +1,9 @@ { "LIT001": { - "limit": 22367 + "limit": 22362 }, "LIT002": { - "limit": 26777 + "limit": 26776 }, "LIT003": { "limit": 269 @@ -15,7 +15,7 @@ "limit": 0 }, "LIT006": { - "limit": 1039 + "limit": 1038 }, "LIT007": { "limit": 0 @@ -27,7 +27,7 @@ "limit": 0 }, "LIT010": { - "limit": 16507 + "limit": 16505 }, "LIT011": { "limit": 5535 From 2286091a9a08e231e87c6e8abe7274364d7a14ad Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:38:38 -0700 Subject: [PATCH 2/4] fix(guardrails): skip None fields in in-memory guardrail updates so constructor defaults survive --- litellm/integrations/custom_guardrail.py | 6 +++++- .../guardrail_hooks/tool_permission.py | 8 ++++---- .../integrations/test_custom_guardrail.py | 18 ++++++++++++++++++ 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 8754d116537..1562f3d092e 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -1298,6 +1298,9 @@ class CustomGuardrail(CustomLogger): resync ``event_hook`` when the update carries a new ``mode``. The new mode is validated against ``supported_event_hooks`` before any state is mutated, so a rejected update leaves the guardrail untouched. + ``None`` values are skipped because both sources serialize every unset + LitellmParams field as ``None``; applying them would clobber + constructor-derived state (e.g. dict defaults) with ``None``. """ updated_params: Final[Mapping[str, object]] = ( litellm_params if isinstance(litellm_params, Mapping) else vars(litellm_params) @@ -1307,7 +1310,8 @@ class CustomGuardrail(CustomLogger): if new_event_hook is not None and self.supported_event_hooks: self._validate_or_warn_event_hook(new_event_hook, self.supported_event_hooks) for key, value in updated_params.items(): - setattr(self, key, value) + if value is not None: + setattr(self, key, value) if new_event_hook is not None: self.event_hook = new_event_hook diff --git a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py index 88e24db207a..d6bea312031 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py +++ b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py @@ -176,10 +176,10 @@ class ToolPermissionGuardrail(CustomGuardrail): super().update_in_memory_litellm_params(litellm_params) # The generic update above sets ``self.rules`` from the incoming value - # (None on a partial update that omits rules), but never rebuilds the - # compiled maps. Rebuild them when rules are provided; otherwise restore - # the previous ruleset so a partial update doesn't silently wipe it. An - # explicit empty list still clears the rules. + # (skipping None) but never rebuilds the compiled maps. Rebuild them + # when a rules list is provided; otherwise restore the previous ruleset + # so a non-list value can't silently wipe it. An explicit empty list + # still clears the rules. rules: Final = params.get("rules") if isinstance(rules, list): try: diff --git a/tests/test_litellm/integrations/test_custom_guardrail.py b/tests/test_litellm/integrations/test_custom_guardrail.py index c84fcd5ac11..3983f698ef0 100644 --- a/tests/test_litellm/integrations/test_custom_guardrail.py +++ b/tests/test_litellm/integrations/test_custom_guardrail.py @@ -2281,6 +2281,24 @@ class TestUpdateInMemoryLitellmParams: assert getattr(guardrail, "api_base", None) == "https://guardrail.example.com" assert guardrail.should_run_guardrail(data={}, event_type=GuardrailEventHooks.post_call) is True + def test_none_values_do_not_clobber_constructor_state(self): + guardrail = self._guardrail() + guardrail.additional_provider_specific_params = {"team": "security"} + guardrail.api_base = "https://guardrail.example.com" + + guardrail.update_in_memory_litellm_params( + { + "mode": "post_call", + "api_base": None, + "additional_provider_specific_params": None, + "extra_headers": None, + } + ) + + assert guardrail.additional_provider_specific_params == {"team": "security"} + assert guardrail.api_base == "https://guardrail.example.com" + assert guardrail.event_hook is GuardrailEventHooks.post_call + def test_strict_mode_rejects_unsupported_mode_without_mutating(self, monkeypatch): monkeypatch.delenv("LITELLM_STRICT_GUARDRAIL_MODES", raising=False) guardrail = self._guardrail() From cffa202bbf8352ec862e4d58c8ceed8579e4f6fb Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:48:47 -0700 Subject: [PATCH 3/4] fix(guardrails): keep the resynced event_hook in the plain-string shape the constructor stores update_in_memory_litellm_params validated mode into GuardrailEventHooks members while __init__ stores the plain strings LitellmParams.mode carries, so readers that stringify event_hook (akto, straiker) saw different values on the serving worker than on re-initialized workers. Presidio forced post_call assignments go through the same shape, and Straiker recomputes configured_modes on every update --- litellm/integrations/custom_guardrail.py | 23 +++++++++-- .../guardrails/guardrail_hooks/presidio.py | 11 ++++-- .../guardrail_hooks/straiker/straiker.py | 6 +++ .../integrations/test_custom_guardrail.py | 38 +++++++++++++++++-- .../guardrail_hooks/test_presidio.py | 3 +- .../guardrail_hooks/test_straiker.py | 15 ++++++++ .../guardrails/test_guardrail_registry.py | 4 +- 7 files changed, 86 insertions(+), 14 deletions(-) diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index e2754cd7723..3a3783e58a5 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -5,7 +5,7 @@ import os import secrets from collections.abc import Mapping, Sequence from datetime import datetime -from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, get_args +from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, cast, get_args from pydantic import TypeAdapter @@ -137,6 +137,23 @@ GUARDRAIL_MODE_ADAPTER: Final[TypeAdapter[GuardrailEventHooks | list[GuardrailEv ) +def event_hook_as_constructed( + validated_mode: GuardrailEventHooks | list[GuardrailEventHooks] | Mode, +) -> GuardrailEventHooks | list[GuardrailEventHooks] | Mode: + """ + Return the shape ``__init__`` stores for the same mode: ``LitellmParams`` + coerces enum members to plain strings, so a resynced ``event_hook`` must + hold plain strings too or workers end up disagreeing on ``str(event_hook)``. + """ + if isinstance(validated_mode, Mode): + return validated_mode + if isinstance(validated_mode, list): + return cast( # cast-ok: __init__ stores the plain strings LitellmParams.mode carries + list[GuardrailEventHooks], [hook.value for hook in validated_mode] + ) + return cast(GuardrailEventHooks, validated_mode.value) # cast-ok: same parity as the list branch + + def get_session_id_from_request_data(request_data: dict[str, Any]) -> str | None: """Extract session_id from request data (litellm_session_id or metadata).""" session_id = request_data.get("litellm_session_id") @@ -1378,7 +1395,7 @@ class CustomGuardrail(CustomLogger): if value is not None: setattr(self, key, value) if new_event_hook is not None: - self.event_hook = new_event_hook + self.event_hook = event_hook_as_constructed(new_event_hook) def get_guardrails_messages_for_call_type( self, call_type: CallTypes, data: dict | None = None @@ -1407,8 +1424,6 @@ class CustomGuardrail(CustomLogger): # User/System messages are stored in the "input" key, use litellm transformation to get the messages ######################################################### if call_type == CallTypes.responses.value or call_type == CallTypes.aresponses.value: - from typing import cast - from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index da2776eda7d..0272247392c 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -37,6 +37,7 @@ from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException from litellm.integrations.custom_guardrail import ( GUARDRAIL_MODE_ADAPTER, CustomGuardrail, + event_hook_as_constructed, log_guardrail_information, ) from litellm.proxy._types import UserAPIKeyAuth @@ -1641,12 +1642,16 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): if self.event_hook == GuardrailEventHooks.logging_only: return if self.apply_to_output: - self.event_hook = GuardrailEventHooks.post_call + self.event_hook = event_hook_as_constructed(GuardrailEventHooks.post_call) return if not self.output_parse_pii: return current_hook: Final = self.event_hook if isinstance(current_hook, str) and current_hook != "post_call": - self.event_hook = GUARDRAIL_MODE_ADAPTER.validate_python((current_hook, GuardrailEventHooks.post_call)) + self.event_hook = event_hook_as_constructed( + GUARDRAIL_MODE_ADAPTER.validate_python((current_hook, GuardrailEventHooks.post_call)) + ) elif isinstance(current_hook, list) and "post_call" not in current_hook: - self.event_hook = GUARDRAIL_MODE_ADAPTER.validate_python((*current_hook, GuardrailEventHooks.post_call)) + self.event_hook = event_hook_as_constructed( + GUARDRAIL_MODE_ADAPTER.validate_python((*current_hook, GuardrailEventHooks.post_call)) + ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/straiker/straiker.py b/litellm/proxy/guardrails/guardrail_hooks/straiker/straiker.py index 7cca1ae2d63..00ccba11be6 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/straiker/straiker.py +++ b/litellm/proxy/guardrails/guardrail_hooks/straiker/straiker.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio import json import random +from collections.abc import Mapping from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn from urllib.parse import urlsplit @@ -47,6 +48,7 @@ from litellm.types.utils import GenericGuardrailAPIInputs if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.types.guardrails import LitellmParams from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel GUARDRAIL_NAME: Final = "straiker" @@ -329,6 +331,10 @@ class StraikerGuardrail(CustomGuardrail): self.configured_modes = _configured_modes(self.event_hook) + def update_in_memory_litellm_params(self, litellm_params: LitellmParams | Mapping[str, object]) -> None: + super().update_in_memory_litellm_params(litellm_params) + self.configured_modes = _configured_modes(self.event_hook) + def _webhook_url(self) -> str: return f"{self.api_base}{WEBHOOK_PATH}" diff --git a/tests/test_litellm/integrations/test_custom_guardrail.py b/tests/test_litellm/integrations/test_custom_guardrail.py index d5f94d553a0..4d162f5bc54 100644 --- a/tests/test_litellm/integrations/test_custom_guardrail.py +++ b/tests/test_litellm/integrations/test_custom_guardrail.py @@ -2261,7 +2261,7 @@ class TestUpdateInMemoryLitellmParams: LitellmParams(guardrail="update-test", mode="post_call", default_on=True) ) - assert guardrail.event_hook is GuardrailEventHooks.post_call + assert guardrail.event_hook == "post_call" assert guardrail.should_run_guardrail(data={}, event_type=GuardrailEventHooks.post_call) is True assert guardrail.should_run_guardrail(data={}, event_type=GuardrailEventHooks.pre_call) is False @@ -2277,10 +2277,40 @@ class TestUpdateInMemoryLitellmParams: } ) - assert guardrail.event_hook is GuardrailEventHooks.post_call + assert guardrail.event_hook == "post_call" assert getattr(guardrail, "api_base", None) == "https://guardrail.example.com" assert guardrail.should_run_guardrail(data={}, event_type=GuardrailEventHooks.post_call) is True + @pytest.mark.parametrize( + "mode", + [ + "post_call", + ["pre_call", "post_call"], + {"default": "post_call", "tags": {"team-a": ["pre_call", "post_call"]}}, + ], + ids=["str", "list", "mode"], + ) + def test_resynced_event_hook_has_the_shape_a_fresh_worker_constructs(self, mode): + """Other workers rebuild the guardrail from the same DB row through + LitellmParams, which coerces enum members to plain strings; the serving + worker's in-place resync must land on that exact shape, or type-sensitive + readers such as str(self.event_hook) disagree across workers.""" + updated = self._guardrail() + updated.update_in_memory_litellm_params({"guardrail": "update-test", "mode": mode, "default_on": True}) + + constructed = CustomGuardrail( + guardrail_name="update-test", + event_hook=LitellmParams(guardrail="update-test", mode=mode).mode, + default_on=True, + supported_event_hooks=[GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call], + ) + + assert updated.event_hook == constructed.event_hook + assert type(updated.event_hook) is type(constructed.event_hook) + assert str(updated.event_hook) == str(constructed.event_hook) + if isinstance(updated.event_hook, list): + assert [type(hook) for hook in updated.event_hook] == [type(hook) for hook in constructed.event_hook] + def test_none_values_do_not_clobber_constructor_state(self): guardrail = self._guardrail() guardrail.additional_provider_specific_params = {"team": "security"} @@ -2297,7 +2327,7 @@ class TestUpdateInMemoryLitellmParams: assert guardrail.additional_provider_specific_params == {"team": "security"} assert guardrail.api_base == "https://guardrail.example.com" - assert guardrail.event_hook is GuardrailEventHooks.post_call + assert guardrail.event_hook == "post_call" def test_strict_mode_rejects_unsupported_mode_without_mutating(self, monkeypatch): monkeypatch.delenv("LITELLM_STRICT_GUARDRAIL_MODES", raising=False) @@ -2317,7 +2347,7 @@ class TestUpdateInMemoryLitellmParams: guardrail.update_in_memory_litellm_params({"mode": "during_call", "api_base": "https://guardrail.example.com"}) - assert guardrail.event_hook is GuardrailEventHooks.during_call + assert guardrail.event_hook == "during_call" assert getattr(guardrail, "api_base", None) == "https://guardrail.example.com" diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py index 519df980d22..0be37155e73 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py @@ -3179,7 +3179,8 @@ def test_update_in_memory_output_callback_keeps_forced_post_call(): guardrail.update_in_memory_litellm_params({"guardrail": "presidio", "mode": "pre_call", "default_on": True}) - assert guardrail.event_hook is GuardrailEventHooks.post_call + assert guardrail.event_hook == "post_call" + assert type(guardrail.event_hook) is str assert guardrail.should_run_guardrail(data={}, event_type=GuardrailEventHooks.post_call) is True assert guardrail.should_run_guardrail(data={}, event_type=GuardrailEventHooks.pre_call) is False diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_straiker.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_straiker.py index 81604e22c87..679c69ecd91 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_straiker.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_straiker.py @@ -431,6 +431,21 @@ async def test_context_mode_omitted_when_event_hook_absent(): assert "mode" not in _posted_payload(g)["context"] +@pytest.mark.asyncio +async def test_context_mode_follows_an_in_memory_mode_update(): + g = _make_guardrail(event_hook="pre_call") + g.update_in_memory_litellm_params({"guardrail": "straiker", "mode": "post_call", "default_on": True}) + g.async_handler.post.return_value = _mock_response("NONE") + await g.apply_guardrail( + inputs={"texts": ["x"]}, + request_data={"model": "m"}, + input_type="response", + logging_obj=_logging_obj(), + ) + assert g.configured_modes == ["post_call"] + assert _posted_payload(g)["context"]["mode"] == ["post_call"] + + @pytest.mark.asyncio async def test_identity_key_and_team_coalesce_alias_over_id(): g = _make_guardrail() diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py index 015d530257b..bd1e93d8866 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py @@ -176,7 +176,7 @@ def test_update_in_memory_guardrail(): ) is True ) - assert handler.guardrail_id_to_custom_guardrail["123"].event_hook is GuardrailEventHooks.pre_call + assert handler.guardrail_id_to_custom_guardrail["123"].event_hook == "pre_call" def test_update_in_memory_guardrail_raw_db_dict_resyncs_event_hook(): @@ -199,7 +199,7 @@ def test_update_in_memory_guardrail_raw_db_dict_resyncs_event_hook(): handler.update_in_memory_guardrail("123", updated_row) callback = handler.guardrail_id_to_custom_guardrail["123"] - assert callback.event_hook is GuardrailEventHooks.post_call + assert callback.event_hook == "post_call" assert callback.should_run_guardrail(data={}, event_type=GuardrailEventHooks.post_call) is True assert callback.should_run_guardrail(data={}, event_type=GuardrailEventHooks.pre_call) is False assert handler.IN_MEMORY_GUARDRAILS["123"] == updated_row From 52f34ff55338e56503f2582777e0bf3aaad64bc7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 2 Sep 2026 14:58:01 -0700 Subject: [PATCH 4/4] fix(guardrails): rebuild the serving worker guardrail on PUT instead of patching it in place update_in_memory_guardrail now goes through reinitialize_guardrail, the same delete-and-construct path the DB poller and PATCH already use, whenever the row name or litellm_params changed. Patching raw DB values over constructor derived state clobbered normalized URLs, derived api_base values, and resolved secrets, which 500d the serving worker in the earlier revision. An unchanged config only refreshes the cached row, and a row the constructor rejects keeps the previous instance enforcing and raises --- basedpyright-code-budget.json | 18 +- litellm/integrations/custom_guardrail.py | 100 +++-------- .../guardrail_hooks/azure/prompt_shield.py | 40 +++-- .../guardrail_hooks/bedrock_guardrails.py | 5 +- .../guardrail_hooks/lakera_ai_v2.py | 25 +-- .../model_armor/model_armor.py | 2 +- .../guardrails/guardrail_hooks/presidio.py | 50 +++--- .../guardrail_hooks/qualifire/qualifire.py | 9 +- .../guardrail_hooks/straiker/straiker.py | 6 - .../guardrail_hooks/tool_permission.py | 26 +-- .../zscaler_ai_guard/zscaler_ai_guard.py | 7 +- .../proxy/guardrails/guardrail_registry.py | 22 +-- ruff-strict-budget.json | 2 +- .../integrations/test_custom_guardrail.py | 112 ------------- .../guardrail_hooks/test_presidio.py | 38 +---- .../guardrail_hooks/test_straiker.py | 15 -- .../guardrails/test_guardrail_registry.py | 157 ++++++++++-------- type-discipline-budget.json | 8 +- 18 files changed, 217 insertions(+), 425 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index dde5e4411f2..da788bf1ce3 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -1,6 +1,6 @@ { "reportAny": { - "limit": 14075 + "limit": 14076 }, "reportArgumentType": { "limit": 2216 @@ -9,7 +9,7 @@ "limit": 319 }, "reportAttributeAccessIssue": { - "limit": 479 + "limit": 480 }, "reportCallIssue": { "limit": 112 @@ -57,7 +57,7 @@ "limit": 5601 }, "reportMissingTypeArgument": { - "limit": 15303 + "limit": 15306 }, "reportMissingTypeStubs": { "limit": 40 @@ -99,22 +99,22 @@ "limit": 0 }, "reportUnknownArgumentType": { - "limit": 44360 + "limit": 44364 }, "reportUnknownLambdaType": { "limit": 109 }, "reportUnknownMemberType": { - "limit": 38346 + "limit": 38350 }, "reportUnknownParameterType": { - "limit": 19623 + "limit": 19626 }, "reportUnknownVariableType": { - "limit": 29881 + "limit": 29890 }, "reportUnnecessaryCast": { - "limit": 110 + "limit": 111 }, "reportUnnecessaryComparison": { "limit": 692 @@ -123,7 +123,7 @@ "limit": 5 }, "reportUnnecessaryIsInstance": { - "limit": 825 + "limit": 826 }, "reportUntypedBaseClass": { "limit": 0 diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 3a3783e58a5..372c9bf6b91 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -3,11 +3,9 @@ import copy import hashlib import os import secrets -from collections.abc import Mapping, Sequence +from collections.abc import Mapping from datetime import datetime -from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, cast, get_args - -from pydantic import TypeAdapter +from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, get_args from litellm._logging import verbose_logger from litellm.caching import DualCache @@ -125,35 +123,6 @@ def _strict_guardrail_modes_enabled() -> bool: return True if parsed is None else parsed -def updated_litellm_param(litellm_params: "LitellmParams | Mapping[str, object]", key: str) -> object: - if isinstance(litellm_params, Mapping): - return litellm_params.get(key) - value: Final[object] = getattr(litellm_params, key, None) - return value - - -GUARDRAIL_MODE_ADAPTER: Final[TypeAdapter[GuardrailEventHooks | list[GuardrailEventHooks] | Mode]] = TypeAdapter( - GuardrailEventHooks | list[GuardrailEventHooks] | Mode -) - - -def event_hook_as_constructed( - validated_mode: GuardrailEventHooks | list[GuardrailEventHooks] | Mode, -) -> GuardrailEventHooks | list[GuardrailEventHooks] | Mode: - """ - Return the shape ``__init__`` stores for the same mode: ``LitellmParams`` - coerces enum members to plain strings, so a resynced ``event_hook`` must - hold plain strings too or workers end up disagreeing on ``str(event_hook)``. - """ - if isinstance(validated_mode, Mode): - return validated_mode - if isinstance(validated_mode, list): - return cast( # cast-ok: __init__ stores the plain strings LitellmParams.mode carries - list[GuardrailEventHooks], [hook.value for hook in validated_mode] - ) - return cast(GuardrailEventHooks, validated_mode.value) # cast-ok: same parity as the list branch - - def get_session_id_from_request_data(request_data: dict[str, Any]) -> str | None: """Extract session_id from request data (litellm_session_id or metadata).""" session_id = request_data.get("litellm_session_id") @@ -247,7 +216,18 @@ class CustomGuardrail(CustomLogger): self.only_scan_new_messages: bool = only_scan_new_messages if supported_event_hooks: - self._validate_or_warn_event_hook(event_hook, supported_event_hooks) + ## validate event_hook is in supported_event_hooks + try: + self._validate_event_hook(event_hook, supported_event_hooks) + except ValueError as validation_error: + if _strict_guardrail_modes_enabled(): + raise + verbose_logger.warning( + "%s. LITELLM_STRICT_GUARDRAIL_MODES=false; continuing " + "with unsupported event_hook. Set the env var to true " + "(default) to enforce validation and fail at startup.", + validation_error, + ) super().__init__(**kwargs) def render_violation_message(self, default: str, context: Mapping[str, object] | None = None) -> str: @@ -610,12 +590,12 @@ class CustomGuardrail(CustomLogger): def _validate_event_hook( self, - event_hook: GuardrailEventHooks | Sequence[GuardrailEventHooks] | Mode | None, - supported_event_hooks: Sequence[GuardrailEventHooks], + event_hook: GuardrailEventHooks | list[GuardrailEventHooks] | Mode | None, + supported_event_hooks: list[GuardrailEventHooks], ) -> None: def _validate_event_hook_list_is_in_supported_event_hooks( - event_hook: Sequence[GuardrailEventHooks] | Sequence[str], - supported_event_hooks: Sequence[GuardrailEventHooks], + event_hook: list[GuardrailEventHooks] | list[str], + supported_event_hooks: list[GuardrailEventHooks], ) -> None: for hook in event_hook: if isinstance(hook, str): @@ -644,23 +624,6 @@ class CustomGuardrail(CustomLogger): if event_hook not in supported_event_hooks: raise ValueError(f"Event hook {event_hook} is not in the supported event hooks {supported_event_hooks}") - def _validate_or_warn_event_hook( - self, - event_hook: GuardrailEventHooks | Sequence[GuardrailEventHooks] | Mode | None, - supported_event_hooks: Sequence[GuardrailEventHooks], - ) -> None: - try: - self._validate_event_hook(event_hook, supported_event_hooks) - except ValueError as validation_error: - if _strict_guardrail_modes_enabled(): - raise - verbose_logger.warning( - "%s. LITELLM_STRICT_GUARDRAIL_MODES=false; continuing " - "with unsupported event_hook. Set the env var to true " - "(default) to enforce validation and fail at startup.", - validation_error, - ) - @staticmethod def _get_admin_metadata(data: dict) -> dict: """Return merged admin-configured key and team metadata from the request data. @@ -1373,29 +1336,12 @@ class CustomGuardrail(CustomLogger): # Mask the content return content_string[:start_index] + mask_string + content_string[end_index:] - def update_in_memory_litellm_params(self, litellm_params: "LitellmParams | Mapping[str, object]") -> None: + def update_in_memory_litellm_params(self, litellm_params: LitellmParams) -> None: """ - Update the guardrails litellm params in memory, accepting either a - LitellmParams object or the raw params mapping stored in the DB, and - resync ``event_hook`` when the update carries a new ``mode``. The new - mode is validated against ``supported_event_hooks`` before any state - is mutated, so a rejected update leaves the guardrail untouched. - ``None`` values are skipped because both sources serialize every unset - LitellmParams field as ``None``; applying them would clobber - constructor-derived state (e.g. dict defaults) with ``None``. + Update the guardrails litellm params in memory """ - updated_params: Final[Mapping[str, object]] = ( - litellm_params if isinstance(litellm_params, Mapping) else vars(litellm_params) - ) - raw_mode: Final = updated_params.get("mode") - new_event_hook: Final = None if raw_mode is None else GUARDRAIL_MODE_ADAPTER.validate_python(raw_mode) - if new_event_hook is not None and self.supported_event_hooks: - self._validate_or_warn_event_hook(new_event_hook, self.supported_event_hooks) - for key, value in updated_params.items(): - if value is not None: - setattr(self, key, value) - if new_event_hook is not None: - self.event_hook = event_hook_as_constructed(new_event_hook) + for key, value in vars(litellm_params).items(): + setattr(self, key, value) def get_guardrails_messages_for_call_type( self, call_type: CallTypes, data: dict | None = None @@ -1424,6 +1370,8 @@ class CustomGuardrail(CustomLogger): # User/System messages are stored in the "input" key, use litellm transformation to get the messages ######################################################### if call_type == CallTypes.responses.value or call_type == CallTypes.aresponses.value: + from typing import cast + from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/azure/prompt_shield.py b/litellm/proxy/guardrails/guardrail_hooks/azure/prompt_shield.py index 58bebfbdb6e..6e29d44662e 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/azure/prompt_shield.py +++ b/litellm/proxy/guardrails/guardrail_hooks/azure/prompt_shield.py @@ -14,7 +14,6 @@ from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, - updated_litellm_param, ) from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import ( AZURE_PROMPT_SHIELD_TEXT_RECORD_UNIT, @@ -62,6 +61,15 @@ def _resolved_secret_value(value: object) -> object: return value +def _updated_param(litellm_params: "LitellmParams | dict", key: str) -> object: # mutable-ok: DB dict + """Read one param from a Mapping or a pydantic object, including pydantic + extras (cost_tier / price_per_1000_text_records live there), which the base + class ``vars()`` loop never sees.""" + if isinstance(litellm_params, Mapping): + return litellm_params.get(key) + return getattr(litellm_params, key, None) + + def _resolved_cost_tier(raw: object) -> str | None: """Normalize the configured cost_tier to 'free' / 'paid' / None.""" value: Final = _resolved_secret_value(raw) @@ -262,27 +270,29 @@ class AzureContentSafetyPromptShieldGuardrail(AzureGuardrailBase, CustomGuardrai verbose_proxy_logger.warning("Azure Prompt Shield: No user prompt found") return None - def update_in_memory_litellm_params(self, litellm_params: "LitellmParams | Mapping[str, object]") -> None: + def update_in_memory_litellm_params(self, litellm_params: "LitellmParams | dict") -> None: # mutable-ok: DB dict """Apply updated params in place, re-resolving billing and credentials. - Pricing is read via ``updated_litellm_param`` (the values are pydantic - extras, and the immediate PUT sync hands this method the raw DB dict). - Pricing and any ``os.environ/`` credential references are validated and - resolved BEFORE any state is mutated, so an invalid update leaves the - running guardrail untouched and a raw reference never overwrites a - resolved credential. Both input shapes flow through the base update so - the event_hook resync applies to each. + Pricing is read via ``_updated_param`` (the values are pydantic extras, and + the immediate PUT sync hands this method the raw DB dict). Pricing and any + ``os.environ/`` credential references are validated and resolved BEFORE any + state is mutated, so an invalid update leaves the running guardrail + untouched and a raw reference never overwrites a resolved credential. """ - cost_tier: Final = _resolved_cost_tier(updated_litellm_param(litellm_params, "cost_tier")) - price: Final = _resolved_price(updated_litellm_param(litellm_params, "price_per_1000_text_records"), cost_tier) + cost_tier: Final = _resolved_cost_tier(_updated_param(litellm_params, "cost_tier")) + price: Final = _resolved_price(_updated_param(litellm_params, "price_per_1000_text_records"), cost_tier) resolved_credentials: dict[str, object] = {} # mutable-ok: staged before mutation for cred_key in ("api_key", "api_base"): - cred_value = updated_litellm_param(litellm_params, cred_key) + cred_value = _updated_param(litellm_params, cred_key) if isinstance(cred_value, str) and cred_value.startswith("os.environ/"): resolved_credentials[cred_key] = _resolved_secret_value(cred_value) - super().update_in_memory_litellm_params(litellm_params) - for cred_key, cred_value in resolved_credentials.items(): - setattr(self, cred_key, cred_value) + if isinstance(litellm_params, Mapping): + for key, value in litellm_params.items(): + setattr(self, key, resolved_credentials.get(key, value)) + else: + super().update_in_memory_litellm_params(litellm_params) + for cred_key, cred_value in resolved_credentials.items(): + setattr(self, cred_key, cred_value) self.cost_tier = cost_tier self.price_per_1000_text_records = price diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 17ca36ea6a4..30526d30dc5 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -317,10 +317,9 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): self.streaming_sampling_rate = streaming_params.streaming_sampling_rate self.streaming_end_of_stream_only = streaming_params.streaming_end_of_stream_only - def update_in_memory_litellm_params(self, litellm_params: "LitellmParams | Mapping[str, object]") -> None: + def update_in_memory_litellm_params(self, litellm_params: LitellmParams) -> None: super().update_in_memory_litellm_params(litellm_params) - extras: Final = litellm_params if isinstance(litellm_params, Mapping) else litellm_params.model_extra - self._set_streaming_params(BedrockGuardrailStreamingParams.from_extras(extras)) + self._set_streaming_params(BedrockGuardrailStreamingParams.from_extras(litellm_params.model_extra)) def _streams_incrementally(self) -> bool: return not self.streaming_buffer_until_moderated and not self.mask_response_content diff --git a/litellm/proxy/guardrails/guardrail_hooks/lakera_ai_v2.py b/litellm/proxy/guardrails/guardrail_hooks/lakera_ai_v2.py index 9e1382feb21..2f98a9afbd8 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/lakera_ai_v2.py +++ b/litellm/proxy/guardrails/guardrail_hooks/lakera_ai_v2.py @@ -13,7 +13,6 @@ from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( DEFAULT_ADVISORY_MESSAGE, CustomGuardrail, - updated_litellm_param, ) from litellm.llms.base_llm.guardrail_translation.utils import ( effective_skip_system_message_for_guardrail, @@ -305,7 +304,7 @@ class LakeraAIGuardrail(CustomGuardrail): breakdown=self.breakdown, ) - def update_in_memory_litellm_params(self, litellm_params: "LitellmParams | Mapping[str, object]") -> None: + def update_in_memory_litellm_params(self, litellm_params: LitellmParams) -> None: """ The base implementation blindly ``setattr``s every field on ``litellm_params`` (including ``on_flagged``/``advisory_system_message``/``payload``/``breakdown``) @@ -314,18 +313,24 @@ class LakeraAIGuardrail(CustomGuardrail): on_flagged combinations __init__ rejects. Validate the prospective post-update state *before* mutating, so a rejected update leaves the live instance untouched instead of raising after it's already been corrupted. + + The base setattr also writes ``litellm_params.mode`` onto a new ``self.mode`` + attribute rather than the ``self.event_hook`` dispatch actually reads + (LitellmParams has no field literally named ``event_hook``), so without the + explicit sync below a hot reload that changes mode would pass validation but + keep dispatching on the stale event_hook. """ - raw_on_flagged: Final = updated_litellm_param(litellm_params, "on_flagged") - raw_advisory: Final = updated_litellm_param(litellm_params, "advisory_system_message") - raw_payload: Final = updated_litellm_param(litellm_params, "payload") - raw_breakdown: Final = updated_litellm_param(litellm_params, "breakdown") + new_event_hook: Final = litellm_params.mode or self.event_hook + prospective_payload: Final = litellm_params.payload + prospective_breakdown: Final = litellm_params.breakdown self._validate_advisory_config( - on_flagged=raw_on_flagged if isinstance(raw_on_flagged, str) and raw_on_flagged else self.on_flagged, - advisory_system_message=raw_advisory if isinstance(raw_advisory, str) else None, - payload=raw_payload if isinstance(raw_payload, bool) else self.payload, - breakdown=raw_breakdown if isinstance(raw_breakdown, bool) else self.breakdown, + on_flagged=litellm_params.on_flagged or self.on_flagged, + advisory_system_message=litellm_params.advisory_system_message, + payload=self.payload if prospective_payload is None else prospective_payload, + breakdown=self.breakdown if prospective_breakdown is None else prospective_breakdown, ) super().update_in_memory_litellm_params(litellm_params=litellm_params) + self.event_hook = new_event_hook def _validate_advisory_config( self, diff --git a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py index c96334fb2f9..d187b5b12e9 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py +++ b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py @@ -185,7 +185,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): if self.optional_params.get("fail_on_error", True): raise e from None - def update_in_memory_litellm_params(self, litellm_params: "LitellmParams | Mapping[str, object]") -> None: + def update_in_memory_litellm_params(self, litellm_params: LitellmParams) -> None: super().update_in_memory_litellm_params(litellm_params) self.sanitize_error_detail = self.sanitize_error_detail is not False diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index 0272247392c..da51a905ae3 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -11,7 +11,7 @@ import asyncio import json import threading -from collections.abc import AsyncGenerator, AsyncIterable, Awaitable, Mapping, Sequence +from collections.abc import AsyncGenerator, AsyncIterable, Awaitable, Sequence from contextlib import asynccontextmanager from datetime import datetime from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, TypedDict, cast @@ -35,9 +35,7 @@ if TYPE_CHECKING: from litellm.caching.caching import DualCache from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException from litellm.integrations.custom_guardrail import ( - GUARDRAIL_MODE_ADAPTER, CustomGuardrail, - event_hook_as_constructed, log_guardrail_information, ) from litellm.proxy._types import UserAPIKeyAuth @@ -532,17 +530,17 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): return created @staticmethod - def _coerce_analyze_chunk_size(value: object) -> int: + def _coerce_analyze_chunk_size(value: int | None) -> int: """ Validate a configured chunk size, falling back to the default. - Non-positive or non-integer values would either bypass chunking entirely - or degenerate it into per-character splits (silently disabling - detection), so they are replaced by the default; values below 4 bytes - are floored to 4 and the splitter always emits at least one character - per chunk, so the chunked path can never re-enter itself. + Non-positive values would either bypass chunking entirely or degenerate + it into per-character splits (silently disabling detection), so they are + replaced by the default; values below 4 bytes are floored to 4 and the + splitter always emits at least one character per chunk, so the chunked + path can never re-enter itself. """ - if not isinstance(value, int) or value <= 0: + if not value or value <= 0: return DEFAULT_PRESIDIO_ANALYZE_CHUNK_SIZE_BYTES return max(value, 4) @@ -1630,28 +1628,20 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): inputs["texts"] = new_texts return inputs - def update_in_memory_litellm_params(self, litellm_params: "LitellmParams | Mapping[str, object]") -> None: + def update_in_memory_litellm_params(self, litellm_params: LitellmParams) -> None: """ Update the guardrails litellm params in memory """ super().update_in_memory_litellm_params(litellm_params) - self.presidio_analyze_chunk_size_bytes = self._coerce_analyze_chunk_size(self.presidio_analyze_chunk_size_bytes) - self._resync_output_stage_event_hook() - - def _resync_output_stage_event_hook(self) -> None: - if self.event_hook == GuardrailEventHooks.logging_only: - return - if self.apply_to_output: - self.event_hook = event_hook_as_constructed(GuardrailEventHooks.post_call) - return - if not self.output_parse_pii: - return - current_hook: Final = self.event_hook - if isinstance(current_hook, str) and current_hook != "post_call": - self.event_hook = event_hook_as_constructed( - GUARDRAIL_MODE_ADAPTER.validate_python((current_hook, GuardrailEventHooks.post_call)) - ) - elif isinstance(current_hook, list) and "post_call" not in current_hook: - self.event_hook = event_hook_as_constructed( - GUARDRAIL_MODE_ADAPTER.validate_python((*current_hook, GuardrailEventHooks.post_call)) + if litellm_params.pii_entities_config: + self.pii_entities_config = litellm_params.pii_entities_config + if litellm_params.presidio_score_thresholds: + self.presidio_score_thresholds = litellm_params.presidio_score_thresholds + if litellm_params.presidio_entities_deny_list: + self.presidio_entities_deny_list = litellm_params.presidio_entities_deny_list + if litellm_params.presidio_analyze_chunk_size_bytes is not None: + # Same validation as __init__: a non-positive value from a guardrail + # update must not silently disable detection via degenerate chunking. + self.presidio_analyze_chunk_size_bytes = self._coerce_analyze_chunk_size( + litellm_params.presidio_analyze_chunk_size_bytes ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py b/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py index cc1234da4d9..f834426d619 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py +++ b/litellm/proxy/guardrails/guardrail_hooks/qualifire/qualifire.py @@ -7,7 +7,6 @@ import json import os -from collections.abc import Mapping from typing import Any, Final, Literal from fastapi import HTTPException @@ -16,7 +15,6 @@ from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, - updated_litellm_param, ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.custom_httpx.http_handler import ( @@ -113,7 +111,7 @@ class QualifireGuardrail(CustomGuardrail): "only 'block' and 'monitor' are supported." ) - def update_in_memory_litellm_params(self, litellm_params: "LitellmParams | Mapping[str, object]") -> None: + def update_in_memory_litellm_params(self, litellm_params: LitellmParams) -> None: """ The base implementation blindly ``setattr``s every field on ``litellm_params`` (including ``on_flagged``) onto this live instance with no revalidation, so an @@ -123,10 +121,7 @@ class QualifireGuardrail(CustomGuardrail): the live instance untouched instead of raising after it's already been corrupted. Mirrors LakeraAIGuardrail's own override of this same method. """ - raw_on_flagged: Final = updated_litellm_param(litellm_params, "on_flagged") - prospective_on_flagged: Final = ( - raw_on_flagged if isinstance(raw_on_flagged, str) and raw_on_flagged else self.on_flagged - ) + prospective_on_flagged: Final = litellm_params.on_flagged or self.on_flagged self._validate_on_flagged(prospective_on_flagged) super().update_in_memory_litellm_params(litellm_params=litellm_params) diff --git a/litellm/proxy/guardrails/guardrail_hooks/straiker/straiker.py b/litellm/proxy/guardrails/guardrail_hooks/straiker/straiker.py index 00ccba11be6..7cca1ae2d63 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/straiker/straiker.py +++ b/litellm/proxy/guardrails/guardrail_hooks/straiker/straiker.py @@ -3,7 +3,6 @@ from __future__ import annotations import asyncio import json import random -from collections.abc import Mapping from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn from urllib.parse import urlsplit @@ -48,7 +47,6 @@ from litellm.types.utils import GenericGuardrailAPIInputs if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.types.guardrails import LitellmParams from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel GUARDRAIL_NAME: Final = "straiker" @@ -331,10 +329,6 @@ class StraikerGuardrail(CustomGuardrail): self.configured_modes = _configured_modes(self.event_hook) - def update_in_memory_litellm_params(self, litellm_params: LitellmParams | Mapping[str, object]) -> None: - super().update_in_memory_litellm_params(litellm_params) - self.configured_modes = _configured_modes(self.event_hook) - def _webhook_url(self) -> str: return f"{self.api_base}{WEBHOOK_PATH}" diff --git a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py index d6bea312031..a8b33109900 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py +++ b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py @@ -159,7 +159,7 @@ class ToolPermissionGuardrail(CustomGuardrail): self._compiled_rule_targets = compiled_targets self._compiled_rule_patterns = compiled_patterns - def update_in_memory_litellm_params(self, litellm_params: "LitellmParams | Mapping[str, object]") -> None: + def update_in_memory_litellm_params(self, litellm_params: LitellmParams | dict) -> None: """Apply updated params in place, rebuilding the compiled rule state. The base implementation only ``setattr``s raw fields, which would leave @@ -169,19 +169,25 @@ class ToolPermissionGuardrail(CustomGuardrail): immediate in-memory sync take effect, mirroring the PresidioGuardrail override of this method. """ + # ``litellm_params`` may arrive as the raw DB dict (the proxy ``cast()``s + # it to ``LitellmParams`` without converting), so handle both shapes. The + # base ``setattr`` loop is model-only, so apply the dict case here. previous_rules: Final = self.rules - params: Final[Mapping[str, object]] = ( - litellm_params if isinstance(litellm_params, Mapping) else vars(litellm_params) - ) - super().update_in_memory_litellm_params(litellm_params) + if isinstance(litellm_params, dict): + params = litellm_params + for key, value in params.items(): + setattr(self, key, value) + else: + super().update_in_memory_litellm_params(litellm_params) + params = vars(litellm_params) # The generic update above sets ``self.rules`` from the incoming value - # (skipping None) but never rebuilds the compiled maps. Rebuild them - # when a rules list is provided; otherwise restore the previous ruleset - # so a non-list value can't silently wipe it. An explicit empty list - # still clears the rules. + # (None on a partial update that omits rules), but never rebuilds the + # compiled maps. Rebuild them when rules are provided; otherwise restore + # the previous ruleset so a partial update doesn't silently wipe it. An + # explicit empty list still clears the rules. rules: Final = params.get("rules") - if isinstance(rules, list): + if rules is not None: try: self._load_rules(rules) except Exception: diff --git a/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py b/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py index 2928ea5d068..1aefa38ecf8 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py +++ b/litellm/proxy/guardrails/guardrail_hooks/zscaler_ai_guard/zscaler_ai_guard.py @@ -4,7 +4,6 @@ # # +-------------------------------------------------------------+ import os -from collections.abc import Mapping from typing import TYPE_CHECKING, Final, Literal, Optional from fastapi import HTTPException @@ -13,7 +12,6 @@ from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, - updated_litellm_param, ) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, @@ -104,10 +102,9 @@ class ZscalerAIGuard(CustomGuardrail): return timeout - def update_in_memory_litellm_params(self, litellm_params: "LitellmParams | Mapping[str, object]") -> None: + def update_in_memory_litellm_params(self, litellm_params: "LitellmParams") -> None: super().update_in_memory_litellm_params(litellm_params) - raw_timeout: Final = updated_litellm_param(litellm_params, "timeout") - self.timeout = self._resolve_timeout(raw_timeout if isinstance(raw_timeout, (int, float)) else None) + self.timeout = self._resolve_timeout(litellm_params.timeout) @staticmethod def _resolve_metadata_value(request_data: dict | None, key: str) -> str | None: diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index 3abd952da8d..90f0a7025ab 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -6,7 +6,7 @@ import os from collections.abc import Callable, Iterator, Mapping from datetime import datetime, timezone from itertools import chain, count -from typing import TYPE_CHECKING, Final, Literal, Optional, Protocol +from typing import TYPE_CHECKING, Final, Literal, Optional, Protocol, cast from pydantic import ValidationError @@ -622,19 +622,15 @@ class InMemoryGuardrailHandler: source: Literal["db", "config"] = "db", ) -> None: """ - Update a guardrail in memory - - - updates the guardrail params in litellm.callback_manager - - stores the guardrail in memory only after the callback update - succeeds, so a failed update stays visible as a diff to the - per-worker DB poller and gets retried instead of going stale + Update a guardrail in memory: a changed name or litellm_params rebuilds the + live callback from the new row (fail-closed: an invalid row keeps the + previous instance and raises), anything else only refreshes the stored row """ - custom_guardrail_callback: Final = self.guardrail_id_to_custom_guardrail.get(guardrail_id) - updated_litellm_params: Final = guardrail.get("litellm_params") - if custom_guardrail_callback and updated_litellm_params: - custom_guardrail_callback.update_in_memory_litellm_params(litellm_params=updated_litellm_params) - - self.IN_MEMORY_GUARDRAILS[guardrail_id] = guardrail + updated_guardrail: Final = cast(Guardrail, {**guardrail, "guardrail_id": guardrail_id}) + if self._has_guardrail_params_changed(guardrail_id, updated_guardrail): + self.reinitialize_guardrail(guardrail=updated_guardrail, source=source) + return + self.IN_MEMORY_GUARDRAILS[guardrail_id] = updated_guardrail self._sources[guardrail_id] = source def delete_in_memory_guardrail(self, guardrail_id: str) -> None: diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index f43e8c6e93e..9b1cc977a64 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -231,7 +231,7 @@ "limit": 5 }, "TID251": { - "limit": 1072 + "limit": 1073 }, "TRY002": { "limit": 524 diff --git a/tests/test_litellm/integrations/test_custom_guardrail.py b/tests/test_litellm/integrations/test_custom_guardrail.py index 4d162f5bc54..7d70b9a8862 100644 --- a/tests/test_litellm/integrations/test_custom_guardrail.py +++ b/tests/test_litellm/integrations/test_custom_guardrail.py @@ -9,7 +9,6 @@ from litellm.integrations.custom_guardrail import ( log_guardrail_information, ) from litellm.proxy._types import CallTypes, UserAPIKeyAuth -from litellm.types.guardrails import GuardrailEventHooks, LitellmParams from litellm.types.utils import GenericGuardrailAPIInputs, GuardrailTracingDetail @@ -2240,117 +2239,6 @@ class TestRecordsOwnGuardrailInformation: assert _guardrail_entries(request_data) == [] -class TestUpdateInMemoryLitellmParams: - """A PUT /guardrails update reaches the live callback through - update_in_memory_litellm_params: it must accept both a LitellmParams object - and the raw DB dict, and resync self.event_hook (which dispatch reads) from - the incoming mode instead of only writing a dead self.mode attribute (LIT-6591).""" - - def _guardrail(self) -> CustomGuardrail: - return CustomGuardrail( - guardrail_name="update-test", - event_hook=GuardrailEventHooks.pre_call, - default_on=True, - supported_event_hooks=[GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call], - ) - - def test_mode_change_resyncs_event_hook_dispatch(self): - guardrail = self._guardrail() - - guardrail.update_in_memory_litellm_params( - LitellmParams(guardrail="update-test", mode="post_call", default_on=True) - ) - - assert guardrail.event_hook == "post_call" - assert guardrail.should_run_guardrail(data={}, event_type=GuardrailEventHooks.post_call) is True - assert guardrail.should_run_guardrail(data={}, event_type=GuardrailEventHooks.pre_call) is False - - def test_raw_db_dict_copies_params_and_resyncs_event_hook(self): - guardrail = self._guardrail() - - guardrail.update_in_memory_litellm_params( - { - "guardrail": "update-test", - "mode": "post_call", - "api_base": "https://guardrail.example.com", - "default_on": True, - } - ) - - assert guardrail.event_hook == "post_call" - assert getattr(guardrail, "api_base", None) == "https://guardrail.example.com" - assert guardrail.should_run_guardrail(data={}, event_type=GuardrailEventHooks.post_call) is True - - @pytest.mark.parametrize( - "mode", - [ - "post_call", - ["pre_call", "post_call"], - {"default": "post_call", "tags": {"team-a": ["pre_call", "post_call"]}}, - ], - ids=["str", "list", "mode"], - ) - def test_resynced_event_hook_has_the_shape_a_fresh_worker_constructs(self, mode): - """Other workers rebuild the guardrail from the same DB row through - LitellmParams, which coerces enum members to plain strings; the serving - worker's in-place resync must land on that exact shape, or type-sensitive - readers such as str(self.event_hook) disagree across workers.""" - updated = self._guardrail() - updated.update_in_memory_litellm_params({"guardrail": "update-test", "mode": mode, "default_on": True}) - - constructed = CustomGuardrail( - guardrail_name="update-test", - event_hook=LitellmParams(guardrail="update-test", mode=mode).mode, - default_on=True, - supported_event_hooks=[GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call], - ) - - assert updated.event_hook == constructed.event_hook - assert type(updated.event_hook) is type(constructed.event_hook) - assert str(updated.event_hook) == str(constructed.event_hook) - if isinstance(updated.event_hook, list): - assert [type(hook) for hook in updated.event_hook] == [type(hook) for hook in constructed.event_hook] - - def test_none_values_do_not_clobber_constructor_state(self): - guardrail = self._guardrail() - guardrail.additional_provider_specific_params = {"team": "security"} - guardrail.api_base = "https://guardrail.example.com" - - guardrail.update_in_memory_litellm_params( - { - "mode": "post_call", - "api_base": None, - "additional_provider_specific_params": None, - "extra_headers": None, - } - ) - - assert guardrail.additional_provider_specific_params == {"team": "security"} - assert guardrail.api_base == "https://guardrail.example.com" - assert guardrail.event_hook == "post_call" - - def test_strict_mode_rejects_unsupported_mode_without_mutating(self, monkeypatch): - monkeypatch.delenv("LITELLM_STRICT_GUARDRAIL_MODES", raising=False) - guardrail = self._guardrail() - - with pytest.raises(ValueError, match="not in the supported event hooks"): - guardrail.update_in_memory_litellm_params( - {"mode": "during_call", "api_base": "https://guardrail.example.com"} - ) - - assert guardrail.event_hook is GuardrailEventHooks.pre_call - assert getattr(guardrail, "api_base", None) is None - - def test_non_strict_mode_warns_and_applies_unsupported_mode(self, monkeypatch): - monkeypatch.setenv("LITELLM_STRICT_GUARDRAIL_MODES", "false") - guardrail = self._guardrail() - - guardrail.update_in_memory_litellm_params({"mode": "during_call", "api_base": "https://guardrail.example.com"}) - - assert guardrail.event_hook == "during_call" - assert getattr(guardrail, "api_base", None) == "https://guardrail.example.com" - - class _ApplyOnlyObserver(CustomGuardrail): """Overrides only apply_guardrail, like panw_prisma_airs; inherits async_logging_hook.""" diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py index 0be37155e73..4ee6741ee02 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py @@ -17,7 +17,7 @@ from litellm.proxy.guardrails.guardrail_hooks.presidio import ( _OPTIONAL_PresidioPIIMasking, ) from litellm.exceptions import GuardrailRaisedException -from litellm.types.guardrails import GuardrailEventHooks, LitellmParams, PiiAction, PiiEntityType +from litellm.types.guardrails import LitellmParams, PiiAction, PiiEntityType from litellm.types.utils import Choices, Message, ModelResponse from litellm.exceptions import BlockedPiiEntityError @@ -3167,42 +3167,6 @@ def test_update_in_memory_coerces_invalid_chunk_size(): assert guardrail.presidio_analyze_chunk_size_bytes == DEFAULT_PRESIDIO_ANALYZE_CHUNK_SIZE_BYTES -def test_update_in_memory_output_callback_keeps_forced_post_call(): - """The registry-tracked callback for filter_scope='output' is initialized with a - forced post_call hook regardless of the configured mode; a mode-changing update - must not move it off the response stage (LIT-6591).""" - guardrail = _OPTIONAL_PresidioPIIMasking( - mock_testing=True, - apply_to_output=True, - event_hook=GuardrailEventHooks.post_call.value, - ) - - guardrail.update_in_memory_litellm_params({"guardrail": "presidio", "mode": "pre_call", "default_on": True}) - - assert guardrail.event_hook == "post_call" - assert type(guardrail.event_hook) is str - assert guardrail.should_run_guardrail(data={}, event_type=GuardrailEventHooks.post_call) is True - assert guardrail.should_run_guardrail(data={}, event_type=GuardrailEventHooks.pre_call) is False - - -def test_update_in_memory_output_parse_pii_keeps_post_call_expansion(): - """A guardrail with output_parse_pii must keep running on post_call to unmask the - response after a mode-changing update, mirroring the constructor's expansion.""" - guardrail = _OPTIONAL_PresidioPIIMasking( - mock_testing=True, - output_parse_pii=True, - event_hook="pre_call", - ) - - guardrail.update_in_memory_litellm_params( - LitellmParams(guardrail="presidio", mode="during_call", output_parse_pii=True, default_on=True) - ) - - assert guardrail.should_run_guardrail(data={}, event_type=GuardrailEventHooks.during_call) is True - assert guardrail.should_run_guardrail(data={}, event_type=GuardrailEventHooks.post_call) is True - assert guardrail.should_run_guardrail(data={}, event_type=GuardrailEventHooks.pre_call) is False - - def test_split_text_handles_chunk_size_below_char_width(): chunks = _OPTIONAL_PresidioPIIMasking._split_text_for_analysis( text="\U0001f642\U0001f642", chunk_size_bytes=3, overlap_chars=8 diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_straiker.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_straiker.py index 679c69ecd91..81604e22c87 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_straiker.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_straiker.py @@ -431,21 +431,6 @@ async def test_context_mode_omitted_when_event_hook_absent(): assert "mode" not in _posted_payload(g)["context"] -@pytest.mark.asyncio -async def test_context_mode_follows_an_in_memory_mode_update(): - g = _make_guardrail(event_hook="pre_call") - g.update_in_memory_litellm_params({"guardrail": "straiker", "mode": "post_call", "default_on": True}) - g.async_handler.post.return_value = _mock_response("NONE") - await g.apply_guardrail( - inputs={"texts": ["x"]}, - request_data={"model": "m"}, - input_type="response", - logging_obj=_logging_obj(), - ) - assert g.configured_modes == ["post_call"] - assert _posted_payload(g)["context"]["mode"] == ["post_call"] - - @pytest.mark.asyncio async def test_identity_key_and_team_coalesce_alias_over_id(): g = _make_guardrail() diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py index bd1e93d8866..f5221578faa 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py @@ -154,88 +154,103 @@ def test_duplicate_config_guardrail_names_get_distinct_stable_ids(): registry_module.guardrail_initializer_registry.pop("dup_name_test", None) -def test_update_in_memory_guardrail(): - handler = InMemoryGuardrailHandler() - handler.guardrail_id_to_custom_guardrail["123"] = CustomGuardrail( - guardrail_name="test-guardrail", - default_on=False, - event_hook=GuardrailEventHooks.pre_call, - ) +def _register_mode_following_initializer(guardrail_type: str): + """Registers like the shipped initializers do: construct, then add the instance to litellm's callbacks.""" + import litellm + from litellm.proxy.guardrails import guardrail_registry as registry_module - handler.update_in_memory_guardrail( - "123", - Guardrail( - guardrail_name="test-guardrail", - litellm_params=LitellmParams(guardrail="test-guardrail", mode="pre_call", default_on=True), - ), - ) - - assert ( - handler.guardrail_id_to_custom_guardrail["123"].should_run_guardrail( - data={}, event_type=GuardrailEventHooks.pre_call + def _initializer(litellm_params, guardrail): + callback = CustomGuardrail( + guardrail_name=guardrail["guardrail_name"], + supported_event_hooks=[GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call], + event_hook=GuardrailEventHooks(litellm_params.mode), + default_on=True, ) - is True - ) - assert handler.guardrail_id_to_custom_guardrail["123"].event_hook == "pre_call" + litellm.logging_callback_manager.add_litellm_callback(callback) + return callback + + registry_module.guardrail_initializer_registry[guardrail_type] = _initializer + return registry_module -def test_update_in_memory_guardrail_raw_db_dict_resyncs_event_hook(): - """PUT /guardrails hands this method the raw DB row, whose litellm_params is a - plain dict; the update must still apply and move dispatch to the new mode - instead of raising inside vars() and leaving the worker stale (LIT-6591).""" - handler = InMemoryGuardrailHandler() - handler.guardrail_id_to_custom_guardrail["123"] = CustomGuardrail( - guardrail_name="test-guardrail", - default_on=True, - event_hook=GuardrailEventHooks.pre_call, - supported_event_hooks=[GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call], +def _mode_following_db_row(guardrail_id: str, mode: str, description: str = "") -> Guardrail: + """The raw row GuardrailRegistry.update_guardrail_in_db hands back: litellm_params is a plain dict.""" + return Guardrail( + guardrail_id=guardrail_id, + guardrail_name="mode-following", + litellm_params={"guardrail": "mode_following_test", "mode": mode, "default_on": True}, + guardrail_info={"description": description}, ) - updated_row = { - "guardrail_id": "123", - "guardrail_name": "test-guardrail", - "litellm_params": {"guardrail": "test-guardrail", "mode": "post_call", "default_on": True}, - } - handler.update_in_memory_guardrail("123", updated_row) - callback = handler.guardrail_id_to_custom_guardrail["123"] - assert callback.event_hook == "post_call" - assert callback.should_run_guardrail(data={}, event_type=GuardrailEventHooks.post_call) is True - assert callback.should_run_guardrail(data={}, event_type=GuardrailEventHooks.pre_call) is False - assert handler.IN_MEMORY_GUARDRAILS["123"] == updated_row +def _live_instances_named(name: str) -> int: + return sum(1 for cb_list in _all_callback_lists() for cb in cb_list if getattr(cb, "guardrail_name", None) == name) -def test_update_in_memory_guardrail_failed_callback_update_stays_visible_to_poller(monkeypatch): - """When the callback update raises, IN_MEMORY_GUARDRAILS must keep the old row: - storing the new row first would make the per-worker DB poller see no diff and - never re-initialize, leaving the PUT-serving worker stale until restart.""" - monkeypatch.delenv("LITELLM_STRICT_GUARDRAIL_MODES", raising=False) - handler = InMemoryGuardrailHandler() - stale_row = Guardrail( - guardrail_id="123", - guardrail_name="test-guardrail", - litellm_params=LitellmParams(guardrail="test-guardrail", mode="pre_call", default_on=True), - ) - handler.IN_MEMORY_GUARDRAILS["123"] = stale_row - handler.guardrail_id_to_custom_guardrail["123"] = CustomGuardrail( - guardrail_name="test-guardrail", - default_on=True, - event_hook=GuardrailEventHooks.pre_call, - supported_event_hooks=[GuardrailEventHooks.pre_call], - ) +def test_update_in_memory_guardrail_raw_db_row_mode_change_gates_at_the_new_stage(): + registry_module = _register_mode_following_initializer("mode_following_test") + lists = _all_callback_lists() + snapshots = [list(cb_list) for cb_list in lists] + try: + handler = InMemoryGuardrailHandler() + handler.initialize_guardrail(guardrail=_mode_following_db_row("123", "pre_call"), source="db") + original = handler.guardrail_id_to_custom_guardrail["123"] - with pytest.raises(ValueError, match="not in the supported event hooks"): - handler.update_in_memory_guardrail( - "123", - { - "guardrail_id": "123", - "guardrail_name": "test-guardrail", - "litellm_params": {"guardrail": "test-guardrail", "mode": "post_call", "default_on": True}, - }, - ) + handler.update_in_memory_guardrail("123", _mode_following_db_row("123", "post_call")) - assert handler.IN_MEMORY_GUARDRAILS["123"] == stale_row - assert handler.guardrail_id_to_custom_guardrail["123"].event_hook is GuardrailEventHooks.pre_call + replacement = handler.guardrail_id_to_custom_guardrail["123"] + assert replacement is not original + assert replacement.should_run_guardrail(data={}, event_type=GuardrailEventHooks.post_call) is True + assert replacement.should_run_guardrail(data={}, event_type=GuardrailEventHooks.pre_call) is False + assert all(original not in cb_list for cb_list in lists) + assert _live_instances_named("mode-following") == 1 + assert handler.IN_MEMORY_GUARDRAILS["123"]["litellm_params"].mode == "post_call" + assert handler.get_source("123") == "db" + finally: + registry_module.guardrail_initializer_registry.pop("mode_following_test", None) + for cb_list, snapshot in zip(lists, snapshots): + cb_list[:] = snapshot + + +def test_update_in_memory_guardrail_unchanged_params_keep_the_live_instance(): + registry_module = _register_mode_following_initializer("mode_following_test") + lists = _all_callback_lists() + snapshots = [list(cb_list) for cb_list in lists] + try: + handler = InMemoryGuardrailHandler() + handler.initialize_guardrail(guardrail=_mode_following_db_row("123", "pre_call", "old"), source="db") + original = handler.guardrail_id_to_custom_guardrail["123"] + + handler.update_in_memory_guardrail("123", _mode_following_db_row("123", "pre_call", "new")) + + assert handler.guardrail_id_to_custom_guardrail["123"] is original + assert handler.IN_MEMORY_GUARDRAILS["123"]["guardrail_info"] == {"description": "new"} + assert _live_instances_named("mode-following") == 1 + finally: + registry_module.guardrail_initializer_registry.pop("mode_following_test", None) + for cb_list, snapshot in zip(lists, snapshots): + cb_list[:] = snapshot + + +def test_update_in_memory_guardrail_invalid_row_keeps_the_previous_instance_enforcing(): + registry_module = _register_mode_following_initializer("mode_following_test") + lists = _all_callback_lists() + snapshots = [list(cb_list) for cb_list in lists] + try: + handler = InMemoryGuardrailHandler() + handler.initialize_guardrail(guardrail=_mode_following_db_row("123", "pre_call"), source="db") + + with pytest.raises(ValueError, match="not in the supported event hooks"): + handler.update_in_memory_guardrail("123", _mode_following_db_row("123", "during_call")) + + restored = handler.guardrail_id_to_custom_guardrail["123"] + assert restored.should_run_guardrail(data={}, event_type=GuardrailEventHooks.pre_call) is True + assert handler.IN_MEMORY_GUARDRAILS["123"]["litellm_params"].mode == "pre_call" + assert _live_instances_named("mode-following") == 1 + finally: + registry_module.guardrail_initializer_registry.pop("mode_following_test", None) + for cb_list, snapshot in zip(lists, snapshots): + cb_list[:] = snapshot def _make_guardrail(guardrail_id: str, name: str = "g") -> Guardrail: diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 0571d3240e0..52cb9628252 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,9 +1,9 @@ { "LIT001": { - "limit": 22359 + "limit": 22364 }, "LIT002": { - "limit": 26776 + "limit": 26777 }, "LIT003": { "limit": 269 @@ -15,7 +15,7 @@ "limit": 0 }, "LIT006": { - "limit": 1038 + "limit": 1039 }, "LIT007": { "limit": 0 @@ -27,7 +27,7 @@ "limit": 0 }, "LIT010": { - "limit": 16505 + "limit": 16507 }, "LIT011": { "limit": 5535