From 7ff5010674348be45b96e9c7e0c8da3b657f08af Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Sat, 10 Oct 2026 01:46:39 -0700 Subject: [PATCH] feat(guardrails): split logging_only_scope direction from input-failure policy (#45709) * feat(guardrails): split logging_only_scope direction from input-failure policy logging_only_scope is now direction-only (input, output, unset for both directions). The new logging_only_continue_on_input_failure flag controls whether a flagged or raising logging_only input scan still lets the response be scanned. "both" stays accepted everywhere as a deprecated alias for unset scope plus the flag set to true, normalized at callback init while stored rows keep the written value. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(dashboard): narrow continue-on-input-failure update type and stay under guardrail_info max-lines Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(dashboard): keep custom-code guardrail controls readable with logging_only Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(dashboard): scope continue-on-input-failure to the default direction and restore verdict assertion Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(guardrails): remove the both value from logging_only_scope logging_only_scope only shipped its both value in dev prereleases, so it is dropped instead of kept as a deprecated alias. Leaving the scope unset and setting logging_only_continue_on_input_failure gives the same behavior. A stored both value is now treated like any other invalid scope: it is logged and ignored on load, and rejected with 422 on create or update. * fix(guardrails): tolerate an invalid logging_only_scope in the v1 guardrail list A config.yaml guardrail whose logging_only_scope is no longer valid, such as the removed both value, made GET /guardrails/list return 500 for every guardrail. The v1 list now parses params with the same tolerant parser as the v2 list, info and PATCH paths, so the bad value is logged and dropped. * test(guardrails): drive continue-flag rejection tests through the real guardrail handler --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> --- litellm/integrations/custom_guardrail.py | 6 +- litellm/proxy/_lazy_openapi_snapshot.json | 34 ++- .../proxy/guardrails/guardrail_endpoints.py | 49 ++-- .../proxy/guardrails/guardrail_registry.py | 69 ++++-- litellm/types/guardrails.py | 16 +- .../_logging_only_scope_support.py | 20 +- .../observability/test_guardrail_effects.py | 13 +- .../test_logging_only_scope_config.py | 172 +++++++++++++- .../test_logging_only_scope_runtime.py | 210 ++++++++++++++++-- .../integrations/test_custom_guardrail.py | 62 ++++-- .../guardrails/test_guardrail_endpoints.py | 157 +++++++++++++ .../guardrails/test_guardrail_registry.py | 94 +++++++- .../_components/GuardrailFormField.tsx | 58 +++-- .../_components/GuardrailModeDisplay.tsx | 12 +- .../_components/GuardrailReadOnlyDetails.tsx | 6 +- ...d_guardrail_form.characterization.test.tsx | 22 +- .../_components/add_guardrail_form.tsx | 12 + .../CustomCodeLoggingOnlyScope.tsx | 23 +- .../custom_code/CustomCodeModal.test.tsx | 51 ++++- .../custom_code/CustomCodeModal.tsx | 31 ++- .../guardrail_info.characterization.test.tsx | 68 +++++- .../guardrails/_components/guardrail_info.tsx | 17 +- .../guardrail_info_helpers.test.tsx | 49 ++-- .../_components/guardrail_info_helpers.tsx | 53 ++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 22 +- 25 files changed, 1169 insertions(+), 157 deletions(-) diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 383a09f0931..fab30deda93 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -223,6 +223,7 @@ class CustomGuardrail(CustomLogger): records_own_guardrail_information: ClassVar[bool] = False logging_only_scope: LoggingOnlyScope | None + logging_only_continue_on_input_failure: bool stream_scope_default: GuardrailStreamScope = DEFAULT_GUARDRAIL_STREAM_SCOPE stream_scope_by_hook: tuple[tuple[str, GuardrailStreamScope], ...] = () @@ -305,6 +306,7 @@ class CustomGuardrail(CustomLogger): stream_scope_arg: Final[object] = cast(object, kwargs.pop("stream_scope", None)) # cast-ok: config self.apply_stream_scope(stream_scope_arg) self.logging_only_scope = None + self.logging_only_continue_on_input_failure = False if timeout is not None: self.timeout = timeout @@ -1064,7 +1066,7 @@ class CustomGuardrail(CustomLogger): except Exception as e: if self.logging_only_scope == "output": return None - if self.logging_only_scope == "both": + if self.logging_only_continue_on_input_failure: verbose_logger.warning( "Guardrail %s: logging_only request copy failed, skipping request scan: %s", self.guardrail_name, @@ -1092,7 +1094,7 @@ class CustomGuardrail(CustomLogger): "metadata": scratch_metadata, } if self.logging_only_scope != "output" and scratch_fields is not None: - if self.logging_only_scope == "both": + if self.logging_only_continue_on_input_failure: try: await translation.process_input_messages(data=scratch_request, guardrail_to_apply=self) except Exception as e: # noqa: BLE001 # one direction's scan failure must not drop the other direction's verdict diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 32d4f1d601a..69db4525a17 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -11620,13 +11620,24 @@ "description": "Google Cloud location/region (e.g., us-central1)", "title": "Location" }, + "logging_only_continue_on_input_failure": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "when True, a flagged or raising logging_only request scan is logged and the response is still scanned, so both verdicts land. Only applies to mode logging_only and is ignored when logging_only_scope is 'input' or 'output'.", + "title": "Logging Only Continue On Input Failure" + }, "logging_only_scope": { "anyOf": [ { "enum": [ "input", - "output", - "both" + "output" ], "type": "string" }, @@ -11634,7 +11645,7 @@ "type": "null" } ], - "description": "which direction a logging_only scan observes: 'input' (request), 'output' (response), or 'both' (default). Only applies to mode logging_only; pre_call/post_call on the same guardrail keep blocking.", + "description": "which direction a logging_only scan observes: 'input' (request) or 'output' (response); unset scans both directions. Only applies to mode logging_only; pre_call/post_call on the same guardrail keep blocking.", "title": "Logging Only Scope" }, "mask_request_content": { @@ -14027,13 +14038,24 @@ "description": "Google Cloud location/region (e.g., us-central1)", "title": "Location" }, + "logging_only_continue_on_input_failure": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "when True, a flagged or raising logging_only request scan is logged and the response is still scanned, so both verdicts land. Only applies to mode logging_only and is ignored when logging_only_scope is 'input' or 'output'.", + "title": "Logging Only Continue On Input Failure" + }, "logging_only_scope": { "anyOf": [ { "enum": [ "input", - "output", - "both" + "output" ], "type": "string" }, @@ -14041,7 +14063,7 @@ "type": "null" } ], - "description": "which direction a logging_only scan observes: 'input' (request), 'output' (response), or 'both' (default). Only applies to mode logging_only; pre_call/post_call on the same guardrail keep blocking.", + "description": "which direction a logging_only scan observes: 'input' (request) or 'output' (response); unset scans both directions. Only applies to mode logging_only; pre_call/post_call on the same guardrail keep blocking.", "title": "Logging Only Scope" }, "mask": { diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index 6072d8bf22a..bc913ea4418 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -143,14 +143,17 @@ def _get_guardrails_list_response( unmasked_length=4, number_of_asterisks=4, ) - guardrail_configs.append( - GuardrailInfoResponse( - guardrail_id=guardrail.get("guardrail_id"), - guardrail_name=guardrail.get("guardrail_name"), - litellm_params=with_tolerated_stream_scope(masked_params), - guardrail_info=guardrail.get("guardrail_info"), - ) + guardrail_info_response = GuardrailInfoResponse( + guardrail_id=guardrail.get("guardrail_id"), + guardrail_name=guardrail.get("guardrail_name"), + guardrail_info=guardrail.get("guardrail_info"), ) + guardrail_info_response.litellm_params = parse_tolerant_litellm_params( + with_tolerated_stream_scope(masked_params), + guardrail_info_response.guardrail_name, + params_model=BaseLitellmParams, + ) + guardrail_configs.append(guardrail_info_response) return ListGuardrailsResponse(guardrails=guardrail_configs) @@ -1282,14 +1285,30 @@ async def patch_guardrail( status_code=422, detail=f"Invalid guardrail configuration, update rejected: {validation_error}", ) from validation_error - clear_stored_scope: Final = ( - "logging_only_scope" not in requested_litellm_params - and parsed_litellm_params.logging_only_scope is not None - and GuardrailEventHooks.logging_only.value not in configured_event_hooks(parsed_litellm_params.mode) + mode_lacks_logging_only: Final = GuardrailEventHooks.logging_only.value not in configured_event_hooks( + parsed_litellm_params.mode + ) + merged_overrides: Final = MappingProxyType( + { + **( + {"logging_only_scope": None} + if "logging_only_scope" not in requested_litellm_params + and parsed_litellm_params.logging_only_scope is not None + and mode_lacks_logging_only + else {} + ), + **( + {"logging_only_continue_on_input_failure": None} + if "logging_only_continue_on_input_failure" not in requested_litellm_params + and parsed_litellm_params.logging_only_continue_on_input_failure + and mode_lacks_logging_only + else {} + ), + } ) litellm_params: Final = ( - LitellmParams(**MappingProxyType({**merged_litellm_params, "logging_only_scope": None})) - if clear_stored_scope + LitellmParams(**MappingProxyType({**merged_litellm_params, **merged_overrides})) + if merged_overrides else parsed_litellm_params ) @@ -1320,7 +1339,9 @@ async def patch_guardrail( try: IN_MEMORY_GUARDRAIL_HANDLER.sync_guardrail_from_db( guardrail=guardrail, - reject_invalid_logging_only_scope="logging_only_scope" in requested_litellm_params, + reject_invalid_logging_only_scope=bool( + {"logging_only_scope", "logging_only_continue_on_input_failure"} & set(requested_litellm_params) + ), ) verbose_proxy_logger.info( "Immediate sync: Successfully updated guardrail '%s' (ID: %s)", guardrail_name, guardrail_id diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index 439d17b8644..7521c1e9f45 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -587,7 +587,7 @@ def _logging_only_scope_error( f"Guardrail {guardrail_name}: logging_only_scope is set, but mode does not include logging_only, " "so it would never apply. Add logging_only to mode or remove logging_only_scope." ) - if logging_only_scope in ("input", "output") and not custom_guardrail_callback.supports_logging_only_scope(): + if logging_only_scope is not None and not custom_guardrail_callback.supports_logging_only_scope(): return ( f"Guardrail {guardrail_name}: logging_only_scope={logging_only_scope!r} is not supported by this " "guardrail, whose logging_only hook scans on its own. Remove logging_only_scope." @@ -595,6 +595,18 @@ def _logging_only_scope_error( return None +def _logging_only_continue_error(guardrail_name: str, litellm_params: LitellmParams) -> str | None: + if litellm_params.logging_only_continue_on_input_failure is True and ( + GuardrailEventHooks.logging_only.value not in configured_event_hooks(litellm_params.mode) + ): + return ( + f"Guardrail {guardrail_name}: logging_only_continue_on_input_failure is set, but mode does not " + "include logging_only, so it would never apply. Add logging_only to mode or remove " + "logging_only_continue_on_input_failure." + ) + return None + + def _configure_callback_scoping( custom_guardrail_callback: CustomGuardrail, guardrail_name: str, @@ -602,20 +614,31 @@ def _configure_callback_scoping( *, reject_invalid_logging_only_scope: bool = False, ) -> None: - logging_only_scope: Final = litellm_params.logging_only_scope logging_only_scope_error: Final = _logging_only_scope_error( custom_guardrail_callback, guardrail_name, litellm_params ) + logging_only_continue_error: Final = _logging_only_continue_error(guardrail_name, litellm_params) + if reject_invalid_logging_only_scope: + for error in (logging_only_scope_error, logging_only_continue_error): + if error is not None: + raise ValueError(error) if logging_only_scope_error is not None: - if reject_invalid_logging_only_scope: - raise ValueError(logging_only_scope_error) verbose_proxy_logger.error( "%s Ignoring logging_only_scope; the guardrail keeps its configured mode.", logging_only_scope_error.replace("\r", "").replace("\n", ""), ) - custom_guardrail_callback.logging_only_scope = None - else: - custom_guardrail_callback.logging_only_scope = logging_only_scope + if logging_only_continue_error is not None: + verbose_proxy_logger.error( + "%s Ignoring logging_only_continue_on_input_failure; the guardrail keeps its configured mode.", + logging_only_continue_error.replace("\r", "").replace("\n", ""), + ) + logging_only_scope: Final = None if logging_only_scope_error is not None else litellm_params.logging_only_scope + custom_guardrail_callback.logging_only_scope = logging_only_scope + custom_guardrail_callback.logging_only_continue_on_input_failure = ( + logging_only_continue_error is None + and logging_only_scope is None + and litellm_params.logging_only_continue_on_input_failure is True + ) for scoping_param in ( "skip_system_message_in_guardrail", "skip_tool_message_in_guardrail", @@ -651,15 +674,24 @@ def parse_tolerant_litellm_params( try: return params_model(**litellm_params_data) except ValidationError as validation_error: - if any(tuple(error["loc"]) != ("logging_only_scope",) for error in validation_error.errors()): + tolerated_fields: Final = ("logging_only_scope", "logging_only_continue_on_input_failure") + error_locs: Final = {tuple(error["loc"]) for error in validation_error.errors()} + tolerated_locs: Final = frozenset((field,) for field in tolerated_fields) + if any(loc not in tolerated_locs for loc in error_locs): raise - verbose_proxy_logger.error( - "Guardrail %s: logging_only_scope=%r is not one of 'input', 'output' or 'both'. " - "Ignoring logging_only_scope; the guardrail keeps its configured mode.", - guardrail_name.replace("\r", "").replace("\n", ""), - str(litellm_params_data.get("logging_only_scope")).replace("\r", "").replace("\n", "")[:100], + sanitized: Final = MappingProxyType( + {**litellm_params_data, **{field: None for field in tolerated_fields if (field,) in error_locs}} ) - return params_model(**MappingProxyType({**litellm_params_data, "logging_only_scope": None})) + for field in tolerated_fields: + if (field,) in error_locs: + verbose_proxy_logger.error( + "Guardrail %s: %s=%r is not a valid value. Ignoring %s; the guardrail keeps its configured mode.", + guardrail_name.replace("\r", "").replace("\n", ""), + field, + str(litellm_params_data.get(field)).replace("\r", "").replace("\n", "")[:100], + field, + ) + return params_model(**sanitized) class InMemoryGuardrailHandler: @@ -829,9 +861,12 @@ class InMemoryGuardrailHandler: litellm_params: Final = LitellmParams(**params) if isinstance(params, dict) else params guardrail_name: Final = guardrail.get("guardrail_name", "Unknown") for custom_guardrail_callback in self._tracked_callbacks(guardrail_id): - scope_error = _logging_only_scope_error(custom_guardrail_callback, guardrail_name, litellm_params) - if scope_error is not None: - raise ValueError(scope_error) + for error in ( + _logging_only_scope_error(custom_guardrail_callback, guardrail_name, litellm_params), + _logging_only_continue_error(guardrail_name, litellm_params), + ): + if error is not None: + raise ValueError(error) def initialize_custom_guardrail( self, diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index ecc3d0d747c..1d6842898ef 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -989,7 +989,7 @@ def runtime_stream_scope( return DEFAULT_GUARDRAIL_STREAM_SCOPE, MappingProxyType(coerced) -LoggingOnlyScope = Literal["input", "output", "both"] +LoggingOnlyScope = Literal["input", "output"] class BaseLitellmParams(ContentFilterConfigModel): # works for new and patch update guardrails @@ -1250,8 +1250,18 @@ class BaseLitellmParams(ContentFilterConfigModel): # works for new and patch up logging_only_scope: LoggingOnlyScope | None = Field( default=None, description=( - "which direction a logging_only scan observes: 'input' (request), 'output' (response), or 'both' " - "(default). Only applies to mode logging_only; pre_call/post_call on the same guardrail keep blocking." + "which direction a logging_only scan observes: 'input' (request) or 'output' (response); " + "unset scans both directions. Only applies to mode logging_only; pre_call/post_call on the " + "same guardrail keep blocking." + ), + ) + + logging_only_continue_on_input_failure: bool | None = Field( + default=None, + description=( + "when True, a flagged or raising logging_only request scan is logged and the response is " + "still scanned, so both verdicts land. Only applies to mode logging_only and is ignored " + "when logging_only_scope is 'input' or 'output'." ), ) diff --git a/tests/integration/observability/_logging_only_scope_support.py b/tests/integration/observability/_logging_only_scope_support.py index 4c7d76d6564..24c5083d95a 100644 --- a/tests/integration/observability/_logging_only_scope_support.py +++ b/tests/integration/observability/_logging_only_scope_support.py @@ -240,7 +240,7 @@ def wire_server(respond: Callable[[Request], Reply], port: int = 0, *, policy_ed def _directions_for_scope(base_default: tuple[Direction, ...], scope: str | None) -> tuple[Direction, ...]: - if scope is None or scope == "both": + if scope is None: return base_default selected_direction: Final = "request" if scope == "input" else "response" return tuple(direction for direction in base_default if direction == selected_direction) @@ -629,6 +629,7 @@ def _configuration( scope: str | None, *, include_scope: bool = True, + continue_on_input_failure: bool | None = None, default_on: bool = True, mode: str | list[str] = "logging_only", cache: bool = False, @@ -648,6 +649,11 @@ def _configuration( "api_key": "synthetic-guardrail-key", "extra_headers": ["x-litellm-call-id"], **({"logging_only_scope": scope} if include_scope else {}), + **( + {"logging_only_continue_on_input_failure": continue_on_input_failure} + if continue_on_input_failure is not None + else {} + ), } config["guardrails"] = [{"guardrail_name": identity, "litellm_params": params}] scope_name: Final = scope if scope is not None else "unset" @@ -760,6 +766,7 @@ def _insert_database_guardrail( *, mode: str = "pre_call", default_on: bool = True, + continue_on_input_failure: bool | None = None, ) -> None: params: Final = { "guardrail": "generic_guardrail_api", @@ -769,6 +776,11 @@ def _insert_database_guardrail( "api_key": "synthetic-guardrail-key", "extra_headers": ["x-litellm-call-id"], **({"logging_only_scope": scope} if scope is not None else {}), + **( + {"logging_only_continue_on_input_failure": continue_on_input_failure} + if continue_on_input_failure is not None + else {} + ), } write_rows( 'INSERT INTO "LiteLLM_GuardrailsTable" ' @@ -805,6 +817,7 @@ def _post_guardrail_body( api_base: str, scope: JsonValue, include_scope: bool = True, + continue_on_input_failure: bool | None = None, ) -> dict[str, JsonValue]: params: Final = { "guardrail": provider, @@ -821,6 +834,11 @@ def _post_guardrail_body( ), **({"extra_headers": ["x-litellm-call-id"]} if provider == "generic_guardrail_api" else {}), **({"logging_only_scope": scope} if include_scope else {}), + **( + {"logging_only_continue_on_input_failure": continue_on_input_failure} + if continue_on_input_failure is not None + else {} + ), } return { "guardrail": { diff --git a/tests/integration/observability/test_guardrail_effects.py b/tests/integration/observability/test_guardrail_effects.py index 5f1219e18cb..4acd9c73ec6 100644 --- a/tests/integration/observability/test_guardrail_effects.py +++ b/tests/integration/observability/test_guardrail_effects.py @@ -1807,11 +1807,16 @@ def test_responses_pre_call_denial_stream_survives_worker_kill(gateway: Gateway, assert response.status_code == 200, response.text assert response.headers["content-type"].startswith("text/event-stream"), response.text @pytest.mark.parametrize( - ("logging_only_scope", "scanned_directions"), - (("input", ("request",)), ("output", ("response",)), ("both", ("request", "response"))), + ("logging_only_settings", "scanned_directions"), + ( + ({"logging_only_scope": "input"}, ("request",)), + ({"logging_only_scope": "output"}, ("response",)), + ({"logging_only_continue_on_input_failure": True}, ("request", "response")), + ), + ids=("input", "output", "continue_on_input_failure"), ) def test_logging_only_scope_observes_only_the_configured_direction_without_blocking( - gateway: Gateway, tmp_path: Path, logging_only_scope: str, scanned_directions: tuple[str, ...] + gateway: Gateway, tmp_path: Path, logging_only_settings: dict[str, object], scanned_directions: tuple[str, ...] ) -> None: identity: Final = "guardrail" + uuid.uuid4().hex prompt: Final = "synthetic observed prompt " + identity @@ -1848,7 +1853,7 @@ def test_logging_only_scope_observes_only_the_configured_direction_without_block "litellm_params": { "guardrail": "generic_guardrail_api", "mode": "logging_only", - "logging_only_scope": logging_only_scope, + **logging_only_settings, "default_on": True, "api_base": policy.url, "api_key": "synthetic-guardrail-key", diff --git a/tests/integration/observability/test_logging_only_scope_config.py b/tests/integration/observability/test_logging_only_scope_config.py index f2445c3a3a5..c484a59af2f 100644 --- a/tests/integration/observability/test_logging_only_scope_config.py +++ b/tests/integration/observability/test_logging_only_scope_config.py @@ -879,14 +879,14 @@ def test_G6_database_guardrail_polling_normalizes_invalid_scope_without_reinitia id="H4-post-rejects-presidio-input-scope", ), pytest.param( - "H4", - "presidio", + "H2", + "generic_guardrail_api", "logging_only", "both", - 200, - "both", - "", - id="H4-post-accepts-presidio-both-scope", + 422, + None, + "logging_only_scope", + id="H2-post-rejects-removed-both-scope", ), ), ) @@ -1589,3 +1589,163 @@ def test_H13_unauthenticated_management_and_chat_requests_do_not_scan(gateway: G assert _drain_upstream(gateway.upstream_url) == () finally: delete_scenario(upstream_handle) + + +def test_K1_post_rejects_continue_flag_outside_logging_only(gateway: Gateway, tmp_path: Path) -> None: + identity: Final = f"logging-scope-k1-{uuid.uuid4().hex}" + config: Final = _empty_proxy_configuration(tmp_path, identity) + try: + with owned_proxy(gateway, tmp_path, {}, config=config, workers=1) as candidate: + before_list: Final = candidate.get("/v2/guardrails/list")["guardrails"] + response: Final = candidate.request( + "POST", + "/guardrails", + _post_guardrail_body( + identity, + "generic_guardrail_api", + "pre_call", + "http://127.0.0.1:9", + None, + include_scope=False, + continue_on_input_failure=True, + ), + ) + assert response.status_code == 400, response.text + assert "logging_only_continue_on_input_failure" in response.text, response.text + assert _management_guardrail_rows(identity) == () + after_list: Final = candidate.get("/v2/guardrails/list")["guardrails"] + assert after_list == before_list, (before_list, after_list) + assert identity not in {object_value(item)["guardrail_name"] for item in after_list}, after_list + finally: + _delete_database_guardrail(identity) + + +def test_K3_patch_flag_rejection_preserves_raw_stored_row(gateway: Gateway, tmp_path: Path) -> None: + identity: Final = f"logging-scope-k3-{uuid.uuid4().hex}" + config: Final = _empty_proxy_configuration(tmp_path, identity) + try: + with owned_proxy(gateway, tmp_path, {}, config=config, workers=1) as candidate: + created: Final = _create_guardrail( + candidate, + identity, + { + "guardrail": "generic_guardrail_api", + "mode": "pre_call", + "default_on": True, + "api_base": "http://127.0.0.1:9", + "api_key": "synthetic-guardrail-key", + }, + ) + guardrail_id: Final = string_value(created["guardrail_id"]) + before_raw: Final = read_rows( + 'SELECT litellm_params::text AS raw_params FROM "LiteLLM_GuardrailsTable" WHERE guardrail_name=%s', + (identity,), + ) + assert len(before_raw) == 1, before_raw + rejected: Final = candidate.request( + "PATCH", + f"/guardrails/{guardrail_id}", + {"litellm_params": {"logging_only_continue_on_input_failure": True}}, + ) + assert rejected.status_code == 422, rejected.text + assert "logging_only_continue_on_input_failure" in rejected.text, rejected.text + after_raw: Final = read_rows( + 'SELECT litellm_params::text AS raw_params FROM "LiteLLM_GuardrailsTable" WHERE guardrail_name=%s', + (identity,), + ) + assert len(after_raw) == 1, after_raw + # Rollback re-encrypts api_key, so ciphertext bytes differ even though the row is unchanged. + before_params: Final = JSON_OBJECT.validate_json(before_raw[0]["raw_params"]) + after_params: Final = JSON_OBJECT.validate_json(after_raw[0]["raw_params"]) + assert set(before_params) == set(after_params), (before_params, after_params) + assert { + key: before_params[key] for key in before_params if key != "api_key" + } == {key: after_params[key] for key in after_params if key != "api_key"}, (before_params, after_params) + assert str(after_params["api_key"]).startswith("litellm_enc::"), after_params + finally: + _delete_database_guardrail(identity) + + +def test_K4_patch_dropping_logging_only_mode_clears_stored_flag(gateway: Gateway, tmp_path: Path) -> None: + identity: Final = f"logging-scope-k4-{uuid.uuid4().hex}" + config: Final = _empty_proxy_configuration(tmp_path, identity) + try: + with owned_proxy(gateway, tmp_path, {}, config=config, workers=1) as candidate: + created: Final = _create_guardrail( + candidate, + identity, + { + "guardrail": "generic_guardrail_api", + "mode": ["pre_call", "logging_only"], + "default_on": True, + "api_base": "http://127.0.0.1:9", + "api_key": "synthetic-guardrail-key", + "logging_only_continue_on_input_failure": True, + }, + ) + guardrail_id: Final = string_value(created["guardrail_id"]) + before: Final = _management_guardrail_rows(identity) + assert len(before) == 1, before + assert object_value(before[0]["litellm_params"]).get("logging_only_continue_on_input_failure") is True, ( + before + ) + patched: Final = candidate.request( + "PATCH", + f"/guardrails/{guardrail_id}", + {"litellm_params": {"mode": ["pre_call"]}}, + ) + assert patched.status_code == 200, patched.text + persisted: Final = _management_guardrail_rows(identity) + assert len(persisted) == 1, persisted + params: Final = object_value(persisted[0]["litellm_params"]) + assert params.get("logging_only_continue_on_input_failure") is None, persisted + assert params["mode"] == ["pre_call"], persisted + finally: + _delete_database_guardrail(identity) + + +def test_K5_yaml_flag_on_blocking_guardrail_boots_and_keeps_blocking(gateway: Gateway, tmp_path: Path) -> None: + identity: Final = f"logging-scope-k5-{uuid.uuid4().hex}" + prompt: Final = f"synthetic yaml flag marker {identity}" + reply: Final = f"synthetic yaml flag response {identity}" + scenario_id: Final = f"phase12-k5-{uuid.uuid4().hex}" + upstream_handle: Final = register_scenario(scenario_id, _provider_response("chat", scenario_id, reply, False)) + + def policy(request: Request) -> Reply: + assert request.target == "/beta/litellm_basic_guardrail_api", request.target + return Reply(body=b'{"action":"BLOCKED","blocked_reason":"synthetic yaml flag denial"}') + + try: + with gateway.scenario() as scenario: + model: Final = scenario.model( + model="openai/gpt-4o-mini", + api_base=f"{upstream_handle.api_base()}/v1", + api_key="synthetic-provider-key", + ) + with wire_server(policy) as guardrail: + config: Final = _configuration( + tmp_path, + identity, + guardrail.url, + None, + include_scope=False, + continue_on_input_failure=True, + mode="pre_call", + ) + with owned_proxy_process(gateway, tmp_path, {}, config=config, workers=1) as owned: + candidate: Final = owned.gateway + guarded: Final = candidate.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": prompt}]}, + headers={"x-litellm-call-id": f"{scenario_id}-guarded"}, + ) + assert guarded.status_code == 400 and "synthetic yaml flag denial" in guarded.text, guarded.text + assert _drain_upstream(gateway.upstream_url) == () + payloads: Final = tuple(JSON_OBJECT.validate_json(call.body) for call in guardrail.drain()) + assert tuple(_direction(payload) for payload in payloads) == ("request",), payloads + log_text: Final = owned.log.read_text() + assert "Ignoring logging_only_continue_on_input_failure" in log_text, log_text + finally: + _delete_database_guardrail(identity) + delete_scenario(upstream_handle) diff --git a/tests/integration/observability/test_logging_only_scope_runtime.py b/tests/integration/observability/test_logging_only_scope_runtime.py index dd00378401e..b637859da52 100644 --- a/tests/integration/observability/test_logging_only_scope_runtime.py +++ b/tests/integration/observability/test_logging_only_scope_runtime.py @@ -64,7 +64,6 @@ from tests.integration.cost_calculation.cost_tracking_case import JsonResponse ( pytest.param("A1", "chat", False, "openai_sync", "input", True, (), id="A1-chat-input"), pytest.param("A2", "chat", False, "openai_sync", "output", True, (), id="A2-chat-output"), - pytest.param("A3", "chat", False, "openai_sync", "both", True, (), id="A3-chat-both"), pytest.param("A4", "chat", False, "httpx", None, False, (), id="A4-chat-missing-scope"), pytest.param("A5", "chat", False, "httpx", None, True, (), id="A5-chat-null-scope"), pytest.param("A6", "chat", True, "openai_async", "input", True, (), id="A6-chat-stream-async-input"), @@ -97,16 +96,6 @@ from tests.integration.cost_calculation.cost_tracking_case import JsonResponse pytest.param("A15", "responses", True, "openai_sync", "input", True, (), id="A15-responses-stream-input"), pytest.param("B1", "chat", False, "openai_sync", "input", True, ("request",), id="B1-logging-block-input"), pytest.param("B2", "chat", False, "openai_sync", "output", True, ("response",), id="B2-logging-block-output"), - pytest.param( - "B3", - "chat", - False, - "openai_sync", - "both", - True, - ("request",), - id="B3-logging-block-both", - ), ), ) def test_runtime_directional_scope_matches_client_call_and_spend_log( @@ -911,10 +900,9 @@ def test_logging_only_scope_respects_guardrail_selection_level( delete_scenario(upstream_handle) -def test_X1_missing_null_and_both_scope_have_identical_scans(gateway: Gateway, tmp_path: Path) -> None: +def test_X1_missing_and_null_scope_have_identical_scans(gateway: Gateway, tmp_path: Path) -> None: identity: Final = f"logging-scope-x1-{uuid.uuid4().hex}" variants: Final = ( - ("both", "both", True), ("missing", None, False), ("null", None, True), ) @@ -1577,3 +1565,199 @@ def test_S4_logging_only_input_scope_scans_every_multipart_text_part(gateway: Ga ) == tuple((identity, "logging_only", "guardrail_intervened") for _ in expected_directions), entries finally: delete_scenario(upstream_handle) + + +@pytest.mark.parametrize( + ("row_id", "scope", "include_scope", "continue_flag", "expected_directions"), + ( + pytest.param("J1", None, False, None, ("request",), id="J1-unset-flag-stops-after-flagged-input"), + pytest.param( + "J2", + None, + False, + True, + ("request", "response"), + id="J2-flag-continues-past-flagged-input", + ), + pytest.param("J5", "input", True, True, ("request",), id="J5-input-scope-ignores-flag"), + pytest.param("J6", "output", True, True, ("response",), id="J6-output-scope-ignores-flag"), + ), +) +def test_continue_on_input_failure_observes_flagged_input_and_response( + gateway: Gateway, + tmp_path: Path, + row_id: str, + scope: str | None, + include_scope: bool, + continue_flag: bool | None, + expected_directions: tuple[Direction, ...], +) -> None: + identity: Final = f"logging-scope-{row_id.lower()}-{uuid.uuid4().hex}" + prompt: Final = f"synthetic flagged request {identity}" + reply: Final = f"synthetic flagged response {identity}" + scenario_id: Final = f"phase12-{row_id.lower()}-{uuid.uuid4().hex}" + guarded_call_id: Final = f"{scenario_id}-guarded" + upstream_handle: Final = register_scenario( + scenario_id, _provider_response("chat", scenario_id, reply, False) + ) + + def policy(_request: Request) -> Reply: + return Reply(body=b'{"action":"BLOCKED","blocked_reason":"synthetic flagged denial"}') + + try: + with gateway.scenario() as scenario: + model: Final = scenario.model( + model="openai/gpt-4o-mini", + api_base=f"{upstream_handle.api_base()}/v1", + api_key="synthetic-provider-key", + ) + with wire_server(policy) as guardrail: + config: Final = _configuration( + tmp_path, + identity, + guardrail.url, + scope, + include_scope=include_scope, + continue_on_input_failure=continue_flag, + ) + with owned_proxy(gateway, tmp_path, {}, config=config, workers=1) as candidate: + baseline: Final = _call_client( + "openai_sync", "chat", gateway, model, prompt, False, f"{scenario_id}-baseline" + ) + assert baseline.status == 200 and baseline.text == reply, baseline + assert len(_drain_upstream(gateway.upstream_url)) == 1 + result: Final = _call_client( + "openai_sync", "chat", candidate, model, prompt, False, guarded_call_id + ) + assert (result.status, _response_body_without_ids(result.body)) == ( + baseline.status, + _response_body_without_ids(baseline.body), + ), (result, baseline) + assert result.text == reply, result + upstream: Final = _drain_upstream(gateway.upstream_url) + assert len(upstream) == 1 and prompt in json.dumps(upstream[0]["body"]), upstream + eventually( + lambda: guardrail.received.qsize(), + lambda count: count >= len(expected_directions), + seconds=20, + ) + calls: Final = guardrail.drain() + assert len(calls) == len(expected_directions), calls + payloads: Final = tuple(JSON_OBJECT.validate_json(call.body) for call in calls) + observed_directions: Final = tuple(_direction(payload) for payload in payloads) + assert tuple(sorted(observed_directions)) == tuple(sorted(expected_directions)), payloads + assert all(_policy_call_id_matches(payload, guarded_call_id) for payload in payloads), payloads + assert tuple(payload["texts"] for payload in payloads) == tuple( + [prompt] if direction == "request" else [reply] for direction in observed_directions + ), payloads + rows: Final = _spend_rows(model, 2) + guarded_rows: Final = tuple(row for row in rows if _guardrail_entries(row)) + assert len(guarded_rows) == 1, rows + _assert_response_id("chat", str(guarded_rows[0]["request_id"]), result.response_id, scenario_id) + entries: Final = _guardrail_entries(guarded_rows[0]) + assert tuple( + (entry["guardrail_name"], entry["guardrail_mode"], entry["guardrail_status"]) + for entry in entries + ) == tuple( + (identity, "logging_only", "guardrail_intervened") for _ in expected_directions + ), entries + finally: + delete_scenario(upstream_handle) + + +@pytest.mark.parametrize( + ("row_id", "continue_flag", "expected_directions", "expected_statuses"), + ( + pytest.param( + "J7", + None, + ("request",), + ("guardrail_failed_to_respond",), + id="J7-raising-input-aborts-response-scan", + ), + pytest.param( + "J8", + True, + ("request", "response"), + ("guardrail_failed_to_respond", "success"), + id="J8-flag-continues-past-raising-input", + ), + ), +) +def test_continue_on_input_failure_recovers_from_raising_input_scan( + gateway: Gateway, + tmp_path: Path, + row_id: str, + continue_flag: bool | None, + expected_directions: tuple[Direction, ...], + expected_statuses: tuple[str, ...], +) -> None: + identity: Final = f"logging-scope-{row_id.lower()}-{uuid.uuid4().hex}" + prompt: Final = f"synthetic outage request {identity}" + reply: Final = f"synthetic outage response {identity}" + scenario_id: Final = f"phase12-{row_id.lower()}-{uuid.uuid4().hex}" + guarded_call_id: Final = f"{scenario_id}-guarded" + upstream_handle: Final = register_scenario( + scenario_id, _provider_response("chat", scenario_id, reply, False) + ) + + def policy(request: Request) -> Reply: + payload: Final = JSON_OBJECT.validate_json(request.body) + if _direction(payload) == "request": + return Reply(status=500, body=b"policy edge outage") + return Reply(body=b'{"action":"NONE"}') + + try: + with gateway.scenario() as scenario: + model: Final = scenario.model( + model="openai/gpt-4o-mini", + api_base=f"{upstream_handle.api_base()}/v1", + api_key="synthetic-provider-key", + ) + with wire_server(policy) as guardrail: + config: Final = _configuration( + tmp_path, + identity, + guardrail.url, + None, + include_scope=False, + continue_on_input_failure=continue_flag, + ) + with owned_proxy(gateway, tmp_path, {}, config=config, workers=1) as candidate: + baseline: Final = _call_client( + "openai_sync", "chat", gateway, model, prompt, False, f"{scenario_id}-baseline" + ) + assert baseline.status == 200 and baseline.text == reply, baseline + assert len(_drain_upstream(gateway.upstream_url)) == 1 + result: Final = _call_client( + "openai_sync", "chat", candidate, model, prompt, False, guarded_call_id + ) + assert (result.status, _response_body_without_ids(result.body)) == ( + baseline.status, + _response_body_without_ids(baseline.body), + ), (result, baseline) + assert result.text == reply, result + upstream: Final = _drain_upstream(gateway.upstream_url) + assert len(upstream) == 1 and prompt in json.dumps(upstream[0]["body"]), upstream + eventually( + lambda: guardrail.received.qsize(), + lambda count: count >= len(expected_directions), + seconds=20, + ) + calls: Final = guardrail.drain() + assert len(calls) == len(expected_directions), calls + payloads: Final = tuple(JSON_OBJECT.validate_json(call.body) for call in calls) + assert tuple(_direction(payload) for payload in payloads) == expected_directions, payloads + assert all(_policy_call_id_matches(payload, guarded_call_id) for payload in payloads), payloads + rows: Final = _spend_rows(model, 2) + guarded_rows: Final = tuple(row for row in rows if _guardrail_entries(row)) + assert len(guarded_rows) == 1, rows + _assert_response_id("chat", str(guarded_rows[0]["request_id"]), result.response_id, scenario_id) + entries: Final = _guardrail_entries(guarded_rows[0]) + assert tuple(entry["guardrail_status"] for entry in entries) == expected_statuses, entries + assert all( + entry["guardrail_name"] == identity and entry["guardrail_mode"] == "logging_only" + for entry in entries + ), entries + finally: + delete_scenario(upstream_handle) diff --git a/tests/unit/integrations/test_custom_guardrail.py b/tests/unit/integrations/test_custom_guardrail.py index e04f8af5a66..e6a44651a17 100644 --- a/tests/unit/integrations/test_custom_guardrail.py +++ b/tests/unit/integrations/test_custom_guardrail.py @@ -2812,22 +2812,24 @@ class TestLoggingOnlyApplyGuardrail: assert kwargs["standard_logging_object"] == {"guardrail_information": None} @pytest.mark.parametrize( - "scope,expected_calls", + "scope,continue_on_input_failure,expected_calls", ( - (None, [("request", ["hello there"]), ("response", ["general kenobi"])]), - ("both", [("request", ["hello there"]), ("response", ["general kenobi"])]), - ("input", [("request", ["hello there"])]), - ("output", [("response", ["general kenobi"])]), + (None, False, [("request", ["hello there"]), ("response", ["general kenobi"])]), + (None, True, [("request", ["hello there"]), ("response", ["general kenobi"])]), + ("input", False, [("request", ["hello there"])]), + ("output", False, [("response", ["general kenobi"])]), ), ) @pytest.mark.asyncio async def test_logging_only_scope_scans_configured_directions( self, scope: LoggingOnlyScope | None, + continue_on_input_failure: bool, expected_calls: list[tuple[str, list[str]]], ) -> None: guardrail: Final = _ApplyOnlyObserver() guardrail.logging_only_scope = scope + guardrail.logging_only_continue_on_input_failure = continue_on_input_failure kwargs, response = _logged_call([{"role": "user", "content": "hello there"}]) out_kwargs, _ = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value) @@ -2881,21 +2883,37 @@ class TestLoggingOnlyApplyGuardrail: assert [entry["guardrail_name"] for entry in entries] == ["apply-only-observer"] assert [entry["guardrail_status"] for entry in entries] == ["success"] + @pytest.mark.parametrize( + "scope,continue_on_input_failure,expected_calls,expected_statuses", + ( + (None, True, [("response", ["general kenobi"])], ["success"]), + ("input", True, [], []), + ("output", True, [("response", ["general kenobi"])], ["success"]), + (None, False, [], []), + ), + ) @pytest.mark.asyncio - async def test_request_copy_failure_does_not_drop_the_response_scan_for_explicit_both_scope(self): + async def test_request_copy_failure_drops_the_response_scan_only_without_continue( + self, + scope: LoggingOnlyScope | None, + continue_on_input_failure: bool, + expected_calls: list[tuple[str, list[str]]], + expected_statuses: list[str], + ): import threading guardrail: Final = _ApplyOnlyObserver() - guardrail.logging_only_scope = "both" + guardrail.logging_only_scope = scope + guardrail.logging_only_continue_on_input_failure = continue_on_input_failure call: Final = _logged_call([{"role": "user", "content": "hello there", "lock": threading.Lock()}]) kwargs: Final = call[0] response: Final = call[1] out_kwargs, _ = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value) - assert guardrail.calls == [("response", ["general kenobi"])] - entries: Final = out_kwargs["standard_logging_object"]["guardrail_information"] - assert [entry["guardrail_status"] for entry in entries] == ["success"] + assert guardrail.calls == expected_calls + entries: Final = out_kwargs["standard_logging_object"].get("guardrail_information") or [] + assert [entry["guardrail_status"] for entry in entries] == expected_statuses @pytest.mark.asyncio async def test_block_verdict_is_recorded_without_raising(self): @@ -2927,8 +2945,23 @@ class TestLoggingOnlyApplyGuardrail: entries = out_kwargs["standard_logging_object"]["guardrail_information"] assert [e["guardrail_status"] for e in entries] == ["guardrail_failed_to_respond"] + @pytest.mark.parametrize( + "scope,continue_on_input_failure,expected_calls,expected_statuses", + ( + (None, True, [("request", ["hello there"]), ("response", ["general kenobi"])], ["guardrail_failed_to_respond", "success"]), + ("output", True, [("response", ["general kenobi"])], ["success"]), + ("input", True, [("request", ["hello there"])], ["guardrail_failed_to_respond"]), + (None, False, [("request", ["hello there"])], ["guardrail_failed_to_respond"]), + ), + ) @pytest.mark.asyncio - async def test_input_scan_error_does_not_drop_the_response_scan_for_explicit_both_scope(self): + async def test_input_scan_error_drops_the_response_scan_only_without_continue( + self, + scope: LoggingOnlyScope | None, + continue_on_input_failure: bool, + expected_calls: list[tuple[str, list[str]]], + expected_statuses: list[str], + ): class _FailingBothObserver(_ApplyOnlyObserver): @log_guardrail_information async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): @@ -2938,14 +2971,15 @@ class TestLoggingOnlyApplyGuardrail: return GenericGuardrailAPIInputs(texts=[]) guardrail = _FailingBothObserver() - guardrail.logging_only_scope = "both" + guardrail.logging_only_scope = scope + guardrail.logging_only_continue_on_input_failure = continue_on_input_failure kwargs, response = _logged_call([{"role": "user", "content": "hello there"}]) out_kwargs, _ = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value) - assert guardrail.calls == [("request", ["hello there"]), ("response", ["general kenobi"])] + assert guardrail.calls == expected_calls entries = out_kwargs["standard_logging_object"]["guardrail_information"] - assert [e["guardrail_status"] for e in entries] == ["guardrail_failed_to_respond", "success"] + assert [e["guardrail_status"] for e in entries] == expected_statuses @pytest.mark.asyncio async def test_call_type_without_translation_is_skipped(self): diff --git a/tests/unit/proxy/guardrails/test_guardrail_endpoints.py b/tests/unit/proxy/guardrails/test_guardrail_endpoints.py index b7bc5f3f8ec..b41e2b00fb0 100644 --- a/tests/unit/proxy/guardrails/test_guardrail_endpoints.py +++ b/tests/unit/proxy/guardrails/test_guardrail_endpoints.py @@ -794,6 +794,40 @@ def test_get_guardrails_list_response_tolerates_invalid_config_stream_scope(): assert response.guardrails[1].litellm_params.stream_scope == "streaming" +def test_get_guardrails_list_response_tolerates_invalid_config_logging_only_scope(): + from litellm.proxy.guardrails.guardrail_endpoints import ( + _get_guardrails_list_response, + ) + + response = _get_guardrails_list_response( + [ + { + "guardrail_id": "removed-both-scope", + "guardrail_name": "removed-both-scope", + "litellm_params": { + "guardrail": "generic_guardrail_api", + "mode": "logging_only", + "logging_only_scope": "both", + }, + }, + { + "guardrail_id": "input-scope", + "guardrail_name": "input-scope", + "litellm_params": { + "guardrail": "generic_guardrail_api", + "mode": "logging_only", + "logging_only_scope": "input", + }, + }, + ] + ) + + assert response.guardrails[0].litellm_params is not None + assert response.guardrails[0].litellm_params.logging_only_scope is None + assert response.guardrails[1].litellm_params is not None + assert response.guardrails[1].litellm_params.logging_only_scope == "input" + + def test_get_provider_specific_params(): """Test getting provider-specific parameters""" from litellm.proxy.guardrails.guardrail_endpoints import _get_fields_from_model @@ -1673,6 +1707,129 @@ async def test_patch_guardrail_rejects_explicit_unsupported_scope_and_rolls_back handler.delete_in_memory_guardrail(stored_guardrail["guardrail_id"]) +@pytest.mark.asyncio +async def test_patch_guardrail_rejects_invalid_continue_flag_with_422(mocker, monkeypatch, mock_guardrail_registry): + handler, stored_guardrail = _setup_patch_scope_guardrail( + mocker, + monkeypatch, + mock_guardrail_registry, + _PatchScopeSupportedGuardrail, + "patch_scope_supported_test", + {"mode": "pre_call", "default_on": True}, + ) + request = PatchGuardrailRequest( + litellm_params=BaseLitellmParams(mode="pre_call", logging_only_continue_on_input_failure=True) + ) + + try: + with pytest.raises(HTTPException) as exc_info: + await patch_guardrail( + stored_guardrail["guardrail_id"], + request, + user_api_key_dict=MOCK_ADMIN_USER, + ) + + assert exc_info.value.status_code == 422 + assert "update rejected" in str(exc_info.value.detail) + assert "logging_only_continue_on_input_failure is set, but mode does not include logging_only" in str( + exc_info.value.detail + ) + assert mock_guardrail_registry.update_guardrail_in_db.call_count == 2 + restored_guardrail = mock_guardrail_registry.update_guardrail_in_db.call_args_list[-1].kwargs["guardrail"] + assert restored_guardrail["litellm_params"] == stored_guardrail["litellm_params"] + callback = handler.guardrail_id_to_custom_guardrail[stored_guardrail["guardrail_id"]] + assert callback.logging_only_continue_on_input_failure is False + finally: + handler.delete_in_memory_guardrail(stored_guardrail["guardrail_id"]) + + +@pytest.mark.asyncio +async def test_patch_guardrail_clears_continue_flag_when_logging_only_mode_is_removed( + mocker, monkeypatch, mock_guardrail_registry +): + handler, stored_guardrail = _setup_patch_scope_guardrail( + mocker, + monkeypatch, + mock_guardrail_registry, + _PatchScopeSupportedGuardrail, + "patch_scope_supported_test", + { + "mode": ["pre_call", "logging_only"], + "logging_only_continue_on_input_failure": True, + "default_on": True, + }, + ) + request = PatchGuardrailRequest(litellm_params=BaseLitellmParams(mode=["pre_call"])) + + try: + result = await patch_guardrail( + stored_guardrail["guardrail_id"], + request, + user_api_key_dict=MOCK_ADMIN_USER, + ) + + assert result["guardrail_id"] == stored_guardrail["guardrail_id"] + persisted_guardrail = mock_guardrail_registry.update_guardrail_in_db.call_args.kwargs["guardrail"] + assert persisted_guardrail["litellm_params"].logging_only_continue_on_input_failure is None + callback = handler.guardrail_id_to_custom_guardrail[stored_guardrail["guardrail_id"]] + assert callback.logging_only_continue_on_input_failure is False + finally: + handler.delete_in_memory_guardrail(stored_guardrail["guardrail_id"]) + + +@pytest.mark.asyncio +async def test_add_guardrail_rejects_continue_flag_outside_logging_only_with_400( + mocker, monkeypatch, mock_guardrail_registry +): + import litellm + from litellm.proxy.guardrails import guardrail_registry as registry_module + + guardrail_type: Final = "patch_scope_supported_test" + litellm_params: Final = { + "guardrail": guardrail_type, + "mode": "pre_call", + "logging_only_continue_on_input_failure": True, + } + mock_guardrail_registry.add_guardrail_to_db = AsyncMock( + return_value={ + "guardrail_id": "flag-400-test", + "guardrail_name": "flag-400-test", + "litellm_params": litellm_params, + "guardrail_info": {}, + } + ) + monkeypatch.setitem( + registry_module.guardrail_initializer_registry, + guardrail_type, + _patch_scope_initializer(_PatchScopeSupportedGuardrail), + ) + handler: Final = InMemoryGuardrailHandler() + mocker.patch("litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY", mock_guardrail_registry) + mocker.patch("litellm.proxy.proxy_server.prisma_client", mocker.Mock()) + mocker.patch("litellm.proxy.guardrails.guardrail_registry.IN_MEMORY_GUARDRAIL_HANDLER", handler) + delete_row = mocker.patch("litellm.proxy.guardrails.guardrail_endpoints._delete_guardrail_row", AsyncMock()) + + with pytest.raises(HTTPException) as exc_info: + await create_guardrail( + CreateGuardrailRequest( + guardrail={ + "guardrail_name": "flag-400-test", + "litellm_params": litellm_params, + "guardrail_info": {}, + } + ), + user_api_key_dict=MOCK_ADMIN_USER, + ) + + assert exc_info.value.status_code == 400 + assert "logging_only_continue_on_input_failure is set, but mode does not include logging_only" in str( + exc_info.value.detail + ) + assert "flag-400-test" not in handler.guardrail_id_to_custom_guardrail + assert all(getattr(callback, "guardrail_name", None) != "flag-400-test" for callback in litellm.callbacks) + delete_row.assert_awaited_once() + + @pytest.mark.asyncio async def test_patch_guardrail_rejected_update_restores_invalid_stored_scope_verbatim( mocker, monkeypatch, mock_guardrail_registry diff --git a/tests/unit/proxy/guardrails/test_guardrail_registry.py b/tests/unit/proxy/guardrails/test_guardrail_registry.py index 3bc6d163e5d..7eabc24e090 100644 --- a/tests/unit/proxy/guardrails/test_guardrail_registry.py +++ b/tests/unit/proxy/guardrails/test_guardrail_registry.py @@ -1273,6 +1273,7 @@ class TestLoggingOnlyScopeValidation: callback_type: type[CustomGuardrail] = _LoggingOnlyScopeSupportedGuardrail, reject_invalid_logging_only_scope: bool = False, assert_registered: bool = False, + continue_on_input_failure: bool | None = None, ) -> CustomGuardrail: import litellm from litellm.proxy.guardrails import guardrail_registry as registry_module @@ -1306,6 +1307,7 @@ class TestLoggingOnlyScopeValidation: "guardrail": guardrail_type, "mode": mode, "logging_only_scope": scope, + "logging_only_continue_on_input_failure": continue_on_input_failure, }, }, reject_invalid_logging_only_scope=reject_invalid_logging_only_scope, @@ -1377,14 +1379,16 @@ class TestLoggingOnlyScopeValidation: "guardrail, whose logging_only hook scans on its own. Remove logging_only_scope." ) - def test_both_scope_accepted_when_guardrail_owns_logging_hook(self) -> None: + def test_continue_flag_accepted_when_guardrail_owns_logging_hook(self) -> None: callback: Final = self._initialize( mode="logging_only", - scope="both", + scope=None, + continue_on_input_failure=True, callback_type=_LoggingOnlyScopeUnsupportedGuardrail, ) - assert callback.logging_only_scope == "both" + assert callback.logging_only_scope is None + assert callback.logging_only_continue_on_input_failure is True def test_output_scope_accepted_for_native_lifecycle_guardrail(self) -> None: callback: Final = self._initialize( @@ -1395,9 +1399,10 @@ class TestLoggingOnlyScopeValidation: assert callback.logging_only_scope == "output" - def test_invalid_scope_fails_litellm_params_validation(self) -> None: + @pytest.mark.parametrize("scope", ("request", "both")) + def test_invalid_scope_fails_litellm_params_validation(self, scope: str) -> None: with pytest.raises(ValidationError): - LitellmParams(guardrail="test", mode="logging_only", logging_only_scope="request") + LitellmParams(guardrail="test", mode="logging_only", logging_only_scope=scope) def test_invalid_scope_literal_keeps_content_filter_registered_and_blocking(self) -> None: import litellm @@ -1438,6 +1443,85 @@ class TestLoggingOnlyScopeValidation: assert all(callback_list == snapshot for callback_list, snapshot in zip(callback_lists, callback_snapshots)) + @pytest.mark.parametrize( + ("scope", "continue_on_input_failure", "expected_direction", "expected_continue"), + ( + (None, None, None, False), + (None, True, None, True), + (None, False, None, False), + ("input", True, "input", False), + ("output", True, "output", False), + ("input", None, "input", False), + ), + ) + def test_logging_only_settings_table( + self, + scope: LoggingOnlyScope | None, + continue_on_input_failure: bool | None, + expected_direction: str | None, + expected_continue: bool, + ) -> None: + callback: Final = self._initialize( + mode="logging_only", + scope=scope, + continue_on_input_failure=continue_on_input_failure, + ) + + assert callback.logging_only_scope == expected_direction + assert callback.logging_only_continue_on_input_failure is expected_continue + + def test_continue_flag_without_logging_only_mode_is_ignored_at_load(self) -> None: + callback: Final = self._initialize( + mode="pre_call", continue_on_input_failure=True, scope=None, assert_registered=True + ) + + assert callback.logging_only_scope is None + assert callback.logging_only_continue_on_input_failure is False + assert callback.should_run_guardrail(data={}, event_type=GuardrailEventHooks.pre_call) is True + + def test_continue_flag_without_logging_only_mode_is_rejected_for_api_writes(self) -> None: + with pytest.raises(ValueError, match="logging_only_continue_on_input_failure is set") as exc_info: + self._initialize( + mode="pre_call", + scope=None, + continue_on_input_failure=True, + reject_invalid_logging_only_scope=True, + ) + + assert str(exc_info.value) == ( + "Guardrail logging-only-scope-guardrail: logging_only_continue_on_input_failure is set, but mode " + "does not include logging_only, so it would never apply. Add logging_only to mode or remove " + "logging_only_continue_on_input_failure." + ) + + def test_continue_flag_is_accepted_when_guardrail_owns_logging_hook(self) -> None: + callback: Final = self._initialize( + mode="logging_only", + scope=None, + continue_on_input_failure=True, + callback_type=_LoggingOnlyScopeUnsupportedGuardrail, + ) + + assert callback.logging_only_scope is None + assert callback.logging_only_continue_on_input_failure is True + + def test_invalid_continue_flag_is_tolerated_by_parse_tolerant(self) -> None: + params: Final = { + "guardrail": "litellm_content_filter", + "mode": "logging_only", + "logging_only_continue_on_input_failure": "banana", + } + + parsed: Final = parse_tolerant_litellm_params(params, "invalid-flag-test") + + assert parsed.logging_only_continue_on_input_failure is None + + def test_invalid_continue_flag_raises_without_tolerance(self) -> None: + with pytest.raises(ValidationError): + LitellmParams( + guardrail="test", mode="logging_only", logging_only_continue_on_input_failure="banana" + ) + def test_invalid_scope_literal_does_not_tolerate_other_litellm_params_errors(self) -> None: with pytest.raises(ValidationError): parse_tolerant_litellm_params( diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailFormField.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailFormField.tsx index ef0c8d074a7..071d8010ea0 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailFormField.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailFormField.tsx @@ -2,9 +2,16 @@ import { CircleHelp } from "lucide-react"; import React, { useEffect, useId } from "react"; -import { useController, type Control, type ControllerRenderProps, type RegisterOptions } from "react-hook-form"; +import { + useController, + useWatch, + type Control, + type ControllerRenderProps, + type RegisterOptions, +} from "react-hook-form"; import { Field, FieldDescription, FieldError, FieldLabel } from "@/components/ui/field"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Switch } from "@/components/ui/switch"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { getLoggingOnlyScopeOptions, @@ -22,6 +29,7 @@ export interface GuardrailCriterion { export interface GuardrailFormValues extends Record { criteria?: GuardrailCriterion[]; logging_only_scope_choice?: LoggingOnlyScopeChoice; + logging_only_continue_on_input_failure?: boolean; } export type GuardrailFormControl = Control; export type GuardrailFieldRules = Pick, "validate">; @@ -172,20 +180,44 @@ export const LoggingOnlyScopeField: React.FC<{ mode: unknown; directionalScopeSupported: boolean; }> = ({ control, mode, directionalScopeSupported }) => { + const scopeChoice = useWatch({ control, name: "logging_only_scope_choice" }); + const continueDisabled = scopeChoice === "input" || scopeChoice === "output"; if (!modeIncludesLoggingOnly(mode)) return null; return ( - - {(fieldControl) => ( - - )} - + <> + + {(fieldControl) => ( + + )} + + + {(fieldControl) => ( + + )} + + ); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailModeDisplay.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailModeDisplay.tsx index e5fa1669a44..2121a83f9fd 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailModeDisplay.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailModeDisplay.tsx @@ -1,12 +1,18 @@ import React from "react"; import { Badge } from "@/components/ui/badge"; import { Card } from "@/components/ui/card"; -import { formatGuardrailMode, formatLoggingOnlyScope, modeIncludesLoggingOnly } from "./guardrail_info_helpers"; +import { + formatGuardrailMode, + formatLoggingOnlyScope, + loggingOnlyContinueFromParams, + modeIncludesLoggingOnly, +} from "./guardrail_info_helpers"; type GuardrailModeParams = { mode?: unknown; default_on?: boolean; logging_only_scope?: string | null; + logging_only_continue_on_input_failure?: boolean | null; }; export const GuardrailModeCard: React.FC<{ litellmParams: GuardrailModeParams }> = ({ litellmParams }) => ( @@ -22,6 +28,8 @@ export const GuardrailModeCard: React.FC<{ litellmParams: GuardrailModeParams }>

Logging only scope

{formatLoggingOnlyScope(litellmParams.logging_only_scope)}

+

Continue after a flagged request

+

{loggingOnlyContinueFromParams(litellmParams) ? "On" : "Off"}

)} @@ -37,6 +45,8 @@ export const GuardrailModeRows: React.FC<{ litellmParams: GuardrailModeParams }>

Logging only scope

{formatLoggingOnlyScope(litellmParams.logging_only_scope)}
+

Continue after a flagged request

+
{loggingOnlyContinueFromParams(litellmParams) ? "On" : "Off"}
)} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailReadOnlyDetails.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailReadOnlyDetails.tsx index 5fe88b4870c..f0583102fba 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailReadOnlyDetails.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailReadOnlyDetails.tsx @@ -19,7 +19,11 @@ export const GuardrailReadOnlyDetails = ({ guardrailId: string; guardrailName: string; displayName: string; - litellmParams: { mode?: unknown; logging_only_scope?: string | null }; + litellmParams: { + mode?: unknown; + logging_only_scope?: string | null; + logging_only_continue_on_input_failure?: boolean | null; + }; streamScope: unknown; defaultOn: boolean | undefined; piiEntityCount: number; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx index 076b2ada77f..05bec9208f7 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx @@ -145,6 +145,26 @@ describe("AddGuardrailForm create payload characterization", () => { expect(payload()?.litellm_params.logging_only_scope).toBe("output"); }); + it("disables and unchecks the continue toggle under a directional scope", async () => { + vi.mocked(networking.getGuardrailUISettings).mockResolvedValue({ + ...uiSettings, + supported_modes: ["pre_call", "logging_only"], + }); + const user = userEvent.setup({ delay: null }); + renderForm(); + + await pickProvider(user, "Bedrock Guardrail"); + await user.click(screen.getByLabelText("Mode")); + await user.click((await screen.findAllByText("logging_only")).at(-1) as HTMLElement); + await chooseSelectOption(user, await screen.findByLabelText("Logging only scope"), "Input only (request)"); + + const toggle = await screen.findByRole("switch", { + name: "Continue observing the response after a flagged request", + }); + expect(toggle).toHaveAttribute("aria-disabled", "true"); + expect(toggle).not.toBeChecked(); + }); + it("hides directional scope choices for providers that do not support them", async () => { vi.mocked(networking.getGuardrailUISettings).mockResolvedValue({ ...uiSettings, @@ -163,7 +183,7 @@ describe("AddGuardrailForm create payload characterization", () => { await user.click((await screen.findAllByText("logging_only")).at(-1) as HTMLElement); await user.click(await screen.findByLabelText("Logging only scope")); - expect(await screen.findByRole("option", { name: "Both (request and response)" })).toBeInTheDocument(); + expect(await screen.findByRole("option", { name: "Default (request and response)" })).toBeInTheDocument(); expect(screen.queryByRole("option", { name: "Input only (request)" })).not.toBeInTheDocument(); expect(screen.queryByRole("option", { name: "Output only (response)" })).not.toBeInTheDocument(); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx index 098245fa34b..88093ac4517 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx @@ -17,6 +17,7 @@ import { getGuardrailProviders, getSupportedModesForProvider, guardrail_provider_map, + effectiveLoggingOnlyContinue, modeIncludesLoggingOnly, populateGuardrailProviderMap, populateGuardrailProviders, @@ -173,6 +174,7 @@ const INITIAL_VALUES: GuardrailFormValues = { mode: "pre_call", default_on: false, logging_only_scope_choice: "default", + logging_only_continue_on_input_failure: false, skip_system_message_choice: "inherit", skip_tool_message_choice: "inherit", stream_scope_by_mode: {}, @@ -294,6 +296,7 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a mode: preset.mode, default_on: preset.defaultOn, logging_only_scope_choice: "default", + logging_only_continue_on_input_failure: false, skip_system_message_choice: "inherit", skip_tool_message_choice: "inherit", }; @@ -494,6 +497,15 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a if (modeIncludesLoggingOnly(values.mode) && loggingOnlyScope !== null) { guardrailData.litellm_params.logging_only_scope = loggingOnlyScope; } + if ( + modeIncludesLoggingOnly(values.mode) && + effectiveLoggingOnlyContinue( + values.logging_only_scope_choice as LoggingOnlyScopeChoice | undefined, + values.logging_only_continue_on_input_failure, + ) + ) { + guardrailData.litellm_params.logging_only_continue_on_input_failure = true; + } // For Presidio PII, add the entity and action configurations if (providerKey === "PresidioPII" && selectedEntities.length > 0) { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/custom_code/CustomCodeLoggingOnlyScope.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/custom_code/CustomCodeLoggingOnlyScope.tsx index 8239933d649..c2b0a472f1f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/custom_code/CustomCodeLoggingOnlyScope.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/custom_code/CustomCodeLoggingOnlyScope.tsx @@ -1,6 +1,7 @@ import React from "react"; import { choiceToLoggingOnlyScope, + effectiveLoggingOnlyContinue, getLoggingOnlyScopeOptions, getLoggingOnlyScopeUpdate, type LoggingOnlyScope, @@ -18,7 +19,7 @@ export const CustomCodeLoggingOnlyScopeSelect: React.FC<{ value: LoggingOnlyScopeChoice; onChange: (choice: LoggingOnlyScopeChoice) => void; }> = ({ value, onChange }) => ( -
+
= ({ visible, onClose, onS {mode.includes("logging_only") && ( )} + {mode.includes("logging_only") && ( +
+ + Continue observing the response after a flagged request + + +
+ )}