From 698f608ad6ee78ee7d84d8947c386244ea7b1e4e Mon Sep 17 00:00:00 2001 From: Yucheng Zhu Date: Mon, 31 Aug 2026 18:26:15 -0700 Subject: [PATCH 01/23] fix(guardrails): transfer guardrail evaluation metadata to spend logs on success path On successful requests, guardrail evaluations run and store results in request_data["litellm_metadata"]["standard_logging_guardrail_information"], but the spend-log serialization reads from request_data["metadata"], so guardrail evaluations were never reported in Request Logs. The failure path already had this transfer; this adds the same logic to the success path so guardrail evaluation info appears in both success and failure spend logs. Fixes LIT-6314. --- .../proxy/hooks/proxy_track_cost_callback.py | 11 ++ .../hooks/test_proxy_track_cost_callback.py | 132 ++++++++++-------- 2 files changed, 85 insertions(+), 58 deletions(-) diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 47aafda2337..ca8341822e9 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -269,6 +269,17 @@ class _ProxyDBLogger(CustomLogger): served_model_id=sl_object.get("model_id") if sl_object is not None else None, router=get_llm_router(), ) + # LIT-6314: post-call guardrail evaluations run after SLP is built, so + # guardrail_information is missing from SLP. Populate it from post-call evals + # before spend-log write so reports show accurate guardrail results. + if sl_object is not None: + guardrail_info_from_hooks: Final = ( + kwargs.get("litellm_metadata", {}).get("standard_logging_guardrail_information") + if isinstance(kwargs.get("litellm_metadata"), dict) + else None + ) + if guardrail_info_from_hooks is not None and not sl_object.get("guardrail_information"): + sl_object["guardrail_information"] = guardrail_info_from_hooks if response_cost is not None: user_api_key: Final = metadata.get("user_api_key", None) diff --git a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py index 8043a1aca3f..5eb59acf4f5 100644 --- a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py @@ -1,4 +1,3 @@ - from datetime import datetime from unittest.mock import AsyncMock, MagicMock, patch @@ -70,9 +69,7 @@ async def test_async_post_call_failure_hook(): # Check that metadata was properly updated assert "litellm_params" in call_args["kwargs"] - assert call_args["kwargs"]["litellm_params"]["proxy_server_request"] == { - "request_id": "test_request_id" - } + assert call_args["kwargs"]["litellm_params"]["proxy_server_request"] == {"request_id": "test_request_id"} metadata = call_args["kwargs"]["litellm_params"]["metadata"] assert metadata["user_api_key"] == "test_api_key" assert metadata["status"] == "failure" @@ -336,9 +333,7 @@ async def test_should_continue_failure_tracking_when_budget_release_fails(): ) assert mock_invalidate_budget_reservation_counters.await_count == 1 assert ( - mock_invalidate_budget_reservation_counters.await_args.kwargs[ - "budget_reservation" - ] + mock_invalidate_budget_reservation_counters.await_args.kwargs["budget_reservation"] is user_api_key_dict.budget_reservation ) assert user_api_key_dict.budget_reservation["finalized"] is True @@ -433,36 +428,21 @@ def test_get_budget_reservation_from_metadata_handles_dict_auth_object(): "entries": [{"counter_key": "spend:key:test_api_key"}], } + assert _get_budget_reservation_from_metadata(metadata={"user_api_key_auth": dict(UserAPIKeyAuth())}) is None assert ( _get_budget_reservation_from_metadata( - metadata={"user_api_key_auth": dict(UserAPIKeyAuth())} - ) - is None - ) - assert ( - _get_budget_reservation_from_metadata( - metadata={ - "user_api_key_auth": UserAPIKeyAuth( - budget_reservation=budget_reservation - ) - } + metadata={"user_api_key_auth": UserAPIKeyAuth(budget_reservation=budget_reservation)} ) == budget_reservation ) assert ( _get_budget_reservation_from_metadata( - metadata={ - "user_api_key_auth": dict( - UserAPIKeyAuth(budget_reservation=budget_reservation) - ) - } + metadata={"user_api_key_auth": dict(UserAPIKeyAuth(budget_reservation=budget_reservation))} ) == budget_reservation ) assert ( - _get_budget_reservation_from_metadata( - metadata={"user_api_key_budget_reservation": budget_reservation} - ) + _get_budget_reservation_from_metadata(metadata={"user_api_key_budget_reservation": budget_reservation}) is budget_reservation ) @@ -470,9 +450,7 @@ def test_get_budget_reservation_from_metadata_handles_dict_auth_object(): @pytest.mark.asyncio async def test_update_database_and_spend_counters_releases_reservation_when_db_update_fails(): proxy_logging_obj = MagicMock() - proxy_logging_obj.db_spend_update_writer.update_database = AsyncMock( - side_effect=Exception("db unavailable") - ) + proxy_logging_obj.db_spend_update_writer.update_database = AsyncMock(side_effect=Exception("db unavailable")) increment_spend_counters = AsyncMock() budget_reservation = {"reserved_cost": 0.5, "entries": []} @@ -508,9 +486,7 @@ async def test_update_database_and_spend_counters_releases_reservation_when_db_u async def test_update_database_and_spend_counters_preserves_db_exception_when_release_fails(): proxy_logging_obj = MagicMock() db_exception = RuntimeError("db unavailable") - proxy_logging_obj.db_spend_update_writer.update_database = AsyncMock( - side_effect=db_exception - ) + proxy_logging_obj.db_spend_update_writer.update_database = AsyncMock(side_effect=db_exception) increment_spend_counters = AsyncMock() budget_reservation = {"reserved_cost": 0.5, "entries": []} @@ -554,12 +530,8 @@ async def test_update_database_and_spend_counters_preserves_db_exception_when_re budget_reservation=budget_reservation, ) assert mock_log_exception.call_count == 2 - mock_log_exception.assert_any_call( - "Failed to release budget reservation after database update failed" - ) - mock_log_exception.assert_any_call( - "Failed to invalidate budget reservation counters after release failed" - ) + mock_log_exception.assert_any_call("Failed to release budget reservation after database update failed") + mock_log_exception.assert_any_call("Failed to invalidate budget reservation counters after release failed") increment_spend_counters.assert_not_awaited() @@ -1101,10 +1073,7 @@ async def test_async_post_call_failure_hook_propagates_trace_id_from_logging_obj # standard_logging_object should have been propagated from logging obj assert call_kwargs.get("standard_logging_object") is not None - assert ( - call_kwargs["standard_logging_object"]["trace_id"] - == "trace-id-from-logging-obj" - ) + assert call_kwargs["standard_logging_object"]["trace_id"] == "trace-id-from-logging-obj" # litellm_trace_id should also be propagated as a fallback assert call_kwargs.get("litellm_trace_id") == "trace-id-from-logging-obj" @@ -1691,9 +1660,7 @@ async def test_async_post_call_failure_hook_records_recovered_partial_spend(): "metadata": {}, "proxy_server_request": {"request_id": "rid"}, "response_cost": 3.5e-05, - "combined_usage_object": Usage( - prompt_tokens=30, completion_tokens=1, total_tokens=31 - ), + "combined_usage_object": Usage(prompt_tokens=30, completion_tokens=1, total_tokens=31), } with patch( @@ -1772,15 +1739,10 @@ async def test_track_cost_callback_enriches_user_id_for_mcp_style_metadata(): assert mock_increment.call_args.kwargs["team_id"] == "team-123" assert mock_increment.call_args.kwargs["org_id"] == "org-456" - update_kwargs = ( - mock_proxy_logging.db_spend_update_writer.update_database.await_args.kwargs - ) + update_kwargs = mock_proxy_logging.db_spend_update_writer.update_database.await_args.kwargs assert update_kwargs["user_id"] == "mcp-user@example.com" assert update_kwargs["team_id"] == "team-123" - assert ( - kwargs["litellm_params"]["metadata"]["user_api_key_user_id"] - == "mcp-user@example.com" - ) + assert kwargs["litellm_params"]["metadata"]["user_api_key_user_id"] == "mcp-user@example.com" @pytest.mark.parametrize( @@ -1828,9 +1790,7 @@ def test_should_track_cost_callback_pass_through_without_owner(call_type, expect ], ) @pytest.mark.asyncio -async def test_track_cost_callback_logs_unauthenticated_pass_through_request( - call_type, expect_spend_log -): +async def test_track_cost_callback_logs_unauthenticated_pass_through_request(call_type, expect_spend_log): """Regression for LIT-3782: a pass-through request with auth=false reaches the cost callback with no key/user/team/end-user. Before the fix the spend-log write was skipped and the request never appeared in request/usage logs. It @@ -1876,9 +1836,7 @@ async def test_track_cost_callback_logs_unauthenticated_pass_through_request( end_time=datetime.now(), ) - assert mock_proxy_logging.db_spend_update_writer.update_database.await_count == ( - 1 if expect_spend_log else 0 - ) + assert mock_proxy_logging.db_spend_update_writer.update_database.await_count == (1 if expect_spend_log else 0) class _FakeDeploymentLookup: @@ -1989,3 +1947,61 @@ async def test_spend_counters_keep_every_granted_group_when_the_deployment_is_un ) assert charged == ("premium", "tier0") + + +@pytest.mark.asyncio +async def test_proxy_track_cost_callback_carries_guardrail_info_to_sl_object(): + """ + LIT-6314 regression: post-call guardrails append evaluations to + request_data["litellm_metadata"]["standard_logging_guardrail_information"], + but the standard_logging_object (built pre-post-call) lacks this field. + The callback must populate standard_logging_object["guardrail_information"] + from post-call evals before spend-log write so reports show results. + """ + logger = _ProxyDBLogger() + guardrail_info = [ + { + "guardrail_name": "bedrock-guard", + "guardrail_status": "success", + "guardrail_cost": 0.0001, + } + ] + sl_object = {"response_cost": 0.01} + kwargs = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}], + "litellm_metadata": {"standard_logging_guardrail_information": guardrail_info}, + "litellm_params": { + "metadata": {"user_api_key": "test_key"}, + }, + "call_type": CallTypes.completion.value, + "response_cost": 0.01, + "standard_logging_object": sl_object, + } + + with ( + patch( + "litellm.proxy.proxy_server.increment_spend_counters", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy.proxy_server.update_cache", + new_callable=AsyncMock, + ), + patch( + "litellm.proxy.proxy_server.proxy_logging_obj", + ) as mock_proxy_logging, + ): + mock_proxy_logging.db_spend_update_writer.update_database = AsyncMock() + mock_proxy_logging.slack_alerting_instance.customer_spend_alert = AsyncMock() + mock_proxy_logging.failed_tracking_alert = AsyncMock() + + await logger._PROXY_track_cost_callback( + kwargs=kwargs, + completion_response=None, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + assert mock_proxy_logging.db_spend_update_writer.update_database.await_count == 1 + assert sl_object.get("guardrail_information") == guardrail_info From ca9e121eb67185debdf1af3dea64ab2e080f3df1 Mon Sep 17 00:00:00 2001 From: Yucheng Zhu Date: Mon, 31 Aug 2026 18:57:20 -0700 Subject: [PATCH 02/23] fix(guardrails): avoid mutable dict literal in guardrail metadata population --- litellm/proxy/hooks/proxy_track_cost_callback.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index ca8341822e9..b78884f7d95 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -273,13 +273,16 @@ class _ProxyDBLogger(CustomLogger): # guardrail_information is missing from SLP. Populate it from post-call evals # before spend-log write so reports show accurate guardrail results. if sl_object is not None: + litellm_metadata: Final = kwargs.get("litellm_metadata") guardrail_info_from_hooks: Final = ( - kwargs.get("litellm_metadata", {}).get("standard_logging_guardrail_information") - if isinstance(kwargs.get("litellm_metadata"), dict) + litellm_metadata.get("standard_logging_guardrail_information") + if isinstance(litellm_metadata, dict) else None ) if guardrail_info_from_hooks is not None and not sl_object.get("guardrail_information"): - sl_object["guardrail_information"] = guardrail_info_from_hooks + sl_object["guardrail_information"] = ( + guardrail_info_from_hooks # mutable-ok: populate SLP before spend-log write + ) if response_cost is not None: user_api_key: Final = metadata.get("user_api_key", None) From 686e8ddae7aaac16089de238d09c676b6b62824a Mon Sep 17 00:00:00 2001 From: Yucheng Zhu Date: Mon, 31 Aug 2026 19:04:14 -0700 Subject: [PATCH 03/23] test: suppress TQ008 on proxy_server global patches, matching file idiom --- .../proxy/hooks/test_proxy_track_cost_callback.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py index 5eb59acf4f5..2d5857f95e5 100644 --- a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py @@ -1980,15 +1980,15 @@ async def test_proxy_track_cost_callback_carries_guardrail_info_to_sl_object(): } with ( - patch( + patch( # test-quality-ok: spend counters are a proxy_server global the callback reads lazily, no seam "litellm.proxy.proxy_server.increment_spend_counters", new_callable=AsyncMock, ), - patch( + patch( # test-quality-ok: update_cache is a proxy_server global the callback reads lazily, no seam "litellm.proxy.proxy_server.update_cache", new_callable=AsyncMock, ), - patch( + patch( # test-quality-ok: callback imports proxy_logging_obj off proxy_server in its body, no seam "litellm.proxy.proxy_server.proxy_logging_obj", ) as mock_proxy_logging, ): From 0e6c4279b6387a26ee43e7ef8b8454bdee65295e Mon Sep 17 00:00:00 2001 From: Yucheng Zhu Date: Mon, 31 Aug 2026 21:35:32 -0700 Subject: [PATCH 04/23] fix(guardrails): record not_run evaluation when scoping leaves nothing to scan Replaces the metadata transfer approach: that block read top-level litellm_metadata which guardrail info never populates, and the SLP builder already reads the nested bucket it lands in, so it was dead code and is reverted. Real cause of missing evaluations: process_input_messages skips apply_guardrail entirely when message scoping (skip_system_message, skip_tool, scan_only_tool_results) leaves no scannable content, so the guardrail shows up in applied_guardrails with no guardrail_information entry. Now records a not_run entry unless the guardrail records its own information. --- .../chat/guardrail_translation/handler.py | 9 ++ .../proxy/hooks/proxy_track_cost_callback.py | 14 -- .../test_openai_guardrail_handler.py | 50 +++++++ .../hooks/test_proxy_track_cost_callback.py | 132 ++++++++---------- 4 files changed, 117 insertions(+), 88 deletions(-) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index de15fefe943..76a7f33454a 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -191,6 +191,15 @@ class OpenAIChatCompletionsHandler(BaseTranslation): task_mappings=tool_call_task_mappings, ) + elif not guardrail_to_apply.records_own_guardrail_information: + # Every guardrail in applied_guardrails needs a persisted evaluation record, + # or request logs report it as silently missing (LIT-6314). + guardrail_to_apply.add_standard_logging_guardrail_information_to_request_data( + guardrail_json_response="no scannable content after message scoping", + request_data=data, + guardrail_status="not_run", + ) + verbose_proxy_logger.debug( "OpenAI Chat Completions: Processed input messages: %s", data.get("messages"), diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index b78884f7d95..47aafda2337 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -269,20 +269,6 @@ class _ProxyDBLogger(CustomLogger): served_model_id=sl_object.get("model_id") if sl_object is not None else None, router=get_llm_router(), ) - # LIT-6314: post-call guardrail evaluations run after SLP is built, so - # guardrail_information is missing from SLP. Populate it from post-call evals - # before spend-log write so reports show accurate guardrail results. - if sl_object is not None: - litellm_metadata: Final = kwargs.get("litellm_metadata") - guardrail_info_from_hooks: Final = ( - litellm_metadata.get("standard_logging_guardrail_information") - if isinstance(litellm_metadata, dict) - else None - ) - if guardrail_info_from_hooks is not None and not sl_object.get("guardrail_information"): - sl_object["guardrail_information"] = ( - guardrail_info_from_hooks # mutable-ok: populate SLP before spend-log write - ) if response_cost is not None: user_api_key: Final = metadata.get("user_api_key", None) diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index a29e0be4655..24060e94e54 100644 --- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1559,3 +1559,53 @@ class TestScanOnlyToolResults: assert data["messages"][3]["content"] == "page says [BLOCKED] here" assert data["messages"][3]["tool_call_id"] == "call_1" assert data["messages"][4]["content"] == "and then?" + + +class TestNoScannableContentRecordsNotRun: + """LIT-6314: a guardrail whose scoping leaves nothing to scan must still persist an evaluation record""" + + def _system_only_data(self) -> dict: + return {"messages": [{"role": "system", "content": "SYSTEM-PROMPT"}]} + + def _recorded_entries(self, data: dict) -> list: + metadata = data.get("metadata") or data.get("litellm_metadata") or {} + return metadata.get("standard_logging_guardrail_information") or [] + + @pytest.mark.asyncio + async def test_skipped_scan_records_not_run_entry(self): + handler = OpenAIChatCompletionsHandler() + guardrail = MockGuardrail(guardrail_name="skip-system-guardrail") + guardrail.skip_system_message_in_guardrail = True + data = self._system_only_data() + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert guardrail.last_inputs is None, "nothing survived scoping, apply_guardrail must not run" + entries = self._recorded_entries(data) + assert len(entries) == 1 + assert entries[0]["guardrail_name"] == "skip-system-guardrail" + assert entries[0]["guardrail_status"] == "not_run" + + @pytest.mark.asyncio + async def test_self_recording_guardrail_is_left_alone(self): + handler = OpenAIChatCompletionsHandler() + guardrail = MockGuardrail(guardrail_name="self-recording-guardrail") + guardrail.skip_system_message_in_guardrail = True + guardrail.records_own_guardrail_information = True + data = self._system_only_data() + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert guardrail.last_inputs is None + assert self._recorded_entries(data) == [] + + @pytest.mark.asyncio + async def test_scannable_content_records_no_extra_entry(self): + handler = OpenAIChatCompletionsHandler() + guardrail = MockGuardrail(guardrail_name="normal-guardrail") + data = {"messages": [{"role": "user", "content": "hello"}]} + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert guardrail.last_inputs is not None + assert all(e.get("guardrail_status") != "not_run" for e in self._recorded_entries(data)) diff --git a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py index 2d5857f95e5..8043a1aca3f 100644 --- a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py @@ -1,3 +1,4 @@ + from datetime import datetime from unittest.mock import AsyncMock, MagicMock, patch @@ -69,7 +70,9 @@ async def test_async_post_call_failure_hook(): # Check that metadata was properly updated assert "litellm_params" in call_args["kwargs"] - assert call_args["kwargs"]["litellm_params"]["proxy_server_request"] == {"request_id": "test_request_id"} + assert call_args["kwargs"]["litellm_params"]["proxy_server_request"] == { + "request_id": "test_request_id" + } metadata = call_args["kwargs"]["litellm_params"]["metadata"] assert metadata["user_api_key"] == "test_api_key" assert metadata["status"] == "failure" @@ -333,7 +336,9 @@ async def test_should_continue_failure_tracking_when_budget_release_fails(): ) assert mock_invalidate_budget_reservation_counters.await_count == 1 assert ( - mock_invalidate_budget_reservation_counters.await_args.kwargs["budget_reservation"] + mock_invalidate_budget_reservation_counters.await_args.kwargs[ + "budget_reservation" + ] is user_api_key_dict.budget_reservation ) assert user_api_key_dict.budget_reservation["finalized"] is True @@ -428,21 +433,36 @@ def test_get_budget_reservation_from_metadata_handles_dict_auth_object(): "entries": [{"counter_key": "spend:key:test_api_key"}], } - assert _get_budget_reservation_from_metadata(metadata={"user_api_key_auth": dict(UserAPIKeyAuth())}) is None assert ( _get_budget_reservation_from_metadata( - metadata={"user_api_key_auth": UserAPIKeyAuth(budget_reservation=budget_reservation)} + metadata={"user_api_key_auth": dict(UserAPIKeyAuth())} + ) + is None + ) + assert ( + _get_budget_reservation_from_metadata( + metadata={ + "user_api_key_auth": UserAPIKeyAuth( + budget_reservation=budget_reservation + ) + } ) == budget_reservation ) assert ( _get_budget_reservation_from_metadata( - metadata={"user_api_key_auth": dict(UserAPIKeyAuth(budget_reservation=budget_reservation))} + metadata={ + "user_api_key_auth": dict( + UserAPIKeyAuth(budget_reservation=budget_reservation) + ) + } ) == budget_reservation ) assert ( - _get_budget_reservation_from_metadata(metadata={"user_api_key_budget_reservation": budget_reservation}) + _get_budget_reservation_from_metadata( + metadata={"user_api_key_budget_reservation": budget_reservation} + ) is budget_reservation ) @@ -450,7 +470,9 @@ def test_get_budget_reservation_from_metadata_handles_dict_auth_object(): @pytest.mark.asyncio async def test_update_database_and_spend_counters_releases_reservation_when_db_update_fails(): proxy_logging_obj = MagicMock() - proxy_logging_obj.db_spend_update_writer.update_database = AsyncMock(side_effect=Exception("db unavailable")) + proxy_logging_obj.db_spend_update_writer.update_database = AsyncMock( + side_effect=Exception("db unavailable") + ) increment_spend_counters = AsyncMock() budget_reservation = {"reserved_cost": 0.5, "entries": []} @@ -486,7 +508,9 @@ async def test_update_database_and_spend_counters_releases_reservation_when_db_u async def test_update_database_and_spend_counters_preserves_db_exception_when_release_fails(): proxy_logging_obj = MagicMock() db_exception = RuntimeError("db unavailable") - proxy_logging_obj.db_spend_update_writer.update_database = AsyncMock(side_effect=db_exception) + proxy_logging_obj.db_spend_update_writer.update_database = AsyncMock( + side_effect=db_exception + ) increment_spend_counters = AsyncMock() budget_reservation = {"reserved_cost": 0.5, "entries": []} @@ -530,8 +554,12 @@ async def test_update_database_and_spend_counters_preserves_db_exception_when_re budget_reservation=budget_reservation, ) assert mock_log_exception.call_count == 2 - mock_log_exception.assert_any_call("Failed to release budget reservation after database update failed") - mock_log_exception.assert_any_call("Failed to invalidate budget reservation counters after release failed") + mock_log_exception.assert_any_call( + "Failed to release budget reservation after database update failed" + ) + mock_log_exception.assert_any_call( + "Failed to invalidate budget reservation counters after release failed" + ) increment_spend_counters.assert_not_awaited() @@ -1073,7 +1101,10 @@ async def test_async_post_call_failure_hook_propagates_trace_id_from_logging_obj # standard_logging_object should have been propagated from logging obj assert call_kwargs.get("standard_logging_object") is not None - assert call_kwargs["standard_logging_object"]["trace_id"] == "trace-id-from-logging-obj" + assert ( + call_kwargs["standard_logging_object"]["trace_id"] + == "trace-id-from-logging-obj" + ) # litellm_trace_id should also be propagated as a fallback assert call_kwargs.get("litellm_trace_id") == "trace-id-from-logging-obj" @@ -1660,7 +1691,9 @@ async def test_async_post_call_failure_hook_records_recovered_partial_spend(): "metadata": {}, "proxy_server_request": {"request_id": "rid"}, "response_cost": 3.5e-05, - "combined_usage_object": Usage(prompt_tokens=30, completion_tokens=1, total_tokens=31), + "combined_usage_object": Usage( + prompt_tokens=30, completion_tokens=1, total_tokens=31 + ), } with patch( @@ -1739,10 +1772,15 @@ async def test_track_cost_callback_enriches_user_id_for_mcp_style_metadata(): assert mock_increment.call_args.kwargs["team_id"] == "team-123" assert mock_increment.call_args.kwargs["org_id"] == "org-456" - update_kwargs = mock_proxy_logging.db_spend_update_writer.update_database.await_args.kwargs + update_kwargs = ( + mock_proxy_logging.db_spend_update_writer.update_database.await_args.kwargs + ) assert update_kwargs["user_id"] == "mcp-user@example.com" assert update_kwargs["team_id"] == "team-123" - assert kwargs["litellm_params"]["metadata"]["user_api_key_user_id"] == "mcp-user@example.com" + assert ( + kwargs["litellm_params"]["metadata"]["user_api_key_user_id"] + == "mcp-user@example.com" + ) @pytest.mark.parametrize( @@ -1790,7 +1828,9 @@ def test_should_track_cost_callback_pass_through_without_owner(call_type, expect ], ) @pytest.mark.asyncio -async def test_track_cost_callback_logs_unauthenticated_pass_through_request(call_type, expect_spend_log): +async def test_track_cost_callback_logs_unauthenticated_pass_through_request( + call_type, expect_spend_log +): """Regression for LIT-3782: a pass-through request with auth=false reaches the cost callback with no key/user/team/end-user. Before the fix the spend-log write was skipped and the request never appeared in request/usage logs. It @@ -1836,7 +1876,9 @@ async def test_track_cost_callback_logs_unauthenticated_pass_through_request(cal end_time=datetime.now(), ) - assert mock_proxy_logging.db_spend_update_writer.update_database.await_count == (1 if expect_spend_log else 0) + assert mock_proxy_logging.db_spend_update_writer.update_database.await_count == ( + 1 if expect_spend_log else 0 + ) class _FakeDeploymentLookup: @@ -1947,61 +1989,3 @@ async def test_spend_counters_keep_every_granted_group_when_the_deployment_is_un ) assert charged == ("premium", "tier0") - - -@pytest.mark.asyncio -async def test_proxy_track_cost_callback_carries_guardrail_info_to_sl_object(): - """ - LIT-6314 regression: post-call guardrails append evaluations to - request_data["litellm_metadata"]["standard_logging_guardrail_information"], - but the standard_logging_object (built pre-post-call) lacks this field. - The callback must populate standard_logging_object["guardrail_information"] - from post-call evals before spend-log write so reports show results. - """ - logger = _ProxyDBLogger() - guardrail_info = [ - { - "guardrail_name": "bedrock-guard", - "guardrail_status": "success", - "guardrail_cost": 0.0001, - } - ] - sl_object = {"response_cost": 0.01} - kwargs = { - "model": "gpt-4", - "messages": [{"role": "user", "content": "Hello"}], - "litellm_metadata": {"standard_logging_guardrail_information": guardrail_info}, - "litellm_params": { - "metadata": {"user_api_key": "test_key"}, - }, - "call_type": CallTypes.completion.value, - "response_cost": 0.01, - "standard_logging_object": sl_object, - } - - with ( - patch( # test-quality-ok: spend counters are a proxy_server global the callback reads lazily, no seam - "litellm.proxy.proxy_server.increment_spend_counters", - new_callable=AsyncMock, - ), - patch( # test-quality-ok: update_cache is a proxy_server global the callback reads lazily, no seam - "litellm.proxy.proxy_server.update_cache", - new_callable=AsyncMock, - ), - patch( # test-quality-ok: callback imports proxy_logging_obj off proxy_server in its body, no seam - "litellm.proxy.proxy_server.proxy_logging_obj", - ) as mock_proxy_logging, - ): - mock_proxy_logging.db_spend_update_writer.update_database = AsyncMock() - mock_proxy_logging.slack_alerting_instance.customer_spend_alert = AsyncMock() - mock_proxy_logging.failed_tracking_alert = AsyncMock() - - await logger._PROXY_track_cost_callback( - kwargs=kwargs, - completion_response=None, - start_time=datetime.now(), - end_time=datetime.now(), - ) - - assert mock_proxy_logging.db_spend_update_writer.update_database.await_count == 1 - assert sl_object.get("guardrail_information") == guardrail_info From 527c36343fd93ebaeffec0d4e07f6f9959f85da9 Mon Sep 17 00:00:00 2001 From: Yucheng Zhu Date: Tue, 1 Sep 2026 00:20:14 -0700 Subject: [PATCH 05/23] fix(guardrails): keep not_run entries out of daily evaluation counts --- litellm/proxy/guardrails/usage_tracking.py | 21 +++++++------ .../proxy/guardrails/test_usage_tracking.py | 31 +++++++++++++++++++ 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/guardrails/usage_tracking.py b/litellm/proxy/guardrails/usage_tracking.py index b8ae09afc00..ec059acb146 100644 --- a/litellm/proxy/guardrails/usage_tracking.py +++ b/litellm/proxy/guardrails/usage_tracking.py @@ -316,15 +316,18 @@ async def process_spend_logs_guardrail_usage( guardrail_id = entry.get("guardrail_id") or entry.get("guardrail_name") or "" if not guardrail_id: continue - key = _MetricsKey(guardrail_id, date_key) - daily_guardrail[key]["requests_evaluated"] += 1 - action = _guardrail_status_to_action(entry.get("guardrail_status")) - if action == "passed": - daily_guardrail[key]["passed_count"] += 1 - elif action == "blocked": - daily_guardrail[key]["blocked_count"] += 1 - else: - daily_guardrail[key]["flagged_count"] += 1 + status = entry.get("guardrail_status") + # not_run means the guardrail never evaluated the request: index it for drill-down, keep it out of counts + if status != "not_run": + key = _MetricsKey(guardrail_id, date_key) + daily_guardrail[key]["requests_evaluated"] += 1 + action = _guardrail_status_to_action(status) + if action == "passed": + daily_guardrail[key]["passed_count"] += 1 + elif action == "blocked": + daily_guardrail[key]["blocked_count"] += 1 + else: + daily_guardrail[key]["flagged_count"] += 1 policy_id = entry.get("policy_id") index_rows.append( { diff --git a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py index 6da121703d7..b98b037e7b8 100644 --- a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py +++ b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py @@ -306,6 +306,37 @@ async def test_zero_and_non_int_usage_counters_are_skipped(): } +@pytest.mark.asyncio +async def test_not_run_entries_are_indexed_but_not_counted_as_evaluations(): + """ + LIT-6314 records a not_run entry when message scoping leaves a guardrail + nothing to scan. The guardrail never evaluated the request, so counting it + as a passed evaluation would inflate daily pass rates; it still gets an + index row so per-request drill-down finds the spend log. + """ + prisma = _prisma() + logs = [_payload("r1", guardrail_status="not_run"), _payload("r2")] + + await process_spend_logs_guardrail_usage(prisma, logs) + + metrics_create = prisma.db.litellm_dailyguardrailmetrics.upsert.call_args.kwargs["data"]["create"] + assert metrics_create["requests_evaluated"] == 1 + assert metrics_create["passed_count"] == 1 + index_rows = prisma.db.litellm_spendlogguardrailindex.create_many.call_args.kwargs["data"] + assert sorted(row["request_id"] for row in index_rows) == ["r1", "r2"] + + +@pytest.mark.asyncio +async def test_batch_of_only_not_run_entries_writes_no_metrics_row(): + prisma = _prisma() + + await process_spend_logs_guardrail_usage(prisma, [_payload("r1", guardrail_status="not_run")]) + + assert prisma.db.litellm_dailyguardrailmetrics.upsert.call_count == 0 + index_rows = prisma.db.litellm_spendlogguardrailindex.create_many.call_args.kwargs["data"] + assert [row["request_id"] for row in index_rows] == ["r1"] + + @pytest.mark.asyncio async def test_payload_without_request_id_is_skipped_like_the_metrics_path(): prisma = _prisma() From 45ff658c6f020665bbe997ba6fc93ebf7c4eb2e3 Mon Sep 17 00:00:00 2001 From: Yucheng Zhu Date: Tue, 1 Sep 2026 01:09:01 -0700 Subject: [PATCH 06/23] fix(ui): render not_run guardrail evaluations as not run instead of failed --- ui/litellm-dashboard/eslint-suppressions.json | 2 +- .../GuardrailViewer/GuardrailViewer.test.tsx | 10 +++ .../GuardrailViewer/GuardrailViewer.tsx | 71 +++++++++++++------ 3 files changed, 60 insertions(+), 23 deletions(-) diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 7de7373b20b..93966c8d0ee 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -2352,7 +2352,7 @@ }, "src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx": { "no-nested-ternary": { - "count": 4 + "count": 3 } }, "src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx": { diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx index b5e04c72440..0fb5504d231 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx @@ -33,6 +33,16 @@ describe("GuardrailViewer", () => { expect(screen.getByText("1235ms")).toBeInTheDocument(); }); + it("renders not_run entries as not run instead of failed", () => { + const data = makeGuardrailInformation({ guardrail_status: "not_run", guardrail_mode: "pre_call" }); + renderWithProviders(); + + expect(screen.getByText(/1 Not run/)).toBeInTheDocument(); + // one NOT RUN badge in the timeline, one in the evaluation card + expect(screen.getAllByText("NOT RUN")).toHaveLength(2); + expect(screen.queryByText("FAILED")).not.toBeInTheDocument(); + }); + it("calculates and displays masked entity totals", async () => { const user = userEvent.setup(); const data = makeGuardrailInformation({ diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx index 863f4117510..08f9002b050 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx @@ -137,6 +137,36 @@ const isEntrySuccess = (entry: GuardrailInformation): boolean => { return (entry.guardrail_status ?? "").toLowerCase() === "success"; }; +const isEntryNotRun = (entry: GuardrailInformation): boolean => { + return (entry.guardrail_status ?? "").toLowerCase() === "not_run"; +}; + +type EntryStatusLabel = "PASSED" | "NOT RUN" | "FAILED"; + +const entryStatusLabel = (entry: GuardrailInformation): EntryStatusLabel => { + if (isEntrySuccess(entry)) return "PASSED"; + if (isEntryNotRun(entry)) return "NOT RUN"; + return "FAILED"; +}; + +const StatusIcon = ({ status }: { status: EntryStatusLabel }) => { + if (status === "PASSED") return ; + if (status === "NOT RUN") return ; + return ; +}; + +const timelineStatusClass = (status: EntryStatusLabel): string => { + if (status === "PASSED") return "bg-success/15 text-success"; + if (status === "NOT RUN") return "bg-muted text-muted-foreground"; + return "bg-destructive/15 text-destructive"; +}; + +const cardStatusClass = (status: EntryStatusLabel): string => { + if (status === "PASSED") return "bg-success/15 text-success border border-success/20"; + if (status === "NOT RUN") return "bg-muted text-muted-foreground border border-border"; + return "bg-destructive/15 text-destructive border border-destructive/20"; +}; + const getRiskColor = (score: number): string => { if (score <= 3) return "text-success bg-success/10 border-success/20"; if (score <= 6) return "text-warning bg-warning/10 border-warning/20"; @@ -318,8 +348,7 @@ interface TimelineEntry { type: "request" | "guardrail" | "llm" | "response"; label: string; offsetMs: number; - status?: string; - isSuccess?: boolean; + status?: EntryStatusLabel; } const RequestLifecycle = ({ entries }: { entries: GuardrailInformation[] }) => { @@ -348,8 +377,7 @@ const RequestLifecycle = ({ entries }: { entries: GuardrailInformation[] }) => { type: "guardrail", label: `Pre-call guardrail: ${getDisplayName(e)}`, offsetMs, - status: isEntrySuccess(e) ? "PASSED" : "FAILED", - isSuccess: isEntrySuccess(e), + status: entryStatusLabel(e), }); } @@ -372,8 +400,7 @@ const RequestLifecycle = ({ entries }: { entries: GuardrailInformation[] }) => { type: "guardrail", label: `During-call guardrail: ${getDisplayName(e)}`, offsetMs, - status: isEntrySuccess(e) ? "PASSED" : "FAILED", - isSuccess: isEntrySuccess(e), + status: entryStatusLabel(e), }); } @@ -384,8 +411,7 @@ const RequestLifecycle = ({ entries }: { entries: GuardrailInformation[] }) => { type: "guardrail", label: `Post-call guardrail: ${getDisplayName(e)}`, offsetMs, - status: isEntrySuccess(e) ? "PASSED" : "FAILED", - isSuccess: isEntrySuccess(e), + status: entryStatusLabel(e), }); } @@ -410,10 +436,8 @@ const RequestLifecycle = ({ entries }: { entries: GuardrailInformation[] }) => { ) : item.type === "llm" ? ( - ) : item.isSuccess ? ( - ) : ( - + )} {idx < timeline.length - 1 &&
} @@ -427,9 +451,7 @@ const RequestLifecycle = ({ entries }: { entries: GuardrailInformation[] }) => { {item.status && ( {item.status} @@ -456,6 +478,7 @@ const formatGuardrailCost = (cost: number): string => { const EvaluationCard = ({ entry }: { entry: GuardrailInformation }) => { const [expanded, setExpanded] = useState(false); const success = isEntrySuccess(entry); + const statusLabel = entryStatusLabel(entry); const totalMasked = getTotalMasked(entry); const displayName = getDisplayName(entry); const durationStr = formatDurationMs(entry.duration); @@ -490,7 +513,9 @@ const EvaluationCard = ({ entry }: { entry: GuardrailInformation }) => { onClick={() => setExpanded(!expanded)} > {/* Status icon */} -
{success ? : }
+
+ +
{/* Name + badges */}
@@ -501,13 +526,9 @@ const EvaluationCard = ({ entry }: { entry: GuardrailInformation }) => { - {success ? "PASSED" : "FAILED"} + {statusLabel} {matchCountStr && ( @@ -673,7 +694,8 @@ const GuardrailViewer = ({ data, accessToken, logEntry }: GuardrailViewerProps) }, [data]); const passedCount = guardrailEntries.filter(isEntrySuccess).length; - const allPassed = passedCount === guardrailEntries.length; + const notRunCount = guardrailEntries.filter(isEntryNotRun).length; + const allPassed = passedCount === guardrailEntries.length - notRunCount; const totalOverheadMs = useMemo(() => { return Math.round(guardrailEntries.reduce((sum, e) => sum + (e.duration ?? 0), 0) * 1000); @@ -728,6 +750,11 @@ const GuardrailViewer = ({ data, accessToken, logEntry }: GuardrailViewerProps) ) : null} {passedCount} Passed + {notRunCount > 0 && ( + + {notRunCount} Not run + + )}
From 26fc1ff221ba445aadf39fa10ead22d759761afe Mon Sep 17 00:00:00 2001 From: Yucheng Zhu Date: Tue, 1 Sep 2026 01:09:01 -0700 Subject: [PATCH 07/23] fix(guardrails): stop readers from scoring not_run evaluations as passed --- litellm/proxy/compliance_checks.py | 3 +- litellm/proxy/guardrails/usage_endpoints.py | 6 +- .../proxy/guardrails/test_usage_endpoints.py | 70 +++++++++++++++++-- .../test_compliance_endpoints.py | 36 +++++++++- .../GuardrailsMonitor/LogViewer.tsx | 11 ++- .../components/GuardrailsMonitor/mockData.ts | 2 +- .../LogDetailContent.test.tsx | 29 +++++++- .../LogDetailsDrawer/LogDetailContent.tsx | 4 +- 8 files changed, 147 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/compliance_checks.py b/litellm/proxy/compliance_checks.py index ff311911742..053c88d10ed 100644 --- a/litellm/proxy/compliance_checks.py +++ b/litellm/proxy/compliance_checks.py @@ -26,7 +26,8 @@ class ComplianceChecker: def __init__(self, data: ComplianceCheckRequest): self.data = data - self.guardrails = data.guardrail_information or [] + # a not_run entry records a guardrail that never evaluated the request, so it cannot evidence compliance + self.guardrails = [g for g in (data.guardrail_information or []) if g.get("guardrail_status") != "not_run"] def _get_guardrails_by_mode(self, mode: str) -> list[dict]: """ diff --git a/litellm/proxy/guardrails/usage_endpoints.py b/litellm/proxy/guardrails/usage_endpoints.py index 7a0edbddca8..75ffc1545dc 100644 --- a/litellm/proxy/guardrails/usage_endpoints.py +++ b/litellm/proxy/guardrails/usage_endpoints.py @@ -256,7 +256,7 @@ class UsageDetailResponse(BaseModel): class UsageLogEntry(BaseModel): id: str timestamp: str - action: str # blocked | passed | flagged + action: str # blocked | passed | flagged | not_run score: float | None latency_ms: float | None model: str | None @@ -691,7 +691,9 @@ def _usage_log_entry_from_row( reason_val = None if entry_for_guardrail: st: Final = (entry_for_guardrail.get("guardrail_status") or "").lower() - if "intervened" in st or "block" in st: + if st == "not_run": + action_val = "not_run" + elif "intervened" in st or "block" in st: action_val = "blocked" elif "fail" in st or "error" in st: action_val = "flagged" diff --git a/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py b/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py index 1665fa03639..e469aef9a61 100644 --- a/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py +++ b/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py @@ -9,12 +9,10 @@ orphans), and logs missed their logical-name alias. """ from datetime import datetime -from typing import Any, Optional +from typing import Any from unittest.mock import AsyncMock, MagicMock, patch import pytest - - from fastapi import HTTPException from prisma.errors import TableNotFoundError @@ -47,7 +45,7 @@ def _yaml_guardrail( guardrail_id: str = "yaml-1", name: str = "yaml-pii", provider: str = "presidio", - info: Optional[dict] = None, + info: dict | None = None, ) -> Guardrail: return Guardrail( guardrail_id=guardrail_id, @@ -433,3 +431,67 @@ async def test_detail_prev_trend_query_is_bounded(): prev_wheres = [w for w in wheres if "lt" in w.get("date", {})] assert prev_wheres assert all("gte" in w["date"] for w in prev_wheres) + + +@pytest.mark.asyncio +async def test_logs_report_not_run_entries_as_not_run_not_passed(): + """LIT-6314: a guardrail that never scanned must not be reported as a pass in the drill-down.""" + index_row = MagicMock() + index_row.request_id = "req-nr" + index_row.guardrail_id = "db-1" + index_row.start_time = datetime(2026, 4, 22) + spend_log = MagicMock() + spend_log.request_id = "req-nr" + spend_log.model = "gpt-4o-mini" + spend_log.startTime = datetime(2026, 4, 22) + spend_log.metadata = { + "guardrail_information": [ + {"guardrail_name": "db-1", "guardrail_status": "not_run", "duration": 0.0}, + ] + } + prisma = _prisma(find_unique=_db_row(), index_find_many=[index_row]) + prisma.db.litellm_spendlogs.find_many = AsyncMock(return_value=[spend_log]) + handler = _config_handler() + p1, p2 = _patches(prisma, handler) + with p1, p2: + resp = await guardrails_usage_logs( + guardrail_id="db-1", + policy_id=None, + page=1, + page_size=50, + action=None, + start_date=START, + end_date=END, + user_api_key_dict=ADMIN, + ) + assert [log.action for log in resp.logs] == ["not_run"] + + +@pytest.mark.asyncio +async def test_logs_action_passed_filter_excludes_not_run_entries(): + """LIT-6314: filtering the drill-down for passes must not return unscanned requests.""" + index_row = MagicMock() + index_row.request_id = "req-nr" + index_row.guardrail_id = "db-1" + index_row.start_time = datetime(2026, 4, 22) + spend_log = MagicMock() + spend_log.request_id = "req-nr" + spend_log.model = "gpt-4o-mini" + spend_log.startTime = datetime(2026, 4, 22) + spend_log.metadata = {"guardrail_information": [{"guardrail_name": "db-1", "guardrail_status": "not_run"}]} + prisma = _prisma(find_unique=_db_row(), index_find_many=[index_row]) + prisma.db.litellm_spendlogs.find_many = AsyncMock(return_value=[spend_log]) + handler = _config_handler() + p1, p2 = _patches(prisma, handler) + with p1, p2: + resp = await guardrails_usage_logs( + guardrail_id="db-1", + policy_id=None, + page=1, + page_size=50, + action="passed", + start_date=START, + end_date=END, + user_api_key_dict=ADMIN, + ) + assert resp.logs == [] diff --git a/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py index dcbe515d5de..8382a5ada96 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py @@ -2,10 +2,8 @@ Unit tests for compliance check endpoints (EU AI Act and GDPR). """ - import pytest - from litellm.proxy.compliance_checks import ComplianceChecker from litellm.types.proxy.compliance_endpoints import ComplianceCheckRequest @@ -591,3 +589,37 @@ class TestModeMatching: continue if matched: assert mode in _guaranteed_modes(g_mode), (g_mode, mode) + + +class TestNotRunGuardrails: + """LIT-6314 logs a not_run entry for a guardrail that message scoping left nothing to scan.""" + + def test_not_run_alone_never_evidences_compliance(self): + data = ComplianceCheckRequest( + request_id="req-601", + user_id="user-1", + model="gpt-4", + timestamp="2026-02-17T00:00:00Z", + guardrail_information=[ + {"guardrail_name": "pii_detection", "guardrail_status": "not_run", "guardrail_mode": "pre_call"}, + ], + ) + results = {c.check_name: c.passed for c in ComplianceChecker(data).check_eu_ai_act()} + assert results["Guardrails applied"] is False + assert results["Content screened before LLM"] is False + assert results["Audit record complete"] is False + + def test_not_run_sibling_does_not_fail_a_passing_request(self): + data = ComplianceCheckRequest( + request_id="req-602", + user_id="user-1", + model="gpt-4", + timestamp="2026-02-17T00:00:00Z", + pii_detected=True, + guardrail_information=[ + {"guardrail_name": "pii_detection", "guardrail_status": "success", "guardrail_mode": "pre_call"}, + {"guardrail_name": "system_only", "guardrail_status": "not_run", "guardrail_mode": "pre_call"}, + ], + ) + results = {c.check_name: c.passed for c in ComplianceChecker(data).check_gdpr()} + assert results["Sensitive data protected"] is True diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.tsx index 8d073feae82..8b4684e87b0 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.tsx +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.tsx @@ -1,4 +1,4 @@ -import { CircleCheck, ChevronDown, TriangleAlert, X } from "lucide-react"; +import { CircleCheck, ChevronDown, MinusCircle, TriangleAlert, X } from "lucide-react"; import { useQuery } from "@tanstack/react-query"; import moment from "moment"; import React, { useState } from "react"; @@ -10,9 +10,16 @@ import type { LogEntry as ViewLogsLogEntry } from "@/components/view_logs/column import type { LogEntry } from "./mockData"; const actionConfig: Record< - "blocked" | "passed" | "flagged", + "blocked" | "passed" | "flagged" | "not_run", { icon: React.ElementType; color: string; bg: string; border: string; label: string } > = { + not_run: { + icon: MinusCircle, + color: "text-muted-foreground", + bg: "bg-muted", + border: "border-border", + label: "Not run", + }, blocked: { icon: X, color: "text-destructive", diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts b/ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts index 7d99ebe7c44..717f0f459e4 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts @@ -43,7 +43,7 @@ export interface LogEntry { input_snippet?: string; output_snippet?: string; score?: number; - action: "blocked" | "passed" | "flagged"; + action: "blocked" | "passed" | "flagged" | "not_run"; model?: string; reason?: string; latency_ms?: number; diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx index a679dc49427..961aad7dc9e 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx @@ -1,7 +1,7 @@ import { render, screen, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { describe, expect, it, vi } from "vitest"; -import { LogDetailContent } from "./LogDetailContent"; +import { GuardrailJumpLink, LogDetailContent } from "./LogDetailContent"; import type { LogEntry } from "../columns"; vi.mock("../GuardrailViewer/GuardrailViewer", () => ({ @@ -489,3 +489,30 @@ describe("LogDetailContent", () => { expect(within(descriptions).getByText("-")).toBeInTheDocument(); }); }); + +describe("GuardrailJumpLink", () => { + it("does not render a not_run entry as a failure", () => { + render( + , + ); + expect(screen.getByText(/2 guardrails/)).toHaveTextContent("✓"); + expect(screen.getByText(/2 guardrails/)).not.toHaveTextContent("✗"); + }); + + it("still renders a real failure as failed", () => { + render( + , + ); + expect(screen.getByText(/2 guardrails/)).toHaveTextContent("✗"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx index 4c5c7b7b43f..105ae4add45 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx @@ -636,7 +636,9 @@ function RequestResponseSection({ } export function GuardrailJumpLink({ guardrailEntries }: { guardrailEntries: any[] }) { - const allPassed = guardrailEntries.every((e) => { + // a not_run entry never evaluated the request, so it neither passes nor fails the banner + const evaluated = guardrailEntries.filter((e) => (e?.guardrail_status || e?.status) !== "not_run"); + const allPassed = evaluated.every((e) => { const status = e?.guardrail_status || e?.status; return status === "pass" || status === "passed" || status === "success"; }); From e41faf54a82c3443d5dbad30ad9d519d57093895 Mon Sep 17 00:00:00 2001 From: yucheng Date: Mon, 14 Sep 2026 18:39:10 +0000 Subject: [PATCH 08/23] fix(ui): anchor guardrail lifecycle on timed entries and show not_run skip reason Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../GuardrailViewer/GuardrailViewer.test.tsx | 48 +++++++++++++++++-- .../GuardrailViewer/GuardrailViewer.tsx | 20 ++++++-- .../GuardrailViewer/__tests__/fixtures.ts | 8 ++-- 3 files changed, 62 insertions(+), 14 deletions(-) diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx index 8cc0d186631..629383d5807 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx @@ -49,17 +49,55 @@ describe("GuardrailViewer", () => { expect(screen.queryByText("FAILED")).not.toBeInTheDocument(); }); - it("renders not_run as NOT RUN (muted) and keeps it out of the evaluated and passed counts", () => { - const data = makeGuardrailInformation({ guardrail_status: "not_run", guardrail_mode: "pre_call" }); + it("renders not_run as NOT RUN (muted) and keeps it out of the evaluated and passed counts", async () => { + const user = userEvent.setup(); + const data = makeGuardrailInformation({ + guardrail_status: "not_run", + guardrail_mode: "pre_call", + guardrail_response: "no scannable content after message scoping", + start_time: null, + end_time: null, + duration: null, + }); renderWithProviders(); expect(screen.getByText(/0 guardrails evaluated/)).toBeInTheDocument(); expect(screen.getByText(/0 Passed/)).toHaveClass("text-muted-foreground"); expect(screen.getByText(/1 Not run/)).toBeInTheDocument(); - const badges = screen.getAllByText("NOT RUN"); - expect(badges).toHaveLength(2); - expect(badges[0]).toHaveClass("text-muted-foreground"); + const badge = screen.getByText("NOT RUN"); + expect(badge).toHaveClass("text-muted-foreground"); expect(screen.queryByText("FAILED")).not.toBeInTheDocument(); + expect(screen.queryByText(/^T\+/)).not.toBeInTheDocument(); + + await user.click(screen.getByText("pii-rail")); + expect(screen.getByText("no scannable content after message scoping")).toBeInTheDocument(); + }); + + it("anchors the lifecycle timeline on timed entries when an untimed not_run entry sorts first", () => { + const skipped = makeGuardrailInformation({ + guardrail_name: "skipped-rail", + guardrail_status: "not_run", + guardrail_mode: "pre_call", + start_time: null, + end_time: null, + duration: null, + }); + const ran = makeGuardrailInformation({ + guardrail_name: "ran-rail", + guardrail_status: "success", + guardrail_mode: "post_call", + start_time: 1_700_000_000, + end_time: 1_700_000_000.25, + duration: 0.25, + }); + renderWithProviders(); + + expect(screen.getByText(/1 guardrail evaluated/)).toBeInTheDocument(); + expect(screen.getByText("Request received").parentElement).toHaveTextContent("T+0ms"); + expect(screen.getByText(/Post-call guardrail: ran-rail/).parentElement).toHaveTextContent("T+250ms"); + expect(screen.getByText("Response returned").parentElement).toHaveTextContent("T+251ms"); + expect(screen.queryByText(/Pre-call guardrail: skipped-rail/)).not.toBeInTheDocument(); + expect(screen.getByText("—")).toBeInTheDocument(); }); it("calculates and displays masked entity totals", async () => { diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx index 7fb4124262b..1de0e3878b2 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx @@ -38,9 +38,9 @@ interface MatchDetail { } interface GuardrailInformation { - duration: number; - end_time: number; - start_time: number; + duration: number | null; + end_time: number | null; + start_time: number | null; guardrail_mode: string | string[] | Record | null; guardrail_name: string; guardrail_status: string; @@ -121,7 +121,8 @@ const formatMode = (mode: GuardrailInformation["guardrail_mode"]): string => { return s.replace(/_/g, "-").toUpperCase(); }; -const formatDurationMs = (seconds: number): string => { +const formatDurationMs = (seconds: number | null): string => { + if (seconds == null) return "—"; const ms = Math.round(seconds * 1000); return `${ms}ms`; }; @@ -364,8 +365,13 @@ interface TimelineEntry { outcome?: EntryOutcome; } +type TimedGuardrailInformation = GuardrailInformation & { start_time: number; end_time: number }; + +const isTimed = (e: GuardrailInformation): e is TimedGuardrailInformation => + typeof e.start_time === "number" && typeof e.end_time === "number"; + const RequestLifecycle = ({ entries }: { entries: GuardrailInformation[] }) => { - const sorted = useMemo(() => [...entries].sort((a, b) => (a.start_time ?? 0) - (b.start_time ?? 0)), [entries]); + const sorted = useMemo(() => entries.filter(isTimed).sort((a, b) => a.start_time - b.start_time), [entries]); const timeline = useMemo(() => { if (sorted.length === 0) return []; @@ -669,6 +675,10 @@ const EvaluationCard = ({ entry }: { entry: GuardrailInformation }) => { )} + {outcome === "not_run" && typeof guardrailResponse === "string" && ( +

{guardrailResponse}

+ )} + {/* Provider-specific details */} {guardrailProvider === "presidio" && presidioEntities.length > 0 && (
diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/__tests__/fixtures.ts b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/__tests__/fixtures.ts index fe27428283d..ab121adf6b4 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/__tests__/fixtures.ts +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/__tests__/fixtures.ts @@ -20,13 +20,13 @@ export interface GuardrailEntity { } export interface GuardrailInformation { - duration: number; - end_time: number; - start_time: number; + duration: number | null; + end_time: number | null; + start_time: number | null; guardrail_mode: string | string[] | Record | null; guardrail_name: string; guardrail_status: string; - guardrail_response: GuardrailEntity[] | BedrockGuardrailResponse; + guardrail_response: GuardrailEntity[] | BedrockGuardrailResponse | string; masked_entity_count: Record; guardrail_usage?: Record; guardrail_cost?: number; From b1a006ea66ffb209c237fe8ab4a4df70d9806877 Mon Sep 17 00:00:00 2001 From: yucheng Date: Mon, 14 Sep 2026 18:54:24 +0000 Subject: [PATCH 09/23] test(ui): hoist not_run guardrail fixtures out of inline call args Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../GuardrailViewer/GuardrailViewer.test.tsx | 46 +++++++++---------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx index 629383d5807..7f343211596 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx @@ -3,6 +3,7 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import userEvent from "@testing-library/user-event"; import { renderWithProviders, screen, waitFor } from "../../../../tests/test-utils"; import { + GuardrailInformation, makeBedrockResponse, makeEntity, makeGuardrailInformation, @@ -14,6 +15,24 @@ import GuardrailViewer from "@/components/view_logs/GuardrailViewer/GuardrailVie const PresidioPath = "@/components/view_logs/GuardrailViewer/PresidioDetectedEntities"; const BedrockPath = "@/components/view_logs/GuardrailViewer/BedrockGuardrailDetails"; +const skippedPreCall: Partial = { + guardrail_status: "not_run", + guardrail_mode: "pre_call", + guardrail_response: "no scannable content after message scoping", + start_time: null, + end_time: null, + duration: null, +}; + +const ranPostCall: Partial = { + guardrail_name: "ran-rail", + guardrail_status: "success", + guardrail_mode: "post_call", + start_time: 1_700_000_000, + end_time: 1_700_000_000.25, + duration: 0.25, +}; + describe("GuardrailViewer", () => { beforeEach(() => { vi.resetModules(); @@ -51,14 +70,7 @@ describe("GuardrailViewer", () => { it("renders not_run as NOT RUN (muted) and keeps it out of the evaluated and passed counts", async () => { const user = userEvent.setup(); - const data = makeGuardrailInformation({ - guardrail_status: "not_run", - guardrail_mode: "pre_call", - guardrail_response: "no scannable content after message scoping", - start_time: null, - end_time: null, - duration: null, - }); + const data = makeGuardrailInformation(skippedPreCall); renderWithProviders(); expect(screen.getByText(/0 guardrails evaluated/)).toBeInTheDocument(); @@ -74,22 +86,8 @@ describe("GuardrailViewer", () => { }); it("anchors the lifecycle timeline on timed entries when an untimed not_run entry sorts first", () => { - const skipped = makeGuardrailInformation({ - guardrail_name: "skipped-rail", - guardrail_status: "not_run", - guardrail_mode: "pre_call", - start_time: null, - end_time: null, - duration: null, - }); - const ran = makeGuardrailInformation({ - guardrail_name: "ran-rail", - guardrail_status: "success", - guardrail_mode: "post_call", - start_time: 1_700_000_000, - end_time: 1_700_000_000.25, - duration: 0.25, - }); + const skipped = makeGuardrailInformation({ ...skippedPreCall, guardrail_name: "skipped-rail" }); + const ran = makeGuardrailInformation(ranPostCall); renderWithProviders(); expect(screen.getByText(/1 guardrail evaluated/)).toBeInTheDocument(); From c3f52fe0d5cd646d2d9fd928830a17dd3ca9b46e Mon Sep 17 00:00:00 2001 From: yucheng Date: Mon, 14 Sep 2026 20:37:09 +0000 Subject: [PATCH 10/23] fix(guardrails): key not_run index rows by the sibling evaluation's guardrail_id A not_run entry from the base guardrail only carries guardrail_name, while the content filter's evaluated entry carries guardrail_id. Keyed apart, one request listed twice in the monitor for a logging_only guardrail (Not run and Passed). Resolve the id from a same-name sibling in the payload so the severity pick applies Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/guardrails/usage_tracking.py | 13 +++++++-- .../proxy/guardrails/test_usage_tracking.py | 28 +++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/guardrails/usage_tracking.py b/litellm/proxy/guardrails/usage_tracking.py index 6d72d94718f..3dfa4a96642 100644 --- a/litellm/proxy/guardrails/usage_tracking.py +++ b/litellm/proxy/guardrails/usage_tracking.py @@ -365,8 +365,17 @@ async def process_spend_logs_guardrail_usage( continue date_key = _date_str(start_time) - for entry in _parse_guardrail_info_from_payload(payload): - guardrail_id = entry.get("guardrail_id") or entry.get("guardrail_name") or "" + entries = _parse_guardrail_info_from_payload(payload) + ids_by_name = MappingProxyType( + { + e["guardrail_name"]: e["guardrail_id"] + for e in entries + if e.get("guardrail_id") and e.get("guardrail_name") + } + ) + for entry in entries: + guardrail_name = entry.get("guardrail_name") or "" + guardrail_id = entry.get("guardrail_id") or ids_by_name.get(guardrail_name) or guardrail_name if not guardrail_id: continue action = guardrail_status_to_action(entry.get("guardrail_status")) diff --git a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py index 0226e8b8e57..3aede9ba5e5 100644 --- a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py +++ b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py @@ -369,6 +369,34 @@ async def test_not_run_entries_are_indexed_but_not_counted_as_evaluations(): assert sorted(row["request_id"] for row in index_rows) == ["r1", "r2"] +@pytest.mark.asyncio +async def test_not_run_entry_shares_index_key_with_evaluated_sibling_of_same_name(): + """ + The not_run entry from the shared base guardrail carries only guardrail_name, + while the evaluated entry from the same guardrail (e.g. content filter on the + output of a logging_only run) carries its guardrail_id. Keying them differently + lists one request twice in the monitor, once as not_run and once as passed. + """ + prisma = _prisma() + payload = _payload("r1") + payload["metadata"] = json.dumps( + { + "guardrail_information": [ + {"guardrail_name": "cf", "guardrail_status": "not_run"}, + {"guardrail_name": "cf", "guardrail_id": "cf-uuid", "guardrail_status": "success"}, + {"guardrail_name": "other", "guardrail_status": "not_run"}, + ] + } + ) + + await process_spend_logs_guardrail_usage(prisma, [payload]) + + index_rows = prisma.db.litellm_spendlogguardrailindex.create_many.call_args.kwargs["data"] + assert sorted(row["guardrail_id"] for row in index_rows) == ["cf-uuid", "cf-uuid", "other"] + metrics_create = prisma.db.litellm_dailyguardrailmetrics.upsert.call_args.kwargs["data"]["create"] + assert (metrics_create["guardrail_id"], metrics_create["requests_evaluated"]) == ("cf-uuid", 1) + + @pytest.mark.asyncio async def test_batch_of_only_not_run_entries_writes_no_metrics_row(): prisma = _prisma() From 7ebb169a4d78f5af7e38a9b8c6dbeedce25f22a5 Mon Sep 17 00:00:00 2001 From: yucheng Date: Mon, 14 Sep 2026 20:45:47 +0000 Subject: [PATCH 11/23] fix(guardrails): coalesce usage index rows per request and guardrail, keeping policy linkage Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/guardrails/usage_tracking.py | 9 +++++---- .../proxy/guardrails/test_usage_tracking.py | 12 ++++++++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/guardrails/usage_tracking.py b/litellm/proxy/guardrails/usage_tracking.py index 3dfa4a96642..4c8222f52ab 100644 --- a/litellm/proxy/guardrails/usage_tracking.py +++ b/litellm/proxy/guardrails/usage_tracking.py @@ -356,7 +356,7 @@ async def process_spend_logs_guardrail_usage( "flagged_count": 0, } ) - index_rows: Final[list[dict[str, object]]] = [] + index_rows_by_key: Final[dict[tuple[object, object], dict[str, object]]] = {} for payload in logs_to_process: request_id = payload.get("request_id") @@ -389,14 +389,15 @@ async def process_spend_logs_guardrail_usage( else: daily_guardrail[key]["flagged_count"] += 1 policy_id = entry.get("policy_id") - index_rows.append( - { + prior = index_rows_by_key.get((request_id, guardrail_id)) + if prior is None or (prior["policy_id"] is None and policy_id is not None): + index_rows_by_key[(request_id, guardrail_id)] = { "request_id": request_id, "guardrail_id": guardrail_id, "policy_id": policy_id, "start_time": start_time, } - ) + index_rows: Final = tuple(index_rows_by_key.values()) async with pending.lock: pending_metrics: Final = pending.metrics diff --git a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py index 3aede9ba5e5..58479d3d740 100644 --- a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py +++ b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py @@ -383,7 +383,12 @@ async def test_not_run_entry_shares_index_key_with_evaluated_sibling_of_same_nam { "guardrail_information": [ {"guardrail_name": "cf", "guardrail_status": "not_run"}, - {"guardrail_name": "cf", "guardrail_id": "cf-uuid", "guardrail_status": "success"}, + { + "guardrail_name": "cf", + "guardrail_id": "cf-uuid", + "policy_id": "pol-1", + "guardrail_status": "success", + }, {"guardrail_name": "other", "guardrail_status": "not_run"}, ] } @@ -392,7 +397,10 @@ async def test_not_run_entry_shares_index_key_with_evaluated_sibling_of_same_nam await process_spend_logs_guardrail_usage(prisma, [payload]) index_rows = prisma.db.litellm_spendlogguardrailindex.create_many.call_args.kwargs["data"] - assert sorted(row["guardrail_id"] for row in index_rows) == ["cf-uuid", "cf-uuid", "other"] + assert sorted((row["guardrail_id"], row["policy_id"]) for row in index_rows) == [ + ("cf-uuid", "pol-1"), + ("other", None), + ] metrics_create = prisma.db.litellm_dailyguardrailmetrics.upsert.call_args.kwargs["data"]["create"] assert (metrics_create["guardrail_id"], metrics_create["requests_evaluated"]) == ("cf-uuid", 1) From 586d51f15e92128c18fe7024bec6277a74831cb3 Mon Sep 17 00:00:00 2001 From: yucheng Date: Mon, 14 Sep 2026 20:50:41 +0000 Subject: [PATCH 12/23] fix(guardrails): skip malformed guardrail entries instead of failing the usage batch Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/guardrails/usage_tracking.py | 11 +++++----- .../proxy/guardrails/test_usage_tracking.py | 21 +++++++++++++++++++ 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/guardrails/usage_tracking.py b/litellm/proxy/guardrails/usage_tracking.py index 4c8222f52ab..b2ba6ec9ec5 100644 --- a/litellm/proxy/guardrails/usage_tracking.py +++ b/litellm/proxy/guardrails/usage_tracking.py @@ -356,12 +356,12 @@ async def process_spend_logs_guardrail_usage( "flagged_count": 0, } ) - index_rows_by_key: Final[dict[tuple[object, object], dict[str, object]]] = {} + index_rows_by_key: Final[dict[tuple[str, str], dict[str, object]]] = {} for payload in logs_to_process: request_id = payload.get("request_id") start_time = _parse_payload_start_time(payload) - if not request_id or start_time is None: + if not isinstance(request_id, str) or not request_id or start_time is None: continue date_key = _date_str(start_time) @@ -370,13 +370,14 @@ async def process_spend_logs_guardrail_usage( { e["guardrail_name"]: e["guardrail_id"] for e in entries - if e.get("guardrail_id") and e.get("guardrail_name") + if e.get("guardrail_id") and isinstance(e.get("guardrail_name"), str) } ) for entry in entries: - guardrail_name = entry.get("guardrail_name") or "" + raw_name = entry.get("guardrail_name") + guardrail_name = raw_name if isinstance(raw_name, str) else "" guardrail_id = entry.get("guardrail_id") or ids_by_name.get(guardrail_name) or guardrail_name - if not guardrail_id: + if not isinstance(guardrail_id, str) or not guardrail_id: continue action = guardrail_status_to_action(entry.get("guardrail_status")) if action != "not_run": diff --git a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py index 58479d3d740..56a8bf2f0b0 100644 --- a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py +++ b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py @@ -405,6 +405,27 @@ async def test_not_run_entry_shares_index_key_with_evaluated_sibling_of_same_nam assert (metrics_create["guardrail_id"], metrics_create["requests_evaluated"]) == ("cf-uuid", 1) +@pytest.mark.asyncio +async def test_malformed_not_run_entry_does_not_drop_the_batch(): + prisma = _prisma() + payload = _payload("r1") + payload["metadata"] = json.dumps( + { + "guardrail_information": [ + {"guardrail_name": ["not", "a", "string"], "guardrail_status": "not_run"}, + {"guardrail_name": "cf", "guardrail_id": "cf-uuid", "guardrail_status": "success"}, + ] + } + ) + + await process_spend_logs_guardrail_usage(prisma, [payload]) + + index_rows = prisma.db.litellm_spendlogguardrailindex.create_many.call_args.kwargs["data"] + assert [row["guardrail_id"] for row in index_rows] == ["cf-uuid"] + metrics_create = prisma.db.litellm_dailyguardrailmetrics.upsert.call_args.kwargs["data"]["create"] + assert metrics_create["requests_evaluated"] == 1 + + @pytest.mark.asyncio async def test_batch_of_only_not_run_entries_writes_no_metrics_row(): prisma = _prisma() From 937179bde1699b2c758a31f8850181d22fcb2302 Mon Sep 17 00:00:00 2001 From: yucheng Date: Mon, 14 Sep 2026 20:53:11 +0000 Subject: [PATCH 13/23] fix(guardrails): never map empty guardrail names onto a sibling id Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/guardrails/usage_tracking.py | 2 +- tests/test_litellm/proxy/guardrails/test_usage_tracking.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/guardrails/usage_tracking.py b/litellm/proxy/guardrails/usage_tracking.py index b2ba6ec9ec5..7e11b69108b 100644 --- a/litellm/proxy/guardrails/usage_tracking.py +++ b/litellm/proxy/guardrails/usage_tracking.py @@ -370,7 +370,7 @@ async def process_spend_logs_guardrail_usage( { e["guardrail_name"]: e["guardrail_id"] for e in entries - if e.get("guardrail_id") and isinstance(e.get("guardrail_name"), str) + if e.get("guardrail_id") and isinstance(e.get("guardrail_name"), str) and e["guardrail_name"] } ) for entry in entries: diff --git a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py index 56a8bf2f0b0..13a53efcb27 100644 --- a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py +++ b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py @@ -412,8 +412,9 @@ async def test_malformed_not_run_entry_does_not_drop_the_batch(): payload["metadata"] = json.dumps( { "guardrail_information": [ - {"guardrail_name": ["not", "a", "string"], "guardrail_status": "not_run"}, - {"guardrail_name": "cf", "guardrail_id": "cf-uuid", "guardrail_status": "success"}, + {"guardrail_name": ["not", "a", "string"], "guardrail_status": "success"}, + {"guardrail_name": "", "guardrail_id": "cf-uuid", "guardrail_status": "success"}, + {"guardrail_status": "not_run"}, ] } ) @@ -423,7 +424,7 @@ async def test_malformed_not_run_entry_does_not_drop_the_batch(): index_rows = prisma.db.litellm_spendlogguardrailindex.create_many.call_args.kwargs["data"] assert [row["guardrail_id"] for row in index_rows] == ["cf-uuid"] metrics_create = prisma.db.litellm_dailyguardrailmetrics.upsert.call_args.kwargs["data"]["create"] - assert metrics_create["requests_evaluated"] == 1 + assert (metrics_create["guardrail_id"], metrics_create["requests_evaluated"]) == ("cf-uuid", 1) @pytest.mark.asyncio From 33fb6625ade5d3ee84d4138375272e35ce9d0133 Mon Sep 17 00:00:00 2001 From: yucheng Date: Mon, 14 Sep 2026 21:18:00 +0000 Subject: [PATCH 14/23] test(guardrails): cover nameless evaluated entries in the malformed usage batch Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/test_litellm/proxy/guardrails/test_usage_tracking.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py index 13a53efcb27..69ec098b840 100644 --- a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py +++ b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py @@ -414,7 +414,7 @@ async def test_malformed_not_run_entry_does_not_drop_the_batch(): "guardrail_information": [ {"guardrail_name": ["not", "a", "string"], "guardrail_status": "success"}, {"guardrail_name": "", "guardrail_id": "cf-uuid", "guardrail_status": "success"}, - {"guardrail_status": "not_run"}, + {"guardrail_status": "success"}, ] } ) From b37ce94075124b4429996cd52313d100fbb5212e Mon Sep 17 00:00:00 2001 From: yucheng Date: Mon, 14 Sep 2026 22:15:53 +0000 Subject: [PATCH 15/23] refactor(guardrails): rename scoped-out evaluation status from not_run to skipped The per-guardrail status a scoped-out evaluation records is now skipped, matching the skip_*_in_guardrail settings that cause it. Request-level rollup still maps it to not_run so the StandardLoggingPayload status contract is unchanged Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/litellm_logging.py | 1 + .../chat/guardrail_translation/handler.py | 2 +- litellm/proxy/compliance_checks.py | 2 +- litellm/proxy/guardrails/usage_endpoints.py | 4 ++-- litellm/proxy/guardrails/usage_tracking.py | 8 +++---- litellm/types/utils.py | 3 ++- .../test_litellm_logging.py | 12 ++++++++++ .../test_openai_guardrail_handler.py | 8 +++---- .../proxy/guardrails/test_usage_endpoints.py | 10 ++++---- .../proxy/guardrails/test_usage_tracking.py | 22 ++++++++--------- .../test_compliance_endpoints.py | 12 +++++----- .../GuardrailsMonitor/LogViewer.test.tsx | 8 +++---- .../GuardrailsMonitor/LogViewer.tsx | 6 ++--- .../components/GuardrailsMonitor/mockData.ts | 2 +- .../GuardrailViewer/GuardrailViewer.test.tsx | 10 ++++---- .../GuardrailViewer/GuardrailViewer.tsx | 24 +++++++++---------- .../LogDetailContent.integration.test.tsx | 14 +++++------ .../LogDetailsDrawer/LogDetailContent.tsx | 12 +++++----- 18 files changed, 87 insertions(+), 73 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 9ba9fd082f3..e88de4acd0e 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -6021,6 +6021,7 @@ def _get_status_fields( "failure": "guardrail_failed_to_respond", # legacy "guardrail_failed_to_respond": "guardrail_failed_to_respond", # direct "not_run": "not_run", + "skipped": "not_run", } # Set LLM API status diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 0f5096d0108..dca90e07421 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -214,7 +214,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): guardrail_to_apply.add_standard_logging_guardrail_information_to_request_data( guardrail_json_response="no scannable content after message scoping", request_data=data, - guardrail_status="not_run", + guardrail_status="skipped", ) verbose_proxy_logger.debug( diff --git a/litellm/proxy/compliance_checks.py b/litellm/proxy/compliance_checks.py index d9cc1d0f4fc..ef2d8fb6e20 100644 --- a/litellm/proxy/compliance_checks.py +++ b/litellm/proxy/compliance_checks.py @@ -26,7 +26,7 @@ class ComplianceChecker: def __init__(self, data: ComplianceCheckRequest): self.data = data - self.guardrails = [g for g in (data.guardrail_information or []) if g.get("guardrail_status") != "not_run"] + self.guardrails = [g for g in (data.guardrail_information or []) if g.get("guardrail_status") != "skipped"] def _get_guardrails_by_mode(self, mode: str) -> list[dict]: """ diff --git a/litellm/proxy/guardrails/usage_endpoints.py b/litellm/proxy/guardrails/usage_endpoints.py index 556b6a4e919..651c4bb1963 100644 --- a/litellm/proxy/guardrails/usage_endpoints.py +++ b/litellm/proxy/guardrails/usage_endpoints.py @@ -42,7 +42,7 @@ if TYPE_CHECKING: router: Final = APIRouter() _EMPTY_UNITS: Final[Mapping[str, int]] = MappingProxyType({}) -_ACTION_SEVERITY: Final[Mapping[str, int]] = MappingProxyType({"not_run": 0, "passed": 1, "flagged": 2, "blocked": 3}) +_ACTION_SEVERITY: Final[Mapping[str, int]] = MappingProxyType({"skipped": 0, "passed": 1, "flagged": 2, "blocked": 3}) _T = TypeVar("_T") @@ -325,7 +325,7 @@ class UsageDetailResponse(BaseModel): class UsageLogEntry(BaseModel): id: str timestamp: str - action: str # blocked | passed | flagged | not_run + action: str # blocked | passed | flagged | skipped score: float | None latency_ms: float | None model: str | None diff --git a/litellm/proxy/guardrails/usage_tracking.py b/litellm/proxy/guardrails/usage_tracking.py index 7e11b69108b..8a131fbfcde 100644 --- a/litellm/proxy/guardrails/usage_tracking.py +++ b/litellm/proxy/guardrails/usage_tracking.py @@ -193,12 +193,12 @@ async def _upsert_rows_with_retry( def guardrail_status_to_action(status: str | None) -> str: - """Map StandardLogging guardrail_status to blocked/passed/flagged/not_run.""" + """Map StandardLogging guardrail_status to blocked/passed/flagged/skipped.""" if not status: return "passed" s: Final = (status or "").lower() - if s == "not_run": - return "not_run" + if s == "skipped": + return "skipped" if "intervened" in s or "block" in s: return "blocked" if "flagged" in s or "fail" in s or "error" in s: @@ -380,7 +380,7 @@ async def process_spend_logs_guardrail_usage( if not isinstance(guardrail_id, str) or not guardrail_id: continue action = guardrail_status_to_action(entry.get("guardrail_status")) - if action != "not_run": + if action != "skipped": key = _MetricsKey(guardrail_id, date_key) daily_guardrail[key]["requests_evaluated"] += 1 if action == "passed": diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 1d73542c9bb..cf302b3f27f 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3120,7 +3120,7 @@ class GuardrailMode(TypedDict, total=False): GuardrailStatus = Literal[ - "success", "guardrail_flagged", "guardrail_intervened", "guardrail_failed_to_respond", "not_run" + "success", "guardrail_flagged", "guardrail_intervened", "guardrail_failed_to_respond", "not_run", "skipped" ] # Fields on a guardrail record whose values can quote the caller's prompt: the payload sent to the @@ -3367,6 +3367,7 @@ class StandardLoggingPayloadStatusFields(TypedDict, total=False): - 'guardrail_intervened': Guardrail blocked or modified content - 'guardrail_failed_to_respond': Guardrail had technical failure - 'not_run': No guardrail was run + - 'skipped': Only used per guardrail entry, message scoping left the guardrail nothing to scan """ diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 70f9bae283b..6e67781944e 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -6926,6 +6926,18 @@ def test_get_status_fields_ranks_guardrail_flagged_between_success_and_intervene )["guardrail_status"] == "guardrail_intervened" +def test_get_status_fields_rolls_skipped_entries_up_to_not_run(): + """LIT-6314: a guardrail that message scoping left nothing to scan records a + skipped entry. At request level that means no guardrail ran, and a skipped + entry must never outrank a sibling that did evaluate.""" + skipped = {"guardrail_status": "skipped"} + + assert _get_status_fields("success", [skipped], None)["guardrail_status"] == "not_run" + assert _get_status_fields( + "success", [skipped, {"guardrail_status": "success"}], None + )["guardrail_status"] == "success" + + def test_get_error_information_redacts_provider_key_from_upstream_url(): """A pass-through upstream failure logs the httpx traceback, whose message quotes the upstream URL with the provider key in its query string. That diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 4f1163ed806..c6d01db45de 100644 --- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1893,7 +1893,7 @@ class TestScanOnlyToolResults: assert data["messages"][4]["content"] == "and then?" -class TestNoScannableContentRecordsNotRun: +class TestNoScannableContentRecordsSkipped: """LIT-6314: a guardrail whose scoping leaves nothing to scan must still persist an evaluation record""" def _system_only_data(self) -> dict: @@ -1904,7 +1904,7 @@ class TestNoScannableContentRecordsNotRun: return metadata.get("standard_logging_guardrail_information") or [] @pytest.mark.asyncio - async def test_skipped_scan_records_not_run_entry(self): + async def test_skipped_scan_records_skipped_entry(self): handler = OpenAIChatCompletionsHandler() guardrail = MockGuardrail(guardrail_name="skip-system-guardrail") guardrail.skip_system_message_in_guardrail = True @@ -1916,7 +1916,7 @@ class TestNoScannableContentRecordsNotRun: entries = self._recorded_entries(data) assert len(entries) == 1 assert entries[0]["guardrail_name"] == "skip-system-guardrail" - assert entries[0]["guardrail_status"] == "not_run" + assert entries[0]["guardrail_status"] == "skipped" @pytest.mark.asyncio async def test_self_recording_guardrail_is_left_alone(self): @@ -1940,7 +1940,7 @@ class TestNoScannableContentRecordsNotRun: await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) assert guardrail.last_inputs is not None - assert all(e.get("guardrail_status") != "not_run" for e in self._recorded_entries(data)) + assert all(e.get("guardrail_status") != "skipped" for e in self._recorded_entries(data)) class TestBuildBlockSseChunks: diff --git a/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py b/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py index db87e12ac88..1df0c1477e2 100644 --- a/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py +++ b/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py @@ -685,7 +685,7 @@ async def test_detail_prev_trend_query_is_bounded(): @pytest.mark.asyncio -async def test_logs_report_not_run_entries_as_not_run_not_passed(): +async def test_logs_report_skipped_entries_as_skipped_not_passed(): """LIT-6314: a guardrail that never scanned must not be reported as a pass in the drill-down.""" index_row = MagicMock() index_row.request_id = "req-nr" @@ -697,7 +697,7 @@ async def test_logs_report_not_run_entries_as_not_run_not_passed(): spend_log.startTime = datetime(2026, 4, 22) spend_log.metadata = { "guardrail_information": [ - {"guardrail_name": "db-1", "guardrail_status": "not_run", "duration": 0.0}, + {"guardrail_name": "db-1", "guardrail_status": "skipped", "duration": 0.0}, ] } prisma = _prisma(find_unique=_db_row(), index_find_many=[index_row]) @@ -715,11 +715,11 @@ async def test_logs_report_not_run_entries_as_not_run_not_passed(): end_date=END, user_api_key_dict=ADMIN, ) - assert [log.action for log in resp.logs] == ["not_run"] + assert [log.action for log in resp.logs] == ["skipped"] @pytest.mark.asyncio -async def test_logs_action_passed_filter_excludes_not_run_entries(): +async def test_logs_action_passed_filter_excludes_skipped_entries(): """LIT-6314: filtering the drill-down for passes must not return unscanned requests.""" index_row = MagicMock() index_row.request_id = "req-nr" @@ -729,7 +729,7 @@ async def test_logs_action_passed_filter_excludes_not_run_entries(): spend_log.request_id = "req-nr" spend_log.model = "gpt-4o-mini" spend_log.startTime = datetime(2026, 4, 22) - spend_log.metadata = {"guardrail_information": [{"guardrail_name": "db-1", "guardrail_status": "not_run"}]} + spend_log.metadata = {"guardrail_information": [{"guardrail_name": "db-1", "guardrail_status": "skipped"}]} prisma = _prisma(find_unique=_db_row(), index_find_many=[index_row]) prisma.db.litellm_spendlogs.find_many = AsyncMock(return_value=[spend_log]) handler = _config_handler() diff --git a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py index 69ec098b840..cb85eb8b5ec 100644 --- a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py +++ b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py @@ -350,15 +350,15 @@ async def test_zero_and_non_int_usage_counters_are_skipped(): @pytest.mark.asyncio -async def test_not_run_entries_are_indexed_but_not_counted_as_evaluations(): +async def test_skipped_entries_are_indexed_but_not_counted_as_evaluations(): """ - LIT-6314 records a not_run entry when message scoping leaves a guardrail + LIT-6314 records a skipped entry when message scoping leaves a guardrail nothing to scan. The guardrail never evaluated the request, so counting it as a passed evaluation would inflate daily pass rates; it still gets an index row so per-request drill-down finds the spend log. """ prisma = _prisma() - logs = [_payload("r1", guardrail_status="not_run"), _payload("r2")] + logs = [_payload("r1", guardrail_status="skipped"), _payload("r2")] await process_spend_logs_guardrail_usage(prisma, logs) @@ -370,26 +370,26 @@ async def test_not_run_entries_are_indexed_but_not_counted_as_evaluations(): @pytest.mark.asyncio -async def test_not_run_entry_shares_index_key_with_evaluated_sibling_of_same_name(): +async def test_skipped_entry_shares_index_key_with_evaluated_sibling_of_same_name(): """ - The not_run entry from the shared base guardrail carries only guardrail_name, + The skipped entry from the shared base guardrail carries only guardrail_name, while the evaluated entry from the same guardrail (e.g. content filter on the output of a logging_only run) carries its guardrail_id. Keying them differently - lists one request twice in the monitor, once as not_run and once as passed. + lists one request twice in the monitor, once as skipped and once as passed. """ prisma = _prisma() payload = _payload("r1") payload["metadata"] = json.dumps( { "guardrail_information": [ - {"guardrail_name": "cf", "guardrail_status": "not_run"}, + {"guardrail_name": "cf", "guardrail_status": "skipped"}, { "guardrail_name": "cf", "guardrail_id": "cf-uuid", "policy_id": "pol-1", "guardrail_status": "success", }, - {"guardrail_name": "other", "guardrail_status": "not_run"}, + {"guardrail_name": "other", "guardrail_status": "skipped"}, ] } ) @@ -406,7 +406,7 @@ async def test_not_run_entry_shares_index_key_with_evaluated_sibling_of_same_nam @pytest.mark.asyncio -async def test_malformed_not_run_entry_does_not_drop_the_batch(): +async def test_malformed_skipped_entry_does_not_drop_the_batch(): prisma = _prisma() payload = _payload("r1") payload["metadata"] = json.dumps( @@ -428,10 +428,10 @@ async def test_malformed_not_run_entry_does_not_drop_the_batch(): @pytest.mark.asyncio -async def test_batch_of_only_not_run_entries_writes_no_metrics_row(): +async def test_batch_of_only_skipped_entries_writes_no_metrics_row(): prisma = _prisma() - await process_spend_logs_guardrail_usage(prisma, [_payload("r1", guardrail_status="not_run")]) + await process_spend_logs_guardrail_usage(prisma, [_payload("r1", guardrail_status="skipped")]) assert prisma.db.litellm_dailyguardrailmetrics.upsert.call_count == 0 index_rows = prisma.db.litellm_spendlogguardrailindex.create_many.call_args.kwargs["data"] diff --git a/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py index 8382a5ada96..9587e3d95ee 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py @@ -591,17 +591,17 @@ class TestModeMatching: assert mode in _guaranteed_modes(g_mode), (g_mode, mode) -class TestNotRunGuardrails: - """LIT-6314 logs a not_run entry for a guardrail that message scoping left nothing to scan.""" +class TestSkippedGuardrails: + """LIT-6314 logs a skipped entry for a guardrail that message scoping left nothing to scan.""" - def test_not_run_alone_never_evidences_compliance(self): + def test_skipped_alone_never_evidences_compliance(self): data = ComplianceCheckRequest( request_id="req-601", user_id="user-1", model="gpt-4", timestamp="2026-02-17T00:00:00Z", guardrail_information=[ - {"guardrail_name": "pii_detection", "guardrail_status": "not_run", "guardrail_mode": "pre_call"}, + {"guardrail_name": "pii_detection", "guardrail_status": "skipped", "guardrail_mode": "pre_call"}, ], ) results = {c.check_name: c.passed for c in ComplianceChecker(data).check_eu_ai_act()} @@ -609,7 +609,7 @@ class TestNotRunGuardrails: assert results["Content screened before LLM"] is False assert results["Audit record complete"] is False - def test_not_run_sibling_does_not_fail_a_passing_request(self): + def test_skipped_sibling_does_not_fail_a_passing_request(self): data = ComplianceCheckRequest( request_id="req-602", user_id="user-1", @@ -618,7 +618,7 @@ class TestNotRunGuardrails: pii_detected=True, guardrail_information=[ {"guardrail_name": "pii_detection", "guardrail_status": "success", "guardrail_mode": "pre_call"}, - {"guardrail_name": "system_only", "guardrail_status": "not_run", "guardrail_mode": "pre_call"}, + {"guardrail_name": "system_only", "guardrail_status": "skipped", "guardrail_mode": "pre_call"}, ], ) results = {c.check_name: c.passed for c in ComplianceChecker(data).check_gdpr()} diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.test.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.test.tsx index 083b1e5f3e2..af5284830dc 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.test.tsx +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.test.tsx @@ -96,14 +96,14 @@ describe("GuardrailsMonitor LogViewer drawer", () => { }); }); -describe("GuardrailsMonitor LogViewer not_run rows", () => { - it("renders a not_run log as a neutral Not run badge instead of a pass or failure", () => { +describe("GuardrailsMonitor LogViewer skipped rows", () => { + it("renders a skipped log as a neutral Skipped badge instead of a pass or failure", () => { renderWithProviders( - , + , ); const row = screen.getByRole("button", { name: /system prompt only/ }); - expect(within(row).getByText("Not run")).toHaveClass("text-muted-foreground"); + expect(within(row).getByText("Skipped")).toHaveClass("text-muted-foreground"); expect(within(row).queryByText("Passed")).not.toBeInTheDocument(); expect(within(row).queryByText("Blocked")).not.toBeInTheDocument(); }); diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.tsx index 2abd699ba86..b113c49e0a3 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.tsx +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.tsx @@ -10,15 +10,15 @@ import type { LogEntry as ViewLogsLogEntry } from "@/components/view_logs/column import type { LogEntry } from "./mockData"; const actionConfig: Record< - "blocked" | "passed" | "flagged" | "not_run", + "blocked" | "passed" | "flagged" | "skipped", { icon: React.ElementType; color: string; bg: string; border: string; label: string } > = { - not_run: { + skipped: { icon: MinusCircle, color: "text-muted-foreground", bg: "bg-muted", border: "border-border", - label: "Not run", + label: "Skipped", }, blocked: { icon: X, diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts b/ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts index 591d5cd3edd..7053efcf88d 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts @@ -10,7 +10,7 @@ export interface LogEntry { input_snippet?: string; output_snippet?: string; score?: number; - action: "blocked" | "passed" | "flagged" | "not_run"; + action: "blocked" | "passed" | "flagged" | "skipped"; model?: string; reason?: string; latency_ms?: number; diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx index 7f343211596..740ef857690 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx @@ -16,7 +16,7 @@ const PresidioPath = "@/components/view_logs/GuardrailViewer/PresidioDetectedEnt const BedrockPath = "@/components/view_logs/GuardrailViewer/BedrockGuardrailDetails"; const skippedPreCall: Partial = { - guardrail_status: "not_run", + guardrail_status: "skipped", guardrail_mode: "pre_call", guardrail_response: "no scannable content after message scoping", start_time: null, @@ -68,15 +68,15 @@ describe("GuardrailViewer", () => { expect(screen.queryByText("FAILED")).not.toBeInTheDocument(); }); - it("renders not_run as NOT RUN (muted) and keeps it out of the evaluated and passed counts", async () => { + it("renders skipped as SKIPPED (muted) and keeps it out of the evaluated and passed counts", async () => { const user = userEvent.setup(); const data = makeGuardrailInformation(skippedPreCall); renderWithProviders(); expect(screen.getByText(/0 guardrails evaluated/)).toBeInTheDocument(); expect(screen.getByText(/0 Passed/)).toHaveClass("text-muted-foreground"); - expect(screen.getByText(/1 Not run/)).toBeInTheDocument(); - const badge = screen.getByText("NOT RUN"); + expect(screen.getByText(/1 Skipped/)).toBeInTheDocument(); + const badge = screen.getByText("SKIPPED"); expect(badge).toHaveClass("text-muted-foreground"); expect(screen.queryByText("FAILED")).not.toBeInTheDocument(); expect(screen.queryByText(/^T\+/)).not.toBeInTheDocument(); @@ -85,7 +85,7 @@ describe("GuardrailViewer", () => { expect(screen.getByText("no scannable content after message scoping")).toBeInTheDocument(); }); - it("anchors the lifecycle timeline on timed entries when an untimed not_run entry sorts first", () => { + it("anchors the lifecycle timeline on timed entries when an untimed skipped entry sorts first", () => { const skipped = makeGuardrailInformation({ ...skippedPreCall, guardrail_name: "skipped-rail" }); const ran = makeGuardrailInformation(ranPostCall); renderWithProviders(); diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx index 1de0e3878b2..c67d682a233 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx @@ -134,13 +134,13 @@ const getTotalMasked = (entry: GuardrailInformation): number => { ); }; -type EntryOutcome = "passed" | "flagged" | "failed" | "not_run"; +type EntryOutcome = "passed" | "flagged" | "failed" | "skipped"; const getEntryOutcome = (entry: GuardrailInformation): EntryOutcome => { const status = (entry.guardrail_status ?? "").toLowerCase(); if (status === "success") return "passed"; if (status === "guardrail_flagged") return "flagged"; - if (status === "not_run") return "not_run"; + if (status === "skipped") return "skipped"; return "failed"; }; @@ -150,18 +150,18 @@ const OUTCOME_LABEL: Record = { passed: "PASSED", flagged: "FLAGGED", failed: "FAILED", - not_run: "NOT RUN", + skipped: "SKIPPED", }; const OUTCOME_BADGE_CLASS: Record = { passed: "bg-success/15 text-success border border-success/20", flagged: "bg-warning/15 text-warning border border-warning/20", failed: "bg-destructive/15 text-destructive border border-destructive/20", - not_run: "bg-muted text-muted-foreground border border-border", + skipped: "bg-muted text-muted-foreground border border-border", }; const getHeaderOutcome = (counts: { evaluated: number; passed: number; flagged: number }): EntryOutcome => { - if (counts.evaluated === 0) return "not_run"; + if (counts.evaluated === 0) return "skipped"; if (counts.passed === counts.evaluated) return "passed"; if (counts.passed + counts.flagged === counts.evaluated) return "flagged"; return "failed"; @@ -242,7 +242,7 @@ const FlagCircleIcon = ({ className }: { className?: string }) => ( const OutcomeIcon = ({ outcome }: { outcome: EntryOutcome }) => { if (outcome === "passed") return ; if (outcome === "flagged") return ; - if (outcome === "not_run") return ; + if (outcome === "skipped") return ; return ; }; @@ -675,7 +675,7 @@ const EvaluationCard = ({ entry }: { entry: GuardrailInformation }) => {
)} - {outcome === "not_run" && typeof guardrailResponse === "string" && ( + {outcome === "skipped" && typeof guardrailResponse === "string" && (

{guardrailResponse}

)} @@ -717,8 +717,8 @@ const GuardrailViewer = ({ data, accessToken, logEntry }: GuardrailViewerProps) const passedCount = guardrailEntries.filter(isEntrySuccess).length; const flaggedCount = guardrailEntries.filter((e) => getEntryOutcome(e) === "flagged").length; - const notRunCount = guardrailEntries.filter((e) => getEntryOutcome(e) === "not_run").length; - const evaluatedCount = guardrailEntries.length - notRunCount; + const skippedCount = guardrailEntries.filter((e) => getEntryOutcome(e) === "skipped").length; + const evaluatedCount = guardrailEntries.length - skippedCount; const allPassed = evaluatedCount > 0 && passedCount === evaluatedCount; const headerOutcome = getHeaderOutcome({ evaluated: evaluatedCount, passed: passedCount, flagged: flaggedCount }); @@ -778,11 +778,11 @@ const GuardrailViewer = ({ data, accessToken, logEntry }: GuardrailViewerProps) {flaggedCount} Flagged )} - {notRunCount > 0 && ( + {skippedCount > 0 && ( - {notRunCount} Not run + {skippedCount} Skipped )} diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.integration.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.integration.test.tsx index 3637b6c55a3..cbc377c1b73 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.integration.test.tsx @@ -637,20 +637,20 @@ describe("GuardrailJumpLink", () => { }); it.each([ - [["success", "not_run"], "text-success", "\u2713"], - [["guardrail_intervened", "not_run"], "text-destructive", "\u2717"], - ])("ignores not_run when styling %j as %s", (statuses, expectedClass, glyph) => { + [["success", "skipped"], "text-success", "\u2713"], + [["guardrail_intervened", "skipped"], "text-destructive", "\u2717"], + ])("ignores skipped when styling %j as %s", (statuses, expectedClass, glyph) => { render( ({ guardrail_status: s }))} />); - const pill = screen.getByText(/1 guardrail evaluated, 1 not run/); + const pill = screen.getByText(/1 guardrail evaluated, 1 skipped/); expect(pill).toHaveClass(expectedClass); expect(pill).toHaveTextContent(glyph); }); - it("renders an all not_run request as neutral rather than passed", () => { - render(); + it("renders an all skipped request as neutral rather than passed", () => { + render(); - const pill = screen.getByText(/0 guardrails evaluated, 1 not run/); + const pill = screen.getByText(/0 guardrails evaluated, 1 skipped/); expect(pill).toHaveClass("text-muted-foreground"); expect(pill).not.toHaveTextContent("\u2713"); }); diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx index 30cd96935f7..e052faa2228 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx @@ -700,15 +700,15 @@ const GUARDRAIL_JUMP_LINK_STYLE = { passed: { className: "border border-success/20 bg-success/10 text-success", glyph: "\u2713" }, flagged: { className: "border border-warning/20 bg-warning/10 text-warning", glyph: "\u26A0" }, failed: { className: "border border-destructive/20 bg-destructive/10 text-destructive", glyph: "\u2717" }, - not_run: { className: "border border-border bg-muted text-muted-foreground", glyph: "\u2013" }, + skipped: { className: "border border-border bg-muted text-muted-foreground", glyph: "\u2013" }, } as const; const isPassedStatus = (status: unknown) => status === "pass" || status === "passed" || status === "success"; const isFlaggedStatus = (status: unknown) => status === "flagged" || status === "guardrail_flagged"; -const isNotRunStatus = (status: unknown) => status === "not_run"; +const isSkippedStatus = (status: unknown) => status === "skipped"; const guardrailJumpLinkOutcome = (evaluated: unknown[]): keyof typeof GUARDRAIL_JUMP_LINK_STYLE => { - if (evaluated.length === 0) return "not_run"; + if (evaluated.length === 0) return "skipped"; if (evaluated.every(isPassedStatus)) return "passed"; if (evaluated.every((s) => isPassedStatus(s) || isFlaggedStatus(s))) return "flagged"; return "failed"; @@ -716,8 +716,8 @@ const guardrailJumpLinkOutcome = (evaluated: unknown[]): keyof typeof GUARDRAIL_ export function GuardrailJumpLink({ guardrailEntries }: { guardrailEntries: any[] }) { const statuses = guardrailEntries.map((e) => e?.guardrail_status || e?.status); - const evaluated = statuses.filter((s) => !isNotRunStatus(s)); - const notRunCount = statuses.length - evaluated.length; + const evaluated = statuses.filter((s) => !isSkippedStatus(s)); + const skippedCount = statuses.length - evaluated.length; const { className, glyph } = GUARDRAIL_JUMP_LINK_STYLE[guardrailJumpLinkOutcome(evaluated)]; const handleClick = () => { @@ -743,7 +743,7 @@ export function GuardrailJumpLink({ guardrailEntries }: { guardrailEntries: any[ > {glyph} {evaluated.length} guardrail {evaluated.length !== 1 ? "s" : ""} evaluated - {notRunCount > 0 ? `, ${notRunCount} not run` : ""} + {skippedCount > 0 ? `, ${skippedCount} skipped` : ""} {"\u2193"} From f78dd921c958c38c3adba9ee490071a658c79e27 Mon Sep 17 00:00:00 2001 From: yucheng Date: Mon, 14 Sep 2026 22:34:33 +0000 Subject: [PATCH 16/23] fix(guardrails): keep legacy not_run neutral and stop labelling image-only input as skipped Usage tracking, compliance and the dashboard now treat both not_run (older spend logs) and skipped as unevaluated through a shared UNEVALUATED_GUARDRAIL_STATUSES set, so old records stop counting as passed. The skipped record is no longer written when the request carried images, since images without text were never dispatched to guardrails before this change and that gap is not a message-scoping skip Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../chat/guardrail_translation/handler.py | 2 +- litellm/proxy/compliance_checks.py | 7 ++++- litellm/proxy/guardrails/usage_tracking.py | 3 +- litellm/types/utils.py | 2 ++ .../test_openai_guardrail_handler.py | 21 +++++++++++++ .../proxy/guardrails/test_usage_tracking.py | 12 ++++--- .../test_compliance_endpoints.py | 5 +-- .../GuardrailViewer/GuardrailViewer.test.tsx | 31 ++++++++++--------- .../GuardrailViewer/GuardrailViewer.tsx | 2 +- .../LogDetailsDrawer/LogDetailContent.tsx | 2 +- 10 files changed, 61 insertions(+), 26 deletions(-) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index dca90e07421..035dce46d27 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -210,7 +210,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): task_mappings=tool_call_task_mappings, ) - elif not guardrail_to_apply.records_own_guardrail_information: + elif not images_to_check and not guardrail_to_apply.records_own_guardrail_information: guardrail_to_apply.add_standard_logging_guardrail_information_to_request_data( guardrail_json_response="no scannable content after message scoping", request_data=data, diff --git a/litellm/proxy/compliance_checks.py b/litellm/proxy/compliance_checks.py index ef2d8fb6e20..18123156411 100644 --- a/litellm/proxy/compliance_checks.py +++ b/litellm/proxy/compliance_checks.py @@ -11,6 +11,7 @@ from litellm.types.proxy.compliance_endpoints import ( ComplianceCheckRequest, ComplianceCheckResult, ) +from litellm.types.utils import UNEVALUATED_GUARDRAIL_STATUSES class ComplianceChecker: @@ -26,7 +27,11 @@ class ComplianceChecker: def __init__(self, data: ComplianceCheckRequest): self.data = data - self.guardrails = [g for g in (data.guardrail_information or []) if g.get("guardrail_status") != "skipped"] + self.guardrails = [ + g + for g in (data.guardrail_information or []) + if g.get("guardrail_status") not in UNEVALUATED_GUARDRAIL_STATUSES + ] def _get_guardrails_by_mode(self, mode: str) -> list[dict]: """ diff --git a/litellm/proxy/guardrails/usage_tracking.py b/litellm/proxy/guardrails/usage_tracking.py index 8a131fbfcde..789b8febfbf 100644 --- a/litellm/proxy/guardrails/usage_tracking.py +++ b/litellm/proxy/guardrails/usage_tracking.py @@ -27,6 +27,7 @@ from litellm.repositories.table_repositories import ( DailyGuardrailUsageUnitsRepository, SpendLogGuardrailIndexRepository, ) +from litellm.types.utils import UNEVALUATED_GUARDRAIL_STATUSES if TYPE_CHECKING: from prisma import types as prisma_types @@ -197,7 +198,7 @@ def guardrail_status_to_action(status: str | None) -> str: if not status: return "passed" s: Final = (status or "").lower() - if s == "skipped": + if s in UNEVALUATED_GUARDRAIL_STATUSES: return "skipped" if "intervened" in s or "block" in s: return "blocked" diff --git a/litellm/types/utils.py b/litellm/types/utils.py index cf302b3f27f..6da44007fb4 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3123,6 +3123,8 @@ GuardrailStatus = Literal[ "success", "guardrail_flagged", "guardrail_intervened", "guardrail_failed_to_respond", "not_run", "skipped" ] +UNEVALUATED_GUARDRAIL_STATUSES: Final[frozenset[GuardrailStatus]] = frozenset({"not_run", "skipped"}) + # Fields on a guardrail record whose values can quote the caller's prompt: the payload sent to the # guardrail, the provider response that echoes it back, and the two first-party hooks that inline # prompt substrings (``block_code_execution`` and ``litellm_content_filter``). Every other field diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index c6d01db45de..7bfd50fa6f8 100644 --- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1942,6 +1942,27 @@ class TestNoScannableContentRecordsSkipped: assert guardrail.last_inputs is not None assert all(e.get("guardrail_status") != "skipped" for e in self._recorded_entries(data)) + @pytest.mark.asyncio + async def test_image_only_content_is_not_reported_as_skipped(self): + """Images are only scanned alongside text, so an image-only request is a + pre-existing scan gap, not a message-scoping skip, and must not be labelled one""" + handler = OpenAIChatCompletionsHandler() + guardrail = MockGuardrail(guardrail_name="image-guardrail") + guardrail.skip_system_message_in_guardrail = True + data = { + "messages": [ + {"role": "system", "content": "SYSTEM-PROMPT"}, + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}], + }, + ] + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert self._recorded_entries(data) == [] + class TestBuildBlockSseChunks: """build_block_sse_chunks turns a streaming ModifyResponseException into 200 SSE chunks""" diff --git a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py index cb85eb8b5ec..ae883d3eaeb 100644 --- a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py +++ b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py @@ -350,15 +350,17 @@ async def test_zero_and_non_int_usage_counters_are_skipped(): @pytest.mark.asyncio -async def test_skipped_entries_are_indexed_but_not_counted_as_evaluations(): +@pytest.mark.parametrize("status", ["skipped", "not_run"]) +async def test_skipped_entries_are_indexed_but_not_counted_as_evaluations(status: str): """ LIT-6314 records a skipped entry when message scoping leaves a guardrail - nothing to scan. The guardrail never evaluated the request, so counting it - as a passed evaluation would inflate daily pass rates; it still gets an - index row so per-request drill-down finds the spend log. + nothing to scan (older spend logs spell it not_run). The guardrail never + evaluated the request, so counting it as a passed evaluation would inflate + daily pass rates; it still gets an index row so per-request drill-down + finds the spend log. """ prisma = _prisma() - logs = [_payload("r1", guardrail_status="skipped"), _payload("r2")] + logs = [_payload("r1", guardrail_status=status), _payload("r2")] await process_spend_logs_guardrail_usage(prisma, logs) diff --git a/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py index 9587e3d95ee..bafe608bff8 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py @@ -594,14 +594,15 @@ class TestModeMatching: class TestSkippedGuardrails: """LIT-6314 logs a skipped entry for a guardrail that message scoping left nothing to scan.""" - def test_skipped_alone_never_evidences_compliance(self): + @pytest.mark.parametrize("status", ["skipped", "not_run"]) + def test_skipped_alone_never_evidences_compliance(self, status: str): data = ComplianceCheckRequest( request_id="req-601", user_id="user-1", model="gpt-4", timestamp="2026-02-17T00:00:00Z", guardrail_information=[ - {"guardrail_name": "pii_detection", "guardrail_status": "skipped", "guardrail_mode": "pre_call"}, + {"guardrail_name": "pii_detection", "guardrail_status": status, "guardrail_mode": "pre_call"}, ], ) results = {c.check_name: c.passed for c in ComplianceChecker(data).check_eu_ai_act()} diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx index 740ef857690..02a4c8008a6 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx @@ -68,22 +68,25 @@ describe("GuardrailViewer", () => { expect(screen.queryByText("FAILED")).not.toBeInTheDocument(); }); - it("renders skipped as SKIPPED (muted) and keeps it out of the evaluated and passed counts", async () => { - const user = userEvent.setup(); - const data = makeGuardrailInformation(skippedPreCall); - renderWithProviders(); + it.each(["skipped", "not_run"])( + "renders %s as SKIPPED (muted) and keeps it out of the evaluated and passed counts", + async (guardrail_status) => { + const user = userEvent.setup(); + const data = makeGuardrailInformation({ ...skippedPreCall, guardrail_status }); + renderWithProviders(); - expect(screen.getByText(/0 guardrails evaluated/)).toBeInTheDocument(); - expect(screen.getByText(/0 Passed/)).toHaveClass("text-muted-foreground"); - expect(screen.getByText(/1 Skipped/)).toBeInTheDocument(); - const badge = screen.getByText("SKIPPED"); - expect(badge).toHaveClass("text-muted-foreground"); - expect(screen.queryByText("FAILED")).not.toBeInTheDocument(); - expect(screen.queryByText(/^T\+/)).not.toBeInTheDocument(); + expect(screen.getByText(/0 guardrails evaluated/)).toBeInTheDocument(); + expect(screen.getByText(/0 Passed/)).toHaveClass("text-muted-foreground"); + expect(screen.getByText(/1 Skipped/)).toBeInTheDocument(); + const badge = screen.getByText("SKIPPED"); + expect(badge).toHaveClass("text-muted-foreground"); + expect(screen.queryByText("FAILED")).not.toBeInTheDocument(); + expect(screen.queryByText(/^T\+/)).not.toBeInTheDocument(); - await user.click(screen.getByText("pii-rail")); - expect(screen.getByText("no scannable content after message scoping")).toBeInTheDocument(); - }); + await user.click(screen.getByText("pii-rail")); + expect(screen.getByText("no scannable content after message scoping")).toBeInTheDocument(); + }, + ); it("anchors the lifecycle timeline on timed entries when an untimed skipped entry sorts first", () => { const skipped = makeGuardrailInformation({ ...skippedPreCall, guardrail_name: "skipped-rail" }); diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx index c67d682a233..d171b7ff4b3 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx @@ -140,7 +140,7 @@ const getEntryOutcome = (entry: GuardrailInformation): EntryOutcome => { const status = (entry.guardrail_status ?? "").toLowerCase(); if (status === "success") return "passed"; if (status === "guardrail_flagged") return "flagged"; - if (status === "skipped") return "skipped"; + if (status === "skipped" || status === "not_run") return "skipped"; return "failed"; }; diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx index e052faa2228..e022d166fe0 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx @@ -705,7 +705,7 @@ const GUARDRAIL_JUMP_LINK_STYLE = { const isPassedStatus = (status: unknown) => status === "pass" || status === "passed" || status === "success"; const isFlaggedStatus = (status: unknown) => status === "flagged" || status === "guardrail_flagged"; -const isSkippedStatus = (status: unknown) => status === "skipped"; +const isSkippedStatus = (status: unknown) => status === "skipped" || status === "not_run"; const guardrailJumpLinkOutcome = (evaluated: unknown[]): keyof typeof GUARDRAIL_JUMP_LINK_STYLE => { if (evaluated.length === 0) return "skipped"; From bd9a87ea7683bfc8d547a8996e665a6631e82af9 Mon Sep 17 00:00:00 2001 From: yucheng Date: Mon, 14 Sep 2026 23:46:09 +0000 Subject: [PATCH 17/23] Revert "fix(guardrails): keep legacy not_run neutral and stop labelling image-only input as skipped" This reverts commit f78dd921c958c38c3adba9ee490071a658c79e27. --- .../chat/guardrail_translation/handler.py | 2 +- litellm/proxy/compliance_checks.py | 7 +---- litellm/proxy/guardrails/usage_tracking.py | 3 +- litellm/types/utils.py | 2 -- .../test_openai_guardrail_handler.py | 21 ------------- .../proxy/guardrails/test_usage_tracking.py | 12 +++---- .../test_compliance_endpoints.py | 5 ++- .../GuardrailViewer/GuardrailViewer.test.tsx | 31 +++++++++---------- .../GuardrailViewer/GuardrailViewer.tsx | 2 +- .../LogDetailsDrawer/LogDetailContent.tsx | 2 +- 10 files changed, 26 insertions(+), 61 deletions(-) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 035dce46d27..dca90e07421 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -210,7 +210,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): task_mappings=tool_call_task_mappings, ) - elif not images_to_check and not guardrail_to_apply.records_own_guardrail_information: + elif not guardrail_to_apply.records_own_guardrail_information: guardrail_to_apply.add_standard_logging_guardrail_information_to_request_data( guardrail_json_response="no scannable content after message scoping", request_data=data, diff --git a/litellm/proxy/compliance_checks.py b/litellm/proxy/compliance_checks.py index 18123156411..ef2d8fb6e20 100644 --- a/litellm/proxy/compliance_checks.py +++ b/litellm/proxy/compliance_checks.py @@ -11,7 +11,6 @@ from litellm.types.proxy.compliance_endpoints import ( ComplianceCheckRequest, ComplianceCheckResult, ) -from litellm.types.utils import UNEVALUATED_GUARDRAIL_STATUSES class ComplianceChecker: @@ -27,11 +26,7 @@ class ComplianceChecker: def __init__(self, data: ComplianceCheckRequest): self.data = data - self.guardrails = [ - g - for g in (data.guardrail_information or []) - if g.get("guardrail_status") not in UNEVALUATED_GUARDRAIL_STATUSES - ] + self.guardrails = [g for g in (data.guardrail_information or []) if g.get("guardrail_status") != "skipped"] def _get_guardrails_by_mode(self, mode: str) -> list[dict]: """ diff --git a/litellm/proxy/guardrails/usage_tracking.py b/litellm/proxy/guardrails/usage_tracking.py index 789b8febfbf..8a131fbfcde 100644 --- a/litellm/proxy/guardrails/usage_tracking.py +++ b/litellm/proxy/guardrails/usage_tracking.py @@ -27,7 +27,6 @@ from litellm.repositories.table_repositories import ( DailyGuardrailUsageUnitsRepository, SpendLogGuardrailIndexRepository, ) -from litellm.types.utils import UNEVALUATED_GUARDRAIL_STATUSES if TYPE_CHECKING: from prisma import types as prisma_types @@ -198,7 +197,7 @@ def guardrail_status_to_action(status: str | None) -> str: if not status: return "passed" s: Final = (status or "").lower() - if s in UNEVALUATED_GUARDRAIL_STATUSES: + if s == "skipped": return "skipped" if "intervened" in s or "block" in s: return "blocked" diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 6da44007fb4..cf302b3f27f 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3123,8 +3123,6 @@ GuardrailStatus = Literal[ "success", "guardrail_flagged", "guardrail_intervened", "guardrail_failed_to_respond", "not_run", "skipped" ] -UNEVALUATED_GUARDRAIL_STATUSES: Final[frozenset[GuardrailStatus]] = frozenset({"not_run", "skipped"}) - # Fields on a guardrail record whose values can quote the caller's prompt: the payload sent to the # guardrail, the provider response that echoes it back, and the two first-party hooks that inline # prompt substrings (``block_code_execution`` and ``litellm_content_filter``). Every other field diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 7bfd50fa6f8..c6d01db45de 100644 --- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1942,27 +1942,6 @@ class TestNoScannableContentRecordsSkipped: assert guardrail.last_inputs is not None assert all(e.get("guardrail_status") != "skipped" for e in self._recorded_entries(data)) - @pytest.mark.asyncio - async def test_image_only_content_is_not_reported_as_skipped(self): - """Images are only scanned alongside text, so an image-only request is a - pre-existing scan gap, not a message-scoping skip, and must not be labelled one""" - handler = OpenAIChatCompletionsHandler() - guardrail = MockGuardrail(guardrail_name="image-guardrail") - guardrail.skip_system_message_in_guardrail = True - data = { - "messages": [ - {"role": "system", "content": "SYSTEM-PROMPT"}, - { - "role": "user", - "content": [{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}], - }, - ] - } - - await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) - - assert self._recorded_entries(data) == [] - class TestBuildBlockSseChunks: """build_block_sse_chunks turns a streaming ModifyResponseException into 200 SSE chunks""" diff --git a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py index ae883d3eaeb..cb85eb8b5ec 100644 --- a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py +++ b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py @@ -350,17 +350,15 @@ async def test_zero_and_non_int_usage_counters_are_skipped(): @pytest.mark.asyncio -@pytest.mark.parametrize("status", ["skipped", "not_run"]) -async def test_skipped_entries_are_indexed_but_not_counted_as_evaluations(status: str): +async def test_skipped_entries_are_indexed_but_not_counted_as_evaluations(): """ LIT-6314 records a skipped entry when message scoping leaves a guardrail - nothing to scan (older spend logs spell it not_run). The guardrail never - evaluated the request, so counting it as a passed evaluation would inflate - daily pass rates; it still gets an index row so per-request drill-down - finds the spend log. + nothing to scan. The guardrail never evaluated the request, so counting it + as a passed evaluation would inflate daily pass rates; it still gets an + index row so per-request drill-down finds the spend log. """ prisma = _prisma() - logs = [_payload("r1", guardrail_status=status), _payload("r2")] + logs = [_payload("r1", guardrail_status="skipped"), _payload("r2")] await process_spend_logs_guardrail_usage(prisma, logs) diff --git a/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py index bafe608bff8..9587e3d95ee 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py @@ -594,15 +594,14 @@ class TestModeMatching: class TestSkippedGuardrails: """LIT-6314 logs a skipped entry for a guardrail that message scoping left nothing to scan.""" - @pytest.mark.parametrize("status", ["skipped", "not_run"]) - def test_skipped_alone_never_evidences_compliance(self, status: str): + def test_skipped_alone_never_evidences_compliance(self): data = ComplianceCheckRequest( request_id="req-601", user_id="user-1", model="gpt-4", timestamp="2026-02-17T00:00:00Z", guardrail_information=[ - {"guardrail_name": "pii_detection", "guardrail_status": status, "guardrail_mode": "pre_call"}, + {"guardrail_name": "pii_detection", "guardrail_status": "skipped", "guardrail_mode": "pre_call"}, ], ) results = {c.check_name: c.passed for c in ComplianceChecker(data).check_eu_ai_act()} diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx index 02a4c8008a6..740ef857690 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx @@ -68,25 +68,22 @@ describe("GuardrailViewer", () => { expect(screen.queryByText("FAILED")).not.toBeInTheDocument(); }); - it.each(["skipped", "not_run"])( - "renders %s as SKIPPED (muted) and keeps it out of the evaluated and passed counts", - async (guardrail_status) => { - const user = userEvent.setup(); - const data = makeGuardrailInformation({ ...skippedPreCall, guardrail_status }); - renderWithProviders(); + it("renders skipped as SKIPPED (muted) and keeps it out of the evaluated and passed counts", async () => { + const user = userEvent.setup(); + const data = makeGuardrailInformation(skippedPreCall); + renderWithProviders(); - expect(screen.getByText(/0 guardrails evaluated/)).toBeInTheDocument(); - expect(screen.getByText(/0 Passed/)).toHaveClass("text-muted-foreground"); - expect(screen.getByText(/1 Skipped/)).toBeInTheDocument(); - const badge = screen.getByText("SKIPPED"); - expect(badge).toHaveClass("text-muted-foreground"); - expect(screen.queryByText("FAILED")).not.toBeInTheDocument(); - expect(screen.queryByText(/^T\+/)).not.toBeInTheDocument(); + expect(screen.getByText(/0 guardrails evaluated/)).toBeInTheDocument(); + expect(screen.getByText(/0 Passed/)).toHaveClass("text-muted-foreground"); + expect(screen.getByText(/1 Skipped/)).toBeInTheDocument(); + const badge = screen.getByText("SKIPPED"); + expect(badge).toHaveClass("text-muted-foreground"); + expect(screen.queryByText("FAILED")).not.toBeInTheDocument(); + expect(screen.queryByText(/^T\+/)).not.toBeInTheDocument(); - await user.click(screen.getByText("pii-rail")); - expect(screen.getByText("no scannable content after message scoping")).toBeInTheDocument(); - }, - ); + await user.click(screen.getByText("pii-rail")); + expect(screen.getByText("no scannable content after message scoping")).toBeInTheDocument(); + }); it("anchors the lifecycle timeline on timed entries when an untimed skipped entry sorts first", () => { const skipped = makeGuardrailInformation({ ...skippedPreCall, guardrail_name: "skipped-rail" }); diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx index d171b7ff4b3..c67d682a233 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx @@ -140,7 +140,7 @@ const getEntryOutcome = (entry: GuardrailInformation): EntryOutcome => { const status = (entry.guardrail_status ?? "").toLowerCase(); if (status === "success") return "passed"; if (status === "guardrail_flagged") return "flagged"; - if (status === "skipped" || status === "not_run") return "skipped"; + if (status === "skipped") return "skipped"; return "failed"; }; diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx index e022d166fe0..e052faa2228 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx @@ -705,7 +705,7 @@ const GUARDRAIL_JUMP_LINK_STYLE = { const isPassedStatus = (status: unknown) => status === "pass" || status === "passed" || status === "success"; const isFlaggedStatus = (status: unknown) => status === "flagged" || status === "guardrail_flagged"; -const isSkippedStatus = (status: unknown) => status === "skipped" || status === "not_run"; +const isSkippedStatus = (status: unknown) => status === "skipped"; const guardrailJumpLinkOutcome = (evaluated: unknown[]): keyof typeof GUARDRAIL_JUMP_LINK_STYLE => { if (evaluated.length === 0) return "skipped"; From 0d0b96ed0670a505c361b2da42605dc4113c4dce Mon Sep 17 00:00:00 2001 From: yucheng Date: Mon, 14 Sep 2026 23:46:09 +0000 Subject: [PATCH 18/23] Revert "refactor(guardrails): rename scoped-out evaluation status from not_run to skipped" This reverts commit b37ce94075124b4429996cd52313d100fbb5212e. --- litellm/litellm_core_utils/litellm_logging.py | 1 - .../chat/guardrail_translation/handler.py | 2 +- litellm/proxy/compliance_checks.py | 2 +- litellm/proxy/guardrails/usage_endpoints.py | 4 ++-- litellm/proxy/guardrails/usage_tracking.py | 8 +++---- litellm/types/utils.py | 3 +-- .../test_litellm_logging.py | 12 ---------- .../test_openai_guardrail_handler.py | 8 +++---- .../proxy/guardrails/test_usage_endpoints.py | 10 ++++---- .../proxy/guardrails/test_usage_tracking.py | 22 ++++++++--------- .../test_compliance_endpoints.py | 12 +++++----- .../GuardrailsMonitor/LogViewer.test.tsx | 8 +++---- .../GuardrailsMonitor/LogViewer.tsx | 6 ++--- .../components/GuardrailsMonitor/mockData.ts | 2 +- .../GuardrailViewer/GuardrailViewer.test.tsx | 10 ++++---- .../GuardrailViewer/GuardrailViewer.tsx | 24 +++++++++---------- .../LogDetailContent.integration.test.tsx | 14 +++++------ .../LogDetailsDrawer/LogDetailContent.tsx | 12 +++++----- 18 files changed, 73 insertions(+), 87 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index e88de4acd0e..9ba9fd082f3 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -6021,7 +6021,6 @@ def _get_status_fields( "failure": "guardrail_failed_to_respond", # legacy "guardrail_failed_to_respond": "guardrail_failed_to_respond", # direct "not_run": "not_run", - "skipped": "not_run", } # Set LLM API status diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index dca90e07421..0f5096d0108 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -214,7 +214,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): guardrail_to_apply.add_standard_logging_guardrail_information_to_request_data( guardrail_json_response="no scannable content after message scoping", request_data=data, - guardrail_status="skipped", + guardrail_status="not_run", ) verbose_proxy_logger.debug( diff --git a/litellm/proxy/compliance_checks.py b/litellm/proxy/compliance_checks.py index ef2d8fb6e20..d9cc1d0f4fc 100644 --- a/litellm/proxy/compliance_checks.py +++ b/litellm/proxy/compliance_checks.py @@ -26,7 +26,7 @@ class ComplianceChecker: def __init__(self, data: ComplianceCheckRequest): self.data = data - self.guardrails = [g for g in (data.guardrail_information or []) if g.get("guardrail_status") != "skipped"] + self.guardrails = [g for g in (data.guardrail_information or []) if g.get("guardrail_status") != "not_run"] def _get_guardrails_by_mode(self, mode: str) -> list[dict]: """ diff --git a/litellm/proxy/guardrails/usage_endpoints.py b/litellm/proxy/guardrails/usage_endpoints.py index 651c4bb1963..556b6a4e919 100644 --- a/litellm/proxy/guardrails/usage_endpoints.py +++ b/litellm/proxy/guardrails/usage_endpoints.py @@ -42,7 +42,7 @@ if TYPE_CHECKING: router: Final = APIRouter() _EMPTY_UNITS: Final[Mapping[str, int]] = MappingProxyType({}) -_ACTION_SEVERITY: Final[Mapping[str, int]] = MappingProxyType({"skipped": 0, "passed": 1, "flagged": 2, "blocked": 3}) +_ACTION_SEVERITY: Final[Mapping[str, int]] = MappingProxyType({"not_run": 0, "passed": 1, "flagged": 2, "blocked": 3}) _T = TypeVar("_T") @@ -325,7 +325,7 @@ class UsageDetailResponse(BaseModel): class UsageLogEntry(BaseModel): id: str timestamp: str - action: str # blocked | passed | flagged | skipped + action: str # blocked | passed | flagged | not_run score: float | None latency_ms: float | None model: str | None diff --git a/litellm/proxy/guardrails/usage_tracking.py b/litellm/proxy/guardrails/usage_tracking.py index 8a131fbfcde..7e11b69108b 100644 --- a/litellm/proxy/guardrails/usage_tracking.py +++ b/litellm/proxy/guardrails/usage_tracking.py @@ -193,12 +193,12 @@ async def _upsert_rows_with_retry( def guardrail_status_to_action(status: str | None) -> str: - """Map StandardLogging guardrail_status to blocked/passed/flagged/skipped.""" + """Map StandardLogging guardrail_status to blocked/passed/flagged/not_run.""" if not status: return "passed" s: Final = (status or "").lower() - if s == "skipped": - return "skipped" + if s == "not_run": + return "not_run" if "intervened" in s or "block" in s: return "blocked" if "flagged" in s or "fail" in s or "error" in s: @@ -380,7 +380,7 @@ async def process_spend_logs_guardrail_usage( if not isinstance(guardrail_id, str) or not guardrail_id: continue action = guardrail_status_to_action(entry.get("guardrail_status")) - if action != "skipped": + if action != "not_run": key = _MetricsKey(guardrail_id, date_key) daily_guardrail[key]["requests_evaluated"] += 1 if action == "passed": diff --git a/litellm/types/utils.py b/litellm/types/utils.py index cf302b3f27f..1d73542c9bb 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3120,7 +3120,7 @@ class GuardrailMode(TypedDict, total=False): GuardrailStatus = Literal[ - "success", "guardrail_flagged", "guardrail_intervened", "guardrail_failed_to_respond", "not_run", "skipped" + "success", "guardrail_flagged", "guardrail_intervened", "guardrail_failed_to_respond", "not_run" ] # Fields on a guardrail record whose values can quote the caller's prompt: the payload sent to the @@ -3367,7 +3367,6 @@ class StandardLoggingPayloadStatusFields(TypedDict, total=False): - 'guardrail_intervened': Guardrail blocked or modified content - 'guardrail_failed_to_respond': Guardrail had technical failure - 'not_run': No guardrail was run - - 'skipped': Only used per guardrail entry, message scoping left the guardrail nothing to scan """ diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 6e67781944e..70f9bae283b 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -6926,18 +6926,6 @@ def test_get_status_fields_ranks_guardrail_flagged_between_success_and_intervene )["guardrail_status"] == "guardrail_intervened" -def test_get_status_fields_rolls_skipped_entries_up_to_not_run(): - """LIT-6314: a guardrail that message scoping left nothing to scan records a - skipped entry. At request level that means no guardrail ran, and a skipped - entry must never outrank a sibling that did evaluate.""" - skipped = {"guardrail_status": "skipped"} - - assert _get_status_fields("success", [skipped], None)["guardrail_status"] == "not_run" - assert _get_status_fields( - "success", [skipped, {"guardrail_status": "success"}], None - )["guardrail_status"] == "success" - - def test_get_error_information_redacts_provider_key_from_upstream_url(): """A pass-through upstream failure logs the httpx traceback, whose message quotes the upstream URL with the provider key in its query string. That diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index c6d01db45de..4f1163ed806 100644 --- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1893,7 +1893,7 @@ class TestScanOnlyToolResults: assert data["messages"][4]["content"] == "and then?" -class TestNoScannableContentRecordsSkipped: +class TestNoScannableContentRecordsNotRun: """LIT-6314: a guardrail whose scoping leaves nothing to scan must still persist an evaluation record""" def _system_only_data(self) -> dict: @@ -1904,7 +1904,7 @@ class TestNoScannableContentRecordsSkipped: return metadata.get("standard_logging_guardrail_information") or [] @pytest.mark.asyncio - async def test_skipped_scan_records_skipped_entry(self): + async def test_skipped_scan_records_not_run_entry(self): handler = OpenAIChatCompletionsHandler() guardrail = MockGuardrail(guardrail_name="skip-system-guardrail") guardrail.skip_system_message_in_guardrail = True @@ -1916,7 +1916,7 @@ class TestNoScannableContentRecordsSkipped: entries = self._recorded_entries(data) assert len(entries) == 1 assert entries[0]["guardrail_name"] == "skip-system-guardrail" - assert entries[0]["guardrail_status"] == "skipped" + assert entries[0]["guardrail_status"] == "not_run" @pytest.mark.asyncio async def test_self_recording_guardrail_is_left_alone(self): @@ -1940,7 +1940,7 @@ class TestNoScannableContentRecordsSkipped: await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) assert guardrail.last_inputs is not None - assert all(e.get("guardrail_status") != "skipped" for e in self._recorded_entries(data)) + assert all(e.get("guardrail_status") != "not_run" for e in self._recorded_entries(data)) class TestBuildBlockSseChunks: diff --git a/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py b/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py index 1df0c1477e2..db87e12ac88 100644 --- a/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py +++ b/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py @@ -685,7 +685,7 @@ async def test_detail_prev_trend_query_is_bounded(): @pytest.mark.asyncio -async def test_logs_report_skipped_entries_as_skipped_not_passed(): +async def test_logs_report_not_run_entries_as_not_run_not_passed(): """LIT-6314: a guardrail that never scanned must not be reported as a pass in the drill-down.""" index_row = MagicMock() index_row.request_id = "req-nr" @@ -697,7 +697,7 @@ async def test_logs_report_skipped_entries_as_skipped_not_passed(): spend_log.startTime = datetime(2026, 4, 22) spend_log.metadata = { "guardrail_information": [ - {"guardrail_name": "db-1", "guardrail_status": "skipped", "duration": 0.0}, + {"guardrail_name": "db-1", "guardrail_status": "not_run", "duration": 0.0}, ] } prisma = _prisma(find_unique=_db_row(), index_find_many=[index_row]) @@ -715,11 +715,11 @@ async def test_logs_report_skipped_entries_as_skipped_not_passed(): end_date=END, user_api_key_dict=ADMIN, ) - assert [log.action for log in resp.logs] == ["skipped"] + assert [log.action for log in resp.logs] == ["not_run"] @pytest.mark.asyncio -async def test_logs_action_passed_filter_excludes_skipped_entries(): +async def test_logs_action_passed_filter_excludes_not_run_entries(): """LIT-6314: filtering the drill-down for passes must not return unscanned requests.""" index_row = MagicMock() index_row.request_id = "req-nr" @@ -729,7 +729,7 @@ async def test_logs_action_passed_filter_excludes_skipped_entries(): spend_log.request_id = "req-nr" spend_log.model = "gpt-4o-mini" spend_log.startTime = datetime(2026, 4, 22) - spend_log.metadata = {"guardrail_information": [{"guardrail_name": "db-1", "guardrail_status": "skipped"}]} + spend_log.metadata = {"guardrail_information": [{"guardrail_name": "db-1", "guardrail_status": "not_run"}]} prisma = _prisma(find_unique=_db_row(), index_find_many=[index_row]) prisma.db.litellm_spendlogs.find_many = AsyncMock(return_value=[spend_log]) handler = _config_handler() diff --git a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py index cb85eb8b5ec..69ec098b840 100644 --- a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py +++ b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py @@ -350,15 +350,15 @@ async def test_zero_and_non_int_usage_counters_are_skipped(): @pytest.mark.asyncio -async def test_skipped_entries_are_indexed_but_not_counted_as_evaluations(): +async def test_not_run_entries_are_indexed_but_not_counted_as_evaluations(): """ - LIT-6314 records a skipped entry when message scoping leaves a guardrail + LIT-6314 records a not_run entry when message scoping leaves a guardrail nothing to scan. The guardrail never evaluated the request, so counting it as a passed evaluation would inflate daily pass rates; it still gets an index row so per-request drill-down finds the spend log. """ prisma = _prisma() - logs = [_payload("r1", guardrail_status="skipped"), _payload("r2")] + logs = [_payload("r1", guardrail_status="not_run"), _payload("r2")] await process_spend_logs_guardrail_usage(prisma, logs) @@ -370,26 +370,26 @@ async def test_skipped_entries_are_indexed_but_not_counted_as_evaluations(): @pytest.mark.asyncio -async def test_skipped_entry_shares_index_key_with_evaluated_sibling_of_same_name(): +async def test_not_run_entry_shares_index_key_with_evaluated_sibling_of_same_name(): """ - The skipped entry from the shared base guardrail carries only guardrail_name, + The not_run entry from the shared base guardrail carries only guardrail_name, while the evaluated entry from the same guardrail (e.g. content filter on the output of a logging_only run) carries its guardrail_id. Keying them differently - lists one request twice in the monitor, once as skipped and once as passed. + lists one request twice in the monitor, once as not_run and once as passed. """ prisma = _prisma() payload = _payload("r1") payload["metadata"] = json.dumps( { "guardrail_information": [ - {"guardrail_name": "cf", "guardrail_status": "skipped"}, + {"guardrail_name": "cf", "guardrail_status": "not_run"}, { "guardrail_name": "cf", "guardrail_id": "cf-uuid", "policy_id": "pol-1", "guardrail_status": "success", }, - {"guardrail_name": "other", "guardrail_status": "skipped"}, + {"guardrail_name": "other", "guardrail_status": "not_run"}, ] } ) @@ -406,7 +406,7 @@ async def test_skipped_entry_shares_index_key_with_evaluated_sibling_of_same_nam @pytest.mark.asyncio -async def test_malformed_skipped_entry_does_not_drop_the_batch(): +async def test_malformed_not_run_entry_does_not_drop_the_batch(): prisma = _prisma() payload = _payload("r1") payload["metadata"] = json.dumps( @@ -428,10 +428,10 @@ async def test_malformed_skipped_entry_does_not_drop_the_batch(): @pytest.mark.asyncio -async def test_batch_of_only_skipped_entries_writes_no_metrics_row(): +async def test_batch_of_only_not_run_entries_writes_no_metrics_row(): prisma = _prisma() - await process_spend_logs_guardrail_usage(prisma, [_payload("r1", guardrail_status="skipped")]) + await process_spend_logs_guardrail_usage(prisma, [_payload("r1", guardrail_status="not_run")]) assert prisma.db.litellm_dailyguardrailmetrics.upsert.call_count == 0 index_rows = prisma.db.litellm_spendlogguardrailindex.create_many.call_args.kwargs["data"] diff --git a/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py index 9587e3d95ee..8382a5ada96 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py @@ -591,17 +591,17 @@ class TestModeMatching: assert mode in _guaranteed_modes(g_mode), (g_mode, mode) -class TestSkippedGuardrails: - """LIT-6314 logs a skipped entry for a guardrail that message scoping left nothing to scan.""" +class TestNotRunGuardrails: + """LIT-6314 logs a not_run entry for a guardrail that message scoping left nothing to scan.""" - def test_skipped_alone_never_evidences_compliance(self): + def test_not_run_alone_never_evidences_compliance(self): data = ComplianceCheckRequest( request_id="req-601", user_id="user-1", model="gpt-4", timestamp="2026-02-17T00:00:00Z", guardrail_information=[ - {"guardrail_name": "pii_detection", "guardrail_status": "skipped", "guardrail_mode": "pre_call"}, + {"guardrail_name": "pii_detection", "guardrail_status": "not_run", "guardrail_mode": "pre_call"}, ], ) results = {c.check_name: c.passed for c in ComplianceChecker(data).check_eu_ai_act()} @@ -609,7 +609,7 @@ class TestSkippedGuardrails: assert results["Content screened before LLM"] is False assert results["Audit record complete"] is False - def test_skipped_sibling_does_not_fail_a_passing_request(self): + def test_not_run_sibling_does_not_fail_a_passing_request(self): data = ComplianceCheckRequest( request_id="req-602", user_id="user-1", @@ -618,7 +618,7 @@ class TestSkippedGuardrails: pii_detected=True, guardrail_information=[ {"guardrail_name": "pii_detection", "guardrail_status": "success", "guardrail_mode": "pre_call"}, - {"guardrail_name": "system_only", "guardrail_status": "skipped", "guardrail_mode": "pre_call"}, + {"guardrail_name": "system_only", "guardrail_status": "not_run", "guardrail_mode": "pre_call"}, ], ) results = {c.check_name: c.passed for c in ComplianceChecker(data).check_gdpr()} diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.test.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.test.tsx index af5284830dc..083b1e5f3e2 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.test.tsx +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.test.tsx @@ -96,14 +96,14 @@ describe("GuardrailsMonitor LogViewer drawer", () => { }); }); -describe("GuardrailsMonitor LogViewer skipped rows", () => { - it("renders a skipped log as a neutral Skipped badge instead of a pass or failure", () => { +describe("GuardrailsMonitor LogViewer not_run rows", () => { + it("renders a not_run log as a neutral Not run badge instead of a pass or failure", () => { renderWithProviders( - , + , ); const row = screen.getByRole("button", { name: /system prompt only/ }); - expect(within(row).getByText("Skipped")).toHaveClass("text-muted-foreground"); + expect(within(row).getByText("Not run")).toHaveClass("text-muted-foreground"); expect(within(row).queryByText("Passed")).not.toBeInTheDocument(); expect(within(row).queryByText("Blocked")).not.toBeInTheDocument(); }); diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.tsx index b113c49e0a3..2abd699ba86 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.tsx +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.tsx @@ -10,15 +10,15 @@ import type { LogEntry as ViewLogsLogEntry } from "@/components/view_logs/column import type { LogEntry } from "./mockData"; const actionConfig: Record< - "blocked" | "passed" | "flagged" | "skipped", + "blocked" | "passed" | "flagged" | "not_run", { icon: React.ElementType; color: string; bg: string; border: string; label: string } > = { - skipped: { + not_run: { icon: MinusCircle, color: "text-muted-foreground", bg: "bg-muted", border: "border-border", - label: "Skipped", + label: "Not run", }, blocked: { icon: X, diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts b/ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts index 7053efcf88d..591d5cd3edd 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts @@ -10,7 +10,7 @@ export interface LogEntry { input_snippet?: string; output_snippet?: string; score?: number; - action: "blocked" | "passed" | "flagged" | "skipped"; + action: "blocked" | "passed" | "flagged" | "not_run"; model?: string; reason?: string; latency_ms?: number; diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx index 740ef857690..7f343211596 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx @@ -16,7 +16,7 @@ const PresidioPath = "@/components/view_logs/GuardrailViewer/PresidioDetectedEnt const BedrockPath = "@/components/view_logs/GuardrailViewer/BedrockGuardrailDetails"; const skippedPreCall: Partial = { - guardrail_status: "skipped", + guardrail_status: "not_run", guardrail_mode: "pre_call", guardrail_response: "no scannable content after message scoping", start_time: null, @@ -68,15 +68,15 @@ describe("GuardrailViewer", () => { expect(screen.queryByText("FAILED")).not.toBeInTheDocument(); }); - it("renders skipped as SKIPPED (muted) and keeps it out of the evaluated and passed counts", async () => { + it("renders not_run as NOT RUN (muted) and keeps it out of the evaluated and passed counts", async () => { const user = userEvent.setup(); const data = makeGuardrailInformation(skippedPreCall); renderWithProviders(); expect(screen.getByText(/0 guardrails evaluated/)).toBeInTheDocument(); expect(screen.getByText(/0 Passed/)).toHaveClass("text-muted-foreground"); - expect(screen.getByText(/1 Skipped/)).toBeInTheDocument(); - const badge = screen.getByText("SKIPPED"); + expect(screen.getByText(/1 Not run/)).toBeInTheDocument(); + const badge = screen.getByText("NOT RUN"); expect(badge).toHaveClass("text-muted-foreground"); expect(screen.queryByText("FAILED")).not.toBeInTheDocument(); expect(screen.queryByText(/^T\+/)).not.toBeInTheDocument(); @@ -85,7 +85,7 @@ describe("GuardrailViewer", () => { expect(screen.getByText("no scannable content after message scoping")).toBeInTheDocument(); }); - it("anchors the lifecycle timeline on timed entries when an untimed skipped entry sorts first", () => { + it("anchors the lifecycle timeline on timed entries when an untimed not_run entry sorts first", () => { const skipped = makeGuardrailInformation({ ...skippedPreCall, guardrail_name: "skipped-rail" }); const ran = makeGuardrailInformation(ranPostCall); renderWithProviders(); diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx index c67d682a233..1de0e3878b2 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx @@ -134,13 +134,13 @@ const getTotalMasked = (entry: GuardrailInformation): number => { ); }; -type EntryOutcome = "passed" | "flagged" | "failed" | "skipped"; +type EntryOutcome = "passed" | "flagged" | "failed" | "not_run"; const getEntryOutcome = (entry: GuardrailInformation): EntryOutcome => { const status = (entry.guardrail_status ?? "").toLowerCase(); if (status === "success") return "passed"; if (status === "guardrail_flagged") return "flagged"; - if (status === "skipped") return "skipped"; + if (status === "not_run") return "not_run"; return "failed"; }; @@ -150,18 +150,18 @@ const OUTCOME_LABEL: Record = { passed: "PASSED", flagged: "FLAGGED", failed: "FAILED", - skipped: "SKIPPED", + not_run: "NOT RUN", }; const OUTCOME_BADGE_CLASS: Record = { passed: "bg-success/15 text-success border border-success/20", flagged: "bg-warning/15 text-warning border border-warning/20", failed: "bg-destructive/15 text-destructive border border-destructive/20", - skipped: "bg-muted text-muted-foreground border border-border", + not_run: "bg-muted text-muted-foreground border border-border", }; const getHeaderOutcome = (counts: { evaluated: number; passed: number; flagged: number }): EntryOutcome => { - if (counts.evaluated === 0) return "skipped"; + if (counts.evaluated === 0) return "not_run"; if (counts.passed === counts.evaluated) return "passed"; if (counts.passed + counts.flagged === counts.evaluated) return "flagged"; return "failed"; @@ -242,7 +242,7 @@ const FlagCircleIcon = ({ className }: { className?: string }) => ( const OutcomeIcon = ({ outcome }: { outcome: EntryOutcome }) => { if (outcome === "passed") return ; if (outcome === "flagged") return ; - if (outcome === "skipped") return ; + if (outcome === "not_run") return ; return ; }; @@ -675,7 +675,7 @@ const EvaluationCard = ({ entry }: { entry: GuardrailInformation }) => { )} - {outcome === "skipped" && typeof guardrailResponse === "string" && ( + {outcome === "not_run" && typeof guardrailResponse === "string" && (

{guardrailResponse}

)} @@ -717,8 +717,8 @@ const GuardrailViewer = ({ data, accessToken, logEntry }: GuardrailViewerProps) const passedCount = guardrailEntries.filter(isEntrySuccess).length; const flaggedCount = guardrailEntries.filter((e) => getEntryOutcome(e) === "flagged").length; - const skippedCount = guardrailEntries.filter((e) => getEntryOutcome(e) === "skipped").length; - const evaluatedCount = guardrailEntries.length - skippedCount; + const notRunCount = guardrailEntries.filter((e) => getEntryOutcome(e) === "not_run").length; + const evaluatedCount = guardrailEntries.length - notRunCount; const allPassed = evaluatedCount > 0 && passedCount === evaluatedCount; const headerOutcome = getHeaderOutcome({ evaluated: evaluatedCount, passed: passedCount, flagged: flaggedCount }); @@ -778,11 +778,11 @@ const GuardrailViewer = ({ data, accessToken, logEntry }: GuardrailViewerProps) {flaggedCount} Flagged )} - {skippedCount > 0 && ( + {notRunCount > 0 && ( - {skippedCount} Skipped + {notRunCount} Not run )} diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.integration.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.integration.test.tsx index cbc377c1b73..3637b6c55a3 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.integration.test.tsx @@ -637,20 +637,20 @@ describe("GuardrailJumpLink", () => { }); it.each([ - [["success", "skipped"], "text-success", "\u2713"], - [["guardrail_intervened", "skipped"], "text-destructive", "\u2717"], - ])("ignores skipped when styling %j as %s", (statuses, expectedClass, glyph) => { + [["success", "not_run"], "text-success", "\u2713"], + [["guardrail_intervened", "not_run"], "text-destructive", "\u2717"], + ])("ignores not_run when styling %j as %s", (statuses, expectedClass, glyph) => { render( ({ guardrail_status: s }))} />); - const pill = screen.getByText(/1 guardrail evaluated, 1 skipped/); + const pill = screen.getByText(/1 guardrail evaluated, 1 not run/); expect(pill).toHaveClass(expectedClass); expect(pill).toHaveTextContent(glyph); }); - it("renders an all skipped request as neutral rather than passed", () => { - render(); + it("renders an all not_run request as neutral rather than passed", () => { + render(); - const pill = screen.getByText(/0 guardrails evaluated, 1 skipped/); + const pill = screen.getByText(/0 guardrails evaluated, 1 not run/); expect(pill).toHaveClass("text-muted-foreground"); expect(pill).not.toHaveTextContent("\u2713"); }); diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx index e052faa2228..30cd96935f7 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx @@ -700,15 +700,15 @@ const GUARDRAIL_JUMP_LINK_STYLE = { passed: { className: "border border-success/20 bg-success/10 text-success", glyph: "\u2713" }, flagged: { className: "border border-warning/20 bg-warning/10 text-warning", glyph: "\u26A0" }, failed: { className: "border border-destructive/20 bg-destructive/10 text-destructive", glyph: "\u2717" }, - skipped: { className: "border border-border bg-muted text-muted-foreground", glyph: "\u2013" }, + not_run: { className: "border border-border bg-muted text-muted-foreground", glyph: "\u2013" }, } as const; const isPassedStatus = (status: unknown) => status === "pass" || status === "passed" || status === "success"; const isFlaggedStatus = (status: unknown) => status === "flagged" || status === "guardrail_flagged"; -const isSkippedStatus = (status: unknown) => status === "skipped"; +const isNotRunStatus = (status: unknown) => status === "not_run"; const guardrailJumpLinkOutcome = (evaluated: unknown[]): keyof typeof GUARDRAIL_JUMP_LINK_STYLE => { - if (evaluated.length === 0) return "skipped"; + if (evaluated.length === 0) return "not_run"; if (evaluated.every(isPassedStatus)) return "passed"; if (evaluated.every((s) => isPassedStatus(s) || isFlaggedStatus(s))) return "flagged"; return "failed"; @@ -716,8 +716,8 @@ const guardrailJumpLinkOutcome = (evaluated: unknown[]): keyof typeof GUARDRAIL_ export function GuardrailJumpLink({ guardrailEntries }: { guardrailEntries: any[] }) { const statuses = guardrailEntries.map((e) => e?.guardrail_status || e?.status); - const evaluated = statuses.filter((s) => !isSkippedStatus(s)); - const skippedCount = statuses.length - evaluated.length; + const evaluated = statuses.filter((s) => !isNotRunStatus(s)); + const notRunCount = statuses.length - evaluated.length; const { className, glyph } = GUARDRAIL_JUMP_LINK_STYLE[guardrailJumpLinkOutcome(evaluated)]; const handleClick = () => { @@ -743,7 +743,7 @@ export function GuardrailJumpLink({ guardrailEntries }: { guardrailEntries: any[ > {glyph} {evaluated.length} guardrail {evaluated.length !== 1 ? "s" : ""} evaluated - {skippedCount > 0 ? `, ${skippedCount} skipped` : ""} + {notRunCount > 0 ? `, ${notRunCount} not run` : ""} {"\u2193"} From 0519d8634600bc404afc48a5d111d7869644e00e Mon Sep 17 00:00:00 2001 From: yucheng Date: Mon, 14 Sep 2026 23:49:03 +0000 Subject: [PATCH 19/23] fix(guardrails): stop labelling image-only input as a not_run scoping skip Images without text were never dispatched to guardrails before this change, so that gap is not a message scoping skip and must not get a not_run entry Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../chat/guardrail_translation/handler.py | 2 +- .../test_openai_guardrail_handler.py | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 0f5096d0108..9d790aa71d4 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -210,7 +210,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): task_mappings=tool_call_task_mappings, ) - elif not guardrail_to_apply.records_own_guardrail_information: + elif not images_to_check and not guardrail_to_apply.records_own_guardrail_information: guardrail_to_apply.add_standard_logging_guardrail_information_to_request_data( guardrail_json_response="no scannable content after message scoping", request_data=data, diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 4f1163ed806..5c971fe2c90 100644 --- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1942,6 +1942,27 @@ class TestNoScannableContentRecordsNotRun: assert guardrail.last_inputs is not None assert all(e.get("guardrail_status") != "not_run" for e in self._recorded_entries(data)) + @pytest.mark.asyncio + async def test_image_only_content_is_not_reported_as_not_run(self): + """Images are only scanned alongside text, so an image-only request is a + pre-existing scan gap, not a message-scoping skip, and must not be labelled one""" + handler = OpenAIChatCompletionsHandler() + guardrail = MockGuardrail(guardrail_name="image-guardrail") + guardrail.skip_system_message_in_guardrail = True + data = { + "messages": [ + {"role": "system", "content": "SYSTEM-PROMPT"}, + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}], + }, + ] + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert self._recorded_entries(data) == [] + class TestBuildBlockSseChunks: """build_block_sse_chunks turns a streaming ModifyResponseException into 200 SSE chunks""" From 52da64a45b76f516b2a0354af93a2aa5b851eec3 Mon Sep 17 00:00:00 2001 From: yucheng Date: Mon, 14 Sep 2026 23:56:32 +0000 Subject: [PATCH 20/23] fix(guardrails): only cite message scoping in the not_run reason when scoping is on A request whose messages carry no scannable content at all, with no skip flag set, now records the neutral reason no scannable content instead of blaming configuration Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../openai/chat/guardrail_translation/handler.py | 6 +++++- .../test_openai_guardrail_handler.py | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 9d790aa71d4..0be4b6a3a20 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -212,7 +212,11 @@ class OpenAIChatCompletionsHandler(BaseTranslation): elif not images_to_check and not guardrail_to_apply.records_own_guardrail_information: guardrail_to_apply.add_standard_logging_guardrail_information_to_request_data( - guardrail_json_response="no scannable content after message scoping", + guardrail_json_response=( + "no scannable content after message scoping" + if skip_system or skip_tool or scan_only_tool_results + else "no scannable content" + ), request_data=data, guardrail_status="not_run", ) diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 5c971fe2c90..40dc5e2df2c 100644 --- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1917,6 +1917,21 @@ class TestNoScannableContentRecordsNotRun: assert len(entries) == 1 assert entries[0]["guardrail_name"] == "skip-system-guardrail" assert entries[0]["guardrail_status"] == "not_run" + assert entries[0]["guardrail_response"] == "no scannable content after message scoping" + + @pytest.mark.asyncio + async def test_empty_content_without_scoping_does_not_blame_scoping(self): + handler = OpenAIChatCompletionsHandler() + guardrail = MockGuardrail(guardrail_name="unscoped-guardrail") + data = {"messages": [{"role": "user", "content": None}]} + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert guardrail.last_inputs is None + entries = self._recorded_entries(data) + assert len(entries) == 1 + assert entries[0]["guardrail_status"] == "not_run" + assert entries[0]["guardrail_response"] == "no scannable content" @pytest.mark.asyncio async def test_self_recording_guardrail_is_left_alone(self): From 5ceec4c21e9dd35f7b0693bdef327a1e59d7d3c8 Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 15 Sep 2026 00:03:02 +0000 Subject: [PATCH 21/23] fix(guardrails): cite message scoping only when an unscoped pass finds content The not_run reason now says after message scoping only when the same messages carry text or tool calls without the skip flags applied. A request that is empty to begin with, whatever the flags, records no scannable content Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../openai/chat/guardrail_translation/handler.py | 14 +++++++++++++- .../test_openai_guardrail_handler.py | 4 +++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 0be4b6a3a20..e8179e9921e 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -211,10 +211,22 @@ class OpenAIChatCompletionsHandler(BaseTranslation): ) elif not images_to_check and not guardrail_to_apply.records_own_guardrail_information: + unscoped_texts: Final[list[str]] = [] + unscoped_tool_calls: Final[list[ChatCompletionToolParam]] = [] + for unscoped_idx, unscoped_message in enumerate(messages): + self._extract_inputs( + message=unscoped_message, + msg_idx=unscoped_idx, + texts_to_check=unscoped_texts, + images_to_check=[], + tool_calls_to_check=unscoped_tool_calls, + text_task_mappings=[], + tool_call_task_mappings=[], + ) guardrail_to_apply.add_standard_logging_guardrail_information_to_request_data( guardrail_json_response=( "no scannable content after message scoping" - if skip_system or skip_tool or scan_only_tool_results + if unscoped_texts or unscoped_tool_calls else "no scannable content" ), request_data=data, diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 40dc5e2df2c..754c89d3d20 100644 --- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1920,9 +1920,11 @@ class TestNoScannableContentRecordsNotRun: assert entries[0]["guardrail_response"] == "no scannable content after message scoping" @pytest.mark.asyncio - async def test_empty_content_without_scoping_does_not_blame_scoping(self): + @pytest.mark.parametrize("skip_system", [False, True]) + async def test_empty_content_does_not_blame_scoping(self, skip_system: bool): handler = OpenAIChatCompletionsHandler() guardrail = MockGuardrail(guardrail_name="unscoped-guardrail") + guardrail.skip_system_message_in_guardrail = skip_system data = {"messages": [{"role": "user", "content": None}]} await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) From 714b113c5fa43e1c6ee147656513200e526219eb Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 15 Sep 2026 05:10:32 +0000 Subject: [PATCH 22/23] fix(guardrails): leave scoped-out image-only input unrecorded and split the not_run helper Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../chat/guardrail_translation/handler.py | 53 +++++++++++-------- .../test_openai_guardrail_handler.py | 20 +++++++ 2 files changed, 52 insertions(+), 21 deletions(-) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 332285722e1..20ca6a95aab 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -214,27 +214,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): ) elif not images_to_check and not guardrail_to_apply.records_own_guardrail_information: - unscoped_texts: Final[list[str]] = [] - unscoped_tool_calls: Final[list[ChatCompletionToolParam]] = [] - for unscoped_idx, unscoped_message in enumerate(messages): - self._extract_inputs( - message=unscoped_message, - msg_idx=unscoped_idx, - texts_to_check=unscoped_texts, - images_to_check=[], - tool_calls_to_check=unscoped_tool_calls, - text_task_mappings=[], - tool_call_task_mappings=[], - ) - guardrail_to_apply.add_standard_logging_guardrail_information_to_request_data( - guardrail_json_response=( - "no scannable content after message scoping" - if unscoped_texts or unscoped_tool_calls - else "no scannable content" - ), - request_data=data, - guardrail_status="not_run", - ) + self._record_not_run(data=data, messages=messages, guardrail_to_apply=guardrail_to_apply) verbose_proxy_logger.debug( "OpenAI Chat Completions: Processed input messages: %s", @@ -243,6 +223,37 @@ class OpenAIChatCompletionsHandler(BaseTranslation): return data + def _record_not_run( + self, + data: dict, + messages: list[dict[str, Any]], + guardrail_to_apply: "CustomGuardrail", + ) -> None: + unscoped_texts: Final[list[str]] = [] + unscoped_images: Final[list[str]] = [] + unscoped_tool_calls: Final[list[ChatCompletionToolParam]] = [] + for msg_idx, message in enumerate(messages): + self._extract_inputs( + message=message, + msg_idx=msg_idx, + texts_to_check=unscoped_texts, + images_to_check=unscoped_images, + tool_calls_to_check=unscoped_tool_calls, + text_task_mappings=[], + tool_call_task_mappings=[], + ) + if unscoped_images: + return + guardrail_to_apply.add_standard_logging_guardrail_information_to_request_data( + guardrail_json_response=( + "no scannable content after message scoping" + if unscoped_texts or unscoped_tool_calls + else "no scannable content" + ), + request_data=data, + guardrail_status="not_run", + ) + def extract_request_tool_names(self, data: dict) -> list[str]: """Extract tool names from OpenAI chat completions request (tools[].function.name, functions[].name).""" names: Final[list[str]] = [] diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 617fba587b2..b176d1e9057 100644 --- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1980,6 +1980,26 @@ class TestNoScannableContentRecordsNotRun: assert self._recorded_entries(data) == [] + @pytest.mark.asyncio + async def test_scoped_out_image_only_message_is_not_reported_as_not_run(self): + """An image in a skipped role must behave like any other image-only request""" + handler = OpenAIChatCompletionsHandler() + guardrail = MockGuardrail(guardrail_name="image-guardrail") + guardrail.skip_system_message_in_guardrail = True + data = { + "messages": [ + { + "role": "system", + "content": [{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}], + }, + ] + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert guardrail.last_inputs is None + assert self._recorded_entries(data) == [] + class ToolDroppingTextGuardrail(CustomGuardrail): """Answers one text per non-tool message it saw, the way a guardrail that From fe0fb97fd2d0a39ad022739547f464b48a7a061b Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 15 Sep 2026 05:20:56 +0000 Subject: [PATCH 23/23] fix(guardrails): record not_run when a skipped role mixes text and images Only image-only unscoped content stays unrecorded; text or tool content removed by scoping is recorded as not_run even when an image sits beside it. Also keeps the type-discipline budget flat by returning the reason from the helper and annotating the accumulator lists _extract_inputs requires. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../chat/guardrail_translation/handler.py | 51 +++++++++---------- litellm/proxy/compliance_checks.py | 2 +- .../test_openai_guardrail_handler.py | 26 ++++++++++ 3 files changed, 52 insertions(+), 27 deletions(-) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 20ca6a95aab..01e14f2248d 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -213,8 +213,16 @@ class OpenAIChatCompletionsHandler(BaseTranslation): task_mappings=tool_call_task_mappings, ) - elif not images_to_check and not guardrail_to_apply.records_own_guardrail_information: - self._record_not_run(data=data, messages=messages, guardrail_to_apply=guardrail_to_apply) + elif ( + not images_to_check + and not guardrail_to_apply.records_own_guardrail_information + and (not_run_reason := self._not_run_reason(messages)) is not None + ): + guardrail_to_apply.add_standard_logging_guardrail_information_to_request_data( + guardrail_json_response=not_run_reason, + request_data=data, + guardrail_status="not_run", + ) verbose_proxy_logger.debug( "OpenAI Chat Completions: Processed input messages: %s", @@ -223,36 +231,27 @@ class OpenAIChatCompletionsHandler(BaseTranslation): return data - def _record_not_run( + def _not_run_reason( self, - data: dict, - messages: list[dict[str, Any]], - guardrail_to_apply: "CustomGuardrail", - ) -> None: - unscoped_texts: Final[list[str]] = [] - unscoped_images: Final[list[str]] = [] - unscoped_tool_calls: Final[list[ChatCompletionToolParam]] = [] + messages: Sequence[dict[str, Any]], # mutable-ok: raw request messages consumed by _extract_inputs + ) -> str | None: + """Why nothing was scanned, or None when the only unscoped content is images, which this handler never scans.""" + texts: Final[list[str]] = [] # mutable-ok: filled by _extract_inputs + images: Final[list[str]] = [] # mutable-ok: filled by _extract_inputs + tool_calls: Final[list[ChatCompletionToolParam]] = [] # mutable-ok: filled by _extract_inputs for msg_idx, message in enumerate(messages): self._extract_inputs( message=message, msg_idx=msg_idx, - texts_to_check=unscoped_texts, - images_to_check=unscoped_images, - tool_calls_to_check=unscoped_tool_calls, - text_task_mappings=[], - tool_call_task_mappings=[], + texts_to_check=texts, + images_to_check=images, + tool_calls_to_check=tool_calls, + text_task_mappings=[], # mutable-ok: required by _extract_inputs, unused here + tool_call_task_mappings=[], # mutable-ok: required by _extract_inputs, unused here ) - if unscoped_images: - return - guardrail_to_apply.add_standard_logging_guardrail_information_to_request_data( - guardrail_json_response=( - "no scannable content after message scoping" - if unscoped_texts or unscoped_tool_calls - else "no scannable content" - ), - request_data=data, - guardrail_status="not_run", - ) + if texts or tool_calls: + return "no scannable content after message scoping" + return None if images else "no scannable content" def extract_request_tool_names(self, data: dict) -> list[str]: """Extract tool names from OpenAI chat completions request (tools[].function.name, functions[].name).""" diff --git a/litellm/proxy/compliance_checks.py b/litellm/proxy/compliance_checks.py index d9cc1d0f4fc..9d2f2dc7c69 100644 --- a/litellm/proxy/compliance_checks.py +++ b/litellm/proxy/compliance_checks.py @@ -26,7 +26,7 @@ class ComplianceChecker: def __init__(self, data: ComplianceCheckRequest): self.data = data - self.guardrails = [g for g in (data.guardrail_information or []) if g.get("guardrail_status") != "not_run"] + self.guardrails = tuple(g for g in data.guardrail_information or () if g.get("guardrail_status") != "not_run") def _get_guardrails_by_mode(self, mode: str) -> list[dict]: """ diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index b176d1e9057..cb884fb7cc1 100644 --- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -2000,6 +2000,32 @@ class TestNoScannableContentRecordsNotRun: assert guardrail.last_inputs is None assert self._recorded_entries(data) == [] + @pytest.mark.asyncio + async def test_scoped_out_text_with_image_records_not_run(self): + """Scoping removed text too, so the skip is recorded even though an image sat beside it""" + handler = OpenAIChatCompletionsHandler() + guardrail = MockGuardrail(guardrail_name="image-guardrail") + guardrail.skip_system_message_in_guardrail = True + data = { + "messages": [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Describe this picture."}, + {"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}, + ], + }, + ] + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert guardrail.last_inputs is None + entries = self._recorded_entries(data) + assert len(entries) == 1 + assert entries[0]["guardrail_status"] == "not_run" + assert entries[0]["guardrail_response"] == "no scannable content after message scoping" + class ToolDroppingTextGuardrail(CustomGuardrail): """Answers one text per non-tool message it saw, the way a guardrail that