From f78dd921c958c38c3adba9ee490071a658c79e27 Mon Sep 17 00:00:00 2001 From: yucheng Date: Mon, 14 Sep 2026 22:34:33 +0000 Subject: [PATCH] fix(guardrails): keep legacy not_run neutral and stop labelling image-only input as skipped Usage tracking, compliance and the dashboard now treat both not_run (older spend logs) and skipped as unevaluated through a shared UNEVALUATED_GUARDRAIL_STATUSES set, so old records stop counting as passed. The skipped record is no longer written when the request carried images, since images without text were never dispatched to guardrails before this change and that gap is not a message-scoping skip Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../chat/guardrail_translation/handler.py | 2 +- litellm/proxy/compliance_checks.py | 7 ++++- litellm/proxy/guardrails/usage_tracking.py | 3 +- litellm/types/utils.py | 2 ++ .../test_openai_guardrail_handler.py | 21 +++++++++++++ .../proxy/guardrails/test_usage_tracking.py | 12 ++++--- .../test_compliance_endpoints.py | 5 +-- .../GuardrailViewer/GuardrailViewer.test.tsx | 31 ++++++++++--------- .../GuardrailViewer/GuardrailViewer.tsx | 2 +- .../LogDetailsDrawer/LogDetailContent.tsx | 2 +- 10 files changed, 61 insertions(+), 26 deletions(-) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index dca90e07421..035dce46d27 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -210,7 +210,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): task_mappings=tool_call_task_mappings, ) - elif not guardrail_to_apply.records_own_guardrail_information: + elif not images_to_check and not guardrail_to_apply.records_own_guardrail_information: guardrail_to_apply.add_standard_logging_guardrail_information_to_request_data( guardrail_json_response="no scannable content after message scoping", request_data=data, diff --git a/litellm/proxy/compliance_checks.py b/litellm/proxy/compliance_checks.py index ef2d8fb6e20..18123156411 100644 --- a/litellm/proxy/compliance_checks.py +++ b/litellm/proxy/compliance_checks.py @@ -11,6 +11,7 @@ from litellm.types.proxy.compliance_endpoints import ( ComplianceCheckRequest, ComplianceCheckResult, ) +from litellm.types.utils import UNEVALUATED_GUARDRAIL_STATUSES class ComplianceChecker: @@ -26,7 +27,11 @@ class ComplianceChecker: def __init__(self, data: ComplianceCheckRequest): self.data = data - self.guardrails = [g for g in (data.guardrail_information or []) if g.get("guardrail_status") != "skipped"] + self.guardrails = [ + g + for g in (data.guardrail_information or []) + if g.get("guardrail_status") not in UNEVALUATED_GUARDRAIL_STATUSES + ] def _get_guardrails_by_mode(self, mode: str) -> list[dict]: """ diff --git a/litellm/proxy/guardrails/usage_tracking.py b/litellm/proxy/guardrails/usage_tracking.py index 8a131fbfcde..789b8febfbf 100644 --- a/litellm/proxy/guardrails/usage_tracking.py +++ b/litellm/proxy/guardrails/usage_tracking.py @@ -27,6 +27,7 @@ from litellm.repositories.table_repositories import ( DailyGuardrailUsageUnitsRepository, SpendLogGuardrailIndexRepository, ) +from litellm.types.utils import UNEVALUATED_GUARDRAIL_STATUSES if TYPE_CHECKING: from prisma import types as prisma_types @@ -197,7 +198,7 @@ def guardrail_status_to_action(status: str | None) -> str: if not status: return "passed" s: Final = (status or "").lower() - if s == "skipped": + if s in UNEVALUATED_GUARDRAIL_STATUSES: return "skipped" if "intervened" in s or "block" in s: return "blocked" diff --git a/litellm/types/utils.py b/litellm/types/utils.py index cf302b3f27f..6da44007fb4 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3123,6 +3123,8 @@ GuardrailStatus = Literal[ "success", "guardrail_flagged", "guardrail_intervened", "guardrail_failed_to_respond", "not_run", "skipped" ] +UNEVALUATED_GUARDRAIL_STATUSES: Final[frozenset[GuardrailStatus]] = frozenset({"not_run", "skipped"}) + # Fields on a guardrail record whose values can quote the caller's prompt: the payload sent to the # guardrail, the provider response that echoes it back, and the two first-party hooks that inline # prompt substrings (``block_code_execution`` and ``litellm_content_filter``). Every other field diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index c6d01db45de..7bfd50fa6f8 100644 --- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1942,6 +1942,27 @@ class TestNoScannableContentRecordsSkipped: assert guardrail.last_inputs is not None assert all(e.get("guardrail_status") != "skipped" for e in self._recorded_entries(data)) + @pytest.mark.asyncio + async def test_image_only_content_is_not_reported_as_skipped(self): + """Images are only scanned alongside text, so an image-only request is a + pre-existing scan gap, not a message-scoping skip, and must not be labelled one""" + handler = OpenAIChatCompletionsHandler() + guardrail = MockGuardrail(guardrail_name="image-guardrail") + guardrail.skip_system_message_in_guardrail = True + data = { + "messages": [ + {"role": "system", "content": "SYSTEM-PROMPT"}, + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}], + }, + ] + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert self._recorded_entries(data) == [] + class TestBuildBlockSseChunks: """build_block_sse_chunks turns a streaming ModifyResponseException into 200 SSE chunks""" diff --git a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py index cb85eb8b5ec..ae883d3eaeb 100644 --- a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py +++ b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py @@ -350,15 +350,17 @@ async def test_zero_and_non_int_usage_counters_are_skipped(): @pytest.mark.asyncio -async def test_skipped_entries_are_indexed_but_not_counted_as_evaluations(): +@pytest.mark.parametrize("status", ["skipped", "not_run"]) +async def test_skipped_entries_are_indexed_but_not_counted_as_evaluations(status: str): """ LIT-6314 records a skipped entry when message scoping leaves a guardrail - nothing to scan. The guardrail never evaluated the request, so counting it - as a passed evaluation would inflate daily pass rates; it still gets an - index row so per-request drill-down finds the spend log. + nothing to scan (older spend logs spell it not_run). The guardrail never + evaluated the request, so counting it as a passed evaluation would inflate + daily pass rates; it still gets an index row so per-request drill-down + finds the spend log. """ prisma = _prisma() - logs = [_payload("r1", guardrail_status="skipped"), _payload("r2")] + logs = [_payload("r1", guardrail_status=status), _payload("r2")] await process_spend_logs_guardrail_usage(prisma, logs) diff --git a/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py index 9587e3d95ee..bafe608bff8 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py @@ -594,14 +594,15 @@ class TestModeMatching: class TestSkippedGuardrails: """LIT-6314 logs a skipped entry for a guardrail that message scoping left nothing to scan.""" - def test_skipped_alone_never_evidences_compliance(self): + @pytest.mark.parametrize("status", ["skipped", "not_run"]) + def test_skipped_alone_never_evidences_compliance(self, status: str): data = ComplianceCheckRequest( request_id="req-601", user_id="user-1", model="gpt-4", timestamp="2026-02-17T00:00:00Z", guardrail_information=[ - {"guardrail_name": "pii_detection", "guardrail_status": "skipped", "guardrail_mode": "pre_call"}, + {"guardrail_name": "pii_detection", "guardrail_status": status, "guardrail_mode": "pre_call"}, ], ) results = {c.check_name: c.passed for c in ComplianceChecker(data).check_eu_ai_act()} diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx index 740ef857690..02a4c8008a6 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx @@ -68,22 +68,25 @@ describe("GuardrailViewer", () => { expect(screen.queryByText("FAILED")).not.toBeInTheDocument(); }); - it("renders skipped as SKIPPED (muted) and keeps it out of the evaluated and passed counts", async () => { - const user = userEvent.setup(); - const data = makeGuardrailInformation(skippedPreCall); - renderWithProviders(); + it.each(["skipped", "not_run"])( + "renders %s as SKIPPED (muted) and keeps it out of the evaluated and passed counts", + async (guardrail_status) => { + const user = userEvent.setup(); + const data = makeGuardrailInformation({ ...skippedPreCall, guardrail_status }); + renderWithProviders(); - expect(screen.getByText(/0 guardrails evaluated/)).toBeInTheDocument(); - expect(screen.getByText(/0 Passed/)).toHaveClass("text-muted-foreground"); - expect(screen.getByText(/1 Skipped/)).toBeInTheDocument(); - const badge = screen.getByText("SKIPPED"); - expect(badge).toHaveClass("text-muted-foreground"); - expect(screen.queryByText("FAILED")).not.toBeInTheDocument(); - expect(screen.queryByText(/^T\+/)).not.toBeInTheDocument(); + expect(screen.getByText(/0 guardrails evaluated/)).toBeInTheDocument(); + expect(screen.getByText(/0 Passed/)).toHaveClass("text-muted-foreground"); + expect(screen.getByText(/1 Skipped/)).toBeInTheDocument(); + const badge = screen.getByText("SKIPPED"); + expect(badge).toHaveClass("text-muted-foreground"); + expect(screen.queryByText("FAILED")).not.toBeInTheDocument(); + expect(screen.queryByText(/^T\+/)).not.toBeInTheDocument(); - await user.click(screen.getByText("pii-rail")); - expect(screen.getByText("no scannable content after message scoping")).toBeInTheDocument(); - }); + await user.click(screen.getByText("pii-rail")); + expect(screen.getByText("no scannable content after message scoping")).toBeInTheDocument(); + }, + ); it("anchors the lifecycle timeline on timed entries when an untimed skipped entry sorts first", () => { const skipped = makeGuardrailInformation({ ...skippedPreCall, guardrail_name: "skipped-rail" }); diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx index c67d682a233..d171b7ff4b3 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx @@ -140,7 +140,7 @@ const getEntryOutcome = (entry: GuardrailInformation): EntryOutcome => { const status = (entry.guardrail_status ?? "").toLowerCase(); if (status === "success") return "passed"; if (status === "guardrail_flagged") return "flagged"; - if (status === "skipped") return "skipped"; + if (status === "skipped" || status === "not_run") return "skipped"; return "failed"; }; diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx index e052faa2228..e022d166fe0 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx @@ -705,7 +705,7 @@ const GUARDRAIL_JUMP_LINK_STYLE = { const isPassedStatus = (status: unknown) => status === "pass" || status === "passed" || status === "success"; const isFlaggedStatus = (status: unknown) => status === "flagged" || status === "guardrail_flagged"; -const isSkippedStatus = (status: unknown) => status === "skipped"; +const isSkippedStatus = (status: unknown) => status === "skipped" || status === "not_run"; const guardrailJumpLinkOutcome = (evaluated: unknown[]): keyof typeof GUARDRAIL_JUMP_LINK_STYLE => { if (evaluated.length === 0) return "skipped";