diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index dca90e07421..035dce46d27 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -210,7 +210,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): task_mappings=tool_call_task_mappings, ) - elif not guardrail_to_apply.records_own_guardrail_information: + elif not images_to_check and not guardrail_to_apply.records_own_guardrail_information: guardrail_to_apply.add_standard_logging_guardrail_information_to_request_data( guardrail_json_response="no scannable content after message scoping", request_data=data, diff --git a/litellm/proxy/compliance_checks.py b/litellm/proxy/compliance_checks.py index ef2d8fb6e20..18123156411 100644 --- a/litellm/proxy/compliance_checks.py +++ b/litellm/proxy/compliance_checks.py @@ -11,6 +11,7 @@ from litellm.types.proxy.compliance_endpoints import ( ComplianceCheckRequest, ComplianceCheckResult, ) +from litellm.types.utils import UNEVALUATED_GUARDRAIL_STATUSES class ComplianceChecker: @@ -26,7 +27,11 @@ class ComplianceChecker: def __init__(self, data: ComplianceCheckRequest): self.data = data - self.guardrails = [g for g in (data.guardrail_information or []) if g.get("guardrail_status") != "skipped"] + self.guardrails = [ + g + for g in (data.guardrail_information or []) + if g.get("guardrail_status") not in UNEVALUATED_GUARDRAIL_STATUSES + ] def _get_guardrails_by_mode(self, mode: str) -> list[dict]: """ diff --git a/litellm/proxy/guardrails/usage_tracking.py b/litellm/proxy/guardrails/usage_tracking.py index 8a131fbfcde..789b8febfbf 100644 --- a/litellm/proxy/guardrails/usage_tracking.py +++ b/litellm/proxy/guardrails/usage_tracking.py @@ -27,6 +27,7 @@ from litellm.repositories.table_repositories import ( DailyGuardrailUsageUnitsRepository, SpendLogGuardrailIndexRepository, ) +from litellm.types.utils import UNEVALUATED_GUARDRAIL_STATUSES if TYPE_CHECKING: from prisma import types as prisma_types @@ -197,7 +198,7 @@ def guardrail_status_to_action(status: str | None) -> str: if not status: return "passed" s: Final = (status or "").lower() - if s == "skipped": + if s in UNEVALUATED_GUARDRAIL_STATUSES: return "skipped" if "intervened" in s or "block" in s: return "blocked" diff --git a/litellm/types/utils.py b/litellm/types/utils.py index cf302b3f27f..6da44007fb4 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3123,6 +3123,8 @@ GuardrailStatus = Literal[ "success", "guardrail_flagged", "guardrail_intervened", "guardrail_failed_to_respond", "not_run", "skipped" ] +UNEVALUATED_GUARDRAIL_STATUSES: Final[frozenset[GuardrailStatus]] = frozenset({"not_run", "skipped"}) + # Fields on a guardrail record whose values can quote the caller's prompt: the payload sent to the # guardrail, the provider response that echoes it back, and the two first-party hooks that inline # prompt substrings (``block_code_execution`` and ``litellm_content_filter``). Every other field diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index c6d01db45de..7bfd50fa6f8 100644 --- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1942,6 +1942,27 @@ class TestNoScannableContentRecordsSkipped: assert guardrail.last_inputs is not None assert all(e.get("guardrail_status") != "skipped" for e in self._recorded_entries(data)) + @pytest.mark.asyncio + async def test_image_only_content_is_not_reported_as_skipped(self): + """Images are only scanned alongside text, so an image-only request is a + pre-existing scan gap, not a message-scoping skip, and must not be labelled one""" + handler = OpenAIChatCompletionsHandler() + guardrail = MockGuardrail(guardrail_name="image-guardrail") + guardrail.skip_system_message_in_guardrail = True + data = { + "messages": [ + {"role": "system", "content": "SYSTEM-PROMPT"}, + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}], + }, + ] + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert self._recorded_entries(data) == [] + class TestBuildBlockSseChunks: """build_block_sse_chunks turns a streaming ModifyResponseException into 200 SSE chunks""" diff --git a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py index cb85eb8b5ec..ae883d3eaeb 100644 --- a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py +++ b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py @@ -350,15 +350,17 @@ async def test_zero_and_non_int_usage_counters_are_skipped(): @pytest.mark.asyncio -async def test_skipped_entries_are_indexed_but_not_counted_as_evaluations(): +@pytest.mark.parametrize("status", ["skipped", "not_run"]) +async def test_skipped_entries_are_indexed_but_not_counted_as_evaluations(status: str): """ LIT-6314 records a skipped entry when message scoping leaves a guardrail - nothing to scan. The guardrail never evaluated the request, so counting it - as a passed evaluation would inflate daily pass rates; it still gets an - index row so per-request drill-down finds the spend log. + nothing to scan (older spend logs spell it not_run). The guardrail never + evaluated the request, so counting it as a passed evaluation would inflate + daily pass rates; it still gets an index row so per-request drill-down + finds the spend log. """ prisma = _prisma() - logs = [_payload("r1", guardrail_status="skipped"), _payload("r2")] + logs = [_payload("r1", guardrail_status=status), _payload("r2")] await process_spend_logs_guardrail_usage(prisma, logs) diff --git a/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py index 9587e3d95ee..bafe608bff8 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py @@ -594,14 +594,15 @@ class TestModeMatching: class TestSkippedGuardrails: """LIT-6314 logs a skipped entry for a guardrail that message scoping left nothing to scan.""" - def test_skipped_alone_never_evidences_compliance(self): + @pytest.mark.parametrize("status", ["skipped", "not_run"]) + def test_skipped_alone_never_evidences_compliance(self, status: str): data = ComplianceCheckRequest( request_id="req-601", user_id="user-1", model="gpt-4", timestamp="2026-02-17T00:00:00Z", guardrail_information=[ - {"guardrail_name": "pii_detection", "guardrail_status": "skipped", "guardrail_mode": "pre_call"}, + {"guardrail_name": "pii_detection", "guardrail_status": status, "guardrail_mode": "pre_call"}, ], ) results = {c.check_name: c.passed for c in ComplianceChecker(data).check_eu_ai_act()} diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx index 740ef857690..02a4c8008a6 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx @@ -68,22 +68,25 @@ describe("GuardrailViewer", () => { expect(screen.queryByText("FAILED")).not.toBeInTheDocument(); }); - it("renders skipped as SKIPPED (muted) and keeps it out of the evaluated and passed counts", async () => { - const user = userEvent.setup(); - const data = makeGuardrailInformation(skippedPreCall); - renderWithProviders(); + it.each(["skipped", "not_run"])( + "renders %s as SKIPPED (muted) and keeps it out of the evaluated and passed counts", + async (guardrail_status) => { + const user = userEvent.setup(); + const data = makeGuardrailInformation({ ...skippedPreCall, guardrail_status }); + renderWithProviders(); - expect(screen.getByText(/0 guardrails evaluated/)).toBeInTheDocument(); - expect(screen.getByText(/0 Passed/)).toHaveClass("text-muted-foreground"); - expect(screen.getByText(/1 Skipped/)).toBeInTheDocument(); - const badge = screen.getByText("SKIPPED"); - expect(badge).toHaveClass("text-muted-foreground"); - expect(screen.queryByText("FAILED")).not.toBeInTheDocument(); - expect(screen.queryByText(/^T\+/)).not.toBeInTheDocument(); + expect(screen.getByText(/0 guardrails evaluated/)).toBeInTheDocument(); + expect(screen.getByText(/0 Passed/)).toHaveClass("text-muted-foreground"); + expect(screen.getByText(/1 Skipped/)).toBeInTheDocument(); + const badge = screen.getByText("SKIPPED"); + expect(badge).toHaveClass("text-muted-foreground"); + expect(screen.queryByText("FAILED")).not.toBeInTheDocument(); + expect(screen.queryByText(/^T\+/)).not.toBeInTheDocument(); - await user.click(screen.getByText("pii-rail")); - expect(screen.getByText("no scannable content after message scoping")).toBeInTheDocument(); - }); + await user.click(screen.getByText("pii-rail")); + expect(screen.getByText("no scannable content after message scoping")).toBeInTheDocument(); + }, + ); it("anchors the lifecycle timeline on timed entries when an untimed skipped entry sorts first", () => { const skipped = makeGuardrailInformation({ ...skippedPreCall, guardrail_name: "skipped-rail" }); diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx index c67d682a233..d171b7ff4b3 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx @@ -140,7 +140,7 @@ const getEntryOutcome = (entry: GuardrailInformation): EntryOutcome => { const status = (entry.guardrail_status ?? "").toLowerCase(); if (status === "success") return "passed"; if (status === "guardrail_flagged") return "flagged"; - if (status === "skipped") return "skipped"; + if (status === "skipped" || status === "not_run") return "skipped"; return "failed"; }; diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx index e052faa2228..e022d166fe0 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx @@ -705,7 +705,7 @@ const GUARDRAIL_JUMP_LINK_STYLE = { const isPassedStatus = (status: unknown) => status === "pass" || status === "passed" || status === "success"; const isFlaggedStatus = (status: unknown) => status === "flagged" || status === "guardrail_flagged"; -const isSkippedStatus = (status: unknown) => status === "skipped"; +const isSkippedStatus = (status: unknown) => status === "skipped" || status === "not_run"; const guardrailJumpLinkOutcome = (evaluated: unknown[]): keyof typeof GUARDRAIL_JUMP_LINK_STYLE => { if (evaluated.length === 0) return "skipped";