diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 035dce46d27..dca90e07421 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -210,7 +210,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): task_mappings=tool_call_task_mappings, ) - elif not images_to_check and not guardrail_to_apply.records_own_guardrail_information: + elif not guardrail_to_apply.records_own_guardrail_information: guardrail_to_apply.add_standard_logging_guardrail_information_to_request_data( guardrail_json_response="no scannable content after message scoping", request_data=data, diff --git a/litellm/proxy/compliance_checks.py b/litellm/proxy/compliance_checks.py index 18123156411..ef2d8fb6e20 100644 --- a/litellm/proxy/compliance_checks.py +++ b/litellm/proxy/compliance_checks.py @@ -11,7 +11,6 @@ from litellm.types.proxy.compliance_endpoints import ( ComplianceCheckRequest, ComplianceCheckResult, ) -from litellm.types.utils import UNEVALUATED_GUARDRAIL_STATUSES class ComplianceChecker: @@ -27,11 +26,7 @@ class ComplianceChecker: def __init__(self, data: ComplianceCheckRequest): self.data = data - self.guardrails = [ - g - for g in (data.guardrail_information or []) - if g.get("guardrail_status") not in UNEVALUATED_GUARDRAIL_STATUSES - ] + self.guardrails = [g for g in (data.guardrail_information or []) if g.get("guardrail_status") != "skipped"] def _get_guardrails_by_mode(self, mode: str) -> list[dict]: """ diff --git a/litellm/proxy/guardrails/usage_tracking.py b/litellm/proxy/guardrails/usage_tracking.py index 789b8febfbf..8a131fbfcde 100644 --- a/litellm/proxy/guardrails/usage_tracking.py +++ b/litellm/proxy/guardrails/usage_tracking.py @@ -27,7 +27,6 @@ from litellm.repositories.table_repositories import ( DailyGuardrailUsageUnitsRepository, SpendLogGuardrailIndexRepository, ) -from litellm.types.utils import UNEVALUATED_GUARDRAIL_STATUSES if TYPE_CHECKING: from prisma import types as prisma_types @@ -198,7 +197,7 @@ def guardrail_status_to_action(status: str | None) -> str: if not status: return "passed" s: Final = (status or "").lower() - if s in UNEVALUATED_GUARDRAIL_STATUSES: + if s == "skipped": return "skipped" if "intervened" in s or "block" in s: return "blocked" diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 6da44007fb4..cf302b3f27f 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3123,8 +3123,6 @@ GuardrailStatus = Literal[ "success", "guardrail_flagged", "guardrail_intervened", "guardrail_failed_to_respond", "not_run", "skipped" ] -UNEVALUATED_GUARDRAIL_STATUSES: Final[frozenset[GuardrailStatus]] = frozenset({"not_run", "skipped"}) - # Fields on a guardrail record whose values can quote the caller's prompt: the payload sent to the # guardrail, the provider response that echoes it back, and the two first-party hooks that inline # prompt substrings (``block_code_execution`` and ``litellm_content_filter``). Every other field diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 7bfd50fa6f8..c6d01db45de 100644 --- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1942,27 +1942,6 @@ class TestNoScannableContentRecordsSkipped: assert guardrail.last_inputs is not None assert all(e.get("guardrail_status") != "skipped" for e in self._recorded_entries(data)) - @pytest.mark.asyncio - async def test_image_only_content_is_not_reported_as_skipped(self): - """Images are only scanned alongside text, so an image-only request is a - pre-existing scan gap, not a message-scoping skip, and must not be labelled one""" - handler = OpenAIChatCompletionsHandler() - guardrail = MockGuardrail(guardrail_name="image-guardrail") - guardrail.skip_system_message_in_guardrail = True - data = { - "messages": [ - {"role": "system", "content": "SYSTEM-PROMPT"}, - { - "role": "user", - "content": [{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}], - }, - ] - } - - await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) - - assert self._recorded_entries(data) == [] - class TestBuildBlockSseChunks: """build_block_sse_chunks turns a streaming ModifyResponseException into 200 SSE chunks""" diff --git a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py index ae883d3eaeb..cb85eb8b5ec 100644 --- a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py +++ b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py @@ -350,17 +350,15 @@ async def test_zero_and_non_int_usage_counters_are_skipped(): @pytest.mark.asyncio -@pytest.mark.parametrize("status", ["skipped", "not_run"]) -async def test_skipped_entries_are_indexed_but_not_counted_as_evaluations(status: str): +async def test_skipped_entries_are_indexed_but_not_counted_as_evaluations(): """ LIT-6314 records a skipped entry when message scoping leaves a guardrail - nothing to scan (older spend logs spell it not_run). The guardrail never - evaluated the request, so counting it as a passed evaluation would inflate - daily pass rates; it still gets an index row so per-request drill-down - finds the spend log. + nothing to scan. The guardrail never evaluated the request, so counting it + as a passed evaluation would inflate daily pass rates; it still gets an + index row so per-request drill-down finds the spend log. """ prisma = _prisma() - logs = [_payload("r1", guardrail_status=status), _payload("r2")] + logs = [_payload("r1", guardrail_status="skipped"), _payload("r2")] await process_spend_logs_guardrail_usage(prisma, logs) diff --git a/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py index bafe608bff8..9587e3d95ee 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py @@ -594,15 +594,14 @@ class TestModeMatching: class TestSkippedGuardrails: """LIT-6314 logs a skipped entry for a guardrail that message scoping left nothing to scan.""" - @pytest.mark.parametrize("status", ["skipped", "not_run"]) - def test_skipped_alone_never_evidences_compliance(self, status: str): + def test_skipped_alone_never_evidences_compliance(self): data = ComplianceCheckRequest( request_id="req-601", user_id="user-1", model="gpt-4", timestamp="2026-02-17T00:00:00Z", guardrail_information=[ - {"guardrail_name": "pii_detection", "guardrail_status": status, "guardrail_mode": "pre_call"}, + {"guardrail_name": "pii_detection", "guardrail_status": "skipped", "guardrail_mode": "pre_call"}, ], ) results = {c.check_name: c.passed for c in ComplianceChecker(data).check_eu_ai_act()} diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx index 02a4c8008a6..740ef857690 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx @@ -68,25 +68,22 @@ describe("GuardrailViewer", () => { expect(screen.queryByText("FAILED")).not.toBeInTheDocument(); }); - it.each(["skipped", "not_run"])( - "renders %s as SKIPPED (muted) and keeps it out of the evaluated and passed counts", - async (guardrail_status) => { - const user = userEvent.setup(); - const data = makeGuardrailInformation({ ...skippedPreCall, guardrail_status }); - renderWithProviders(); + it("renders skipped as SKIPPED (muted) and keeps it out of the evaluated and passed counts", async () => { + const user = userEvent.setup(); + const data = makeGuardrailInformation(skippedPreCall); + renderWithProviders(); - expect(screen.getByText(/0 guardrails evaluated/)).toBeInTheDocument(); - expect(screen.getByText(/0 Passed/)).toHaveClass("text-muted-foreground"); - expect(screen.getByText(/1 Skipped/)).toBeInTheDocument(); - const badge = screen.getByText("SKIPPED"); - expect(badge).toHaveClass("text-muted-foreground"); - expect(screen.queryByText("FAILED")).not.toBeInTheDocument(); - expect(screen.queryByText(/^T\+/)).not.toBeInTheDocument(); + expect(screen.getByText(/0 guardrails evaluated/)).toBeInTheDocument(); + expect(screen.getByText(/0 Passed/)).toHaveClass("text-muted-foreground"); + expect(screen.getByText(/1 Skipped/)).toBeInTheDocument(); + const badge = screen.getByText("SKIPPED"); + expect(badge).toHaveClass("text-muted-foreground"); + expect(screen.queryByText("FAILED")).not.toBeInTheDocument(); + expect(screen.queryByText(/^T\+/)).not.toBeInTheDocument(); - await user.click(screen.getByText("pii-rail")); - expect(screen.getByText("no scannable content after message scoping")).toBeInTheDocument(); - }, - ); + await user.click(screen.getByText("pii-rail")); + expect(screen.getByText("no scannable content after message scoping")).toBeInTheDocument(); + }); it("anchors the lifecycle timeline on timed entries when an untimed skipped entry sorts first", () => { const skipped = makeGuardrailInformation({ ...skippedPreCall, guardrail_name: "skipped-rail" }); diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx index d171b7ff4b3..c67d682a233 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx @@ -140,7 +140,7 @@ const getEntryOutcome = (entry: GuardrailInformation): EntryOutcome => { const status = (entry.guardrail_status ?? "").toLowerCase(); if (status === "success") return "passed"; if (status === "guardrail_flagged") return "flagged"; - if (status === "skipped" || status === "not_run") return "skipped"; + if (status === "skipped") return "skipped"; return "failed"; }; diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx index e022d166fe0..e052faa2228 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx @@ -705,7 +705,7 @@ const GUARDRAIL_JUMP_LINK_STYLE = { const isPassedStatus = (status: unknown) => status === "pass" || status === "passed" || status === "success"; const isFlaggedStatus = (status: unknown) => status === "flagged" || status === "guardrail_flagged"; -const isSkippedStatus = (status: unknown) => status === "skipped" || status === "not_run"; +const isSkippedStatus = (status: unknown) => status === "skipped"; const guardrailJumpLinkOutcome = (evaluated: unknown[]): keyof typeof GUARDRAIL_JUMP_LINK_STYLE => { if (evaluated.length === 0) return "skipped";