diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index ee68f8f6546..01e14f2248d 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -213,6 +213,17 @@ class OpenAIChatCompletionsHandler(BaseTranslation): task_mappings=tool_call_task_mappings, ) + elif ( + not images_to_check + and not guardrail_to_apply.records_own_guardrail_information + and (not_run_reason := self._not_run_reason(messages)) is not None + ): + guardrail_to_apply.add_standard_logging_guardrail_information_to_request_data( + guardrail_json_response=not_run_reason, + request_data=data, + guardrail_status="not_run", + ) + verbose_proxy_logger.debug( "OpenAI Chat Completions: Processed input messages: %s", data.get("messages"), @@ -220,6 +231,28 @@ class OpenAIChatCompletionsHandler(BaseTranslation): return data + def _not_run_reason( + self, + messages: Sequence[dict[str, Any]], # mutable-ok: raw request messages consumed by _extract_inputs + ) -> str | None: + """Why nothing was scanned, or None when the only unscoped content is images, which this handler never scans.""" + texts: Final[list[str]] = [] # mutable-ok: filled by _extract_inputs + images: Final[list[str]] = [] # mutable-ok: filled by _extract_inputs + tool_calls: Final[list[ChatCompletionToolParam]] = [] # mutable-ok: filled by _extract_inputs + for msg_idx, message in enumerate(messages): + self._extract_inputs( + message=message, + msg_idx=msg_idx, + texts_to_check=texts, + images_to_check=images, + tool_calls_to_check=tool_calls, + text_task_mappings=[], # mutable-ok: required by _extract_inputs, unused here + tool_call_task_mappings=[], # mutable-ok: required by _extract_inputs, unused here + ) + if texts or tool_calls: + return "no scannable content after message scoping" + return None if images else "no scannable content" + def extract_request_tool_names(self, data: dict) -> list[str]: """Extract tool names from OpenAI chat completions request (tools[].function.name, functions[].name).""" names: Final[list[str]] = [] diff --git a/litellm/proxy/compliance_checks.py b/litellm/proxy/compliance_checks.py index ff311911742..9d2f2dc7c69 100644 --- a/litellm/proxy/compliance_checks.py +++ b/litellm/proxy/compliance_checks.py @@ -26,7 +26,7 @@ class ComplianceChecker: def __init__(self, data: ComplianceCheckRequest): self.data = data - self.guardrails = data.guardrail_information or [] + self.guardrails = tuple(g for g in data.guardrail_information or () if g.get("guardrail_status") != "not_run") def _get_guardrails_by_mode(self, mode: str) -> list[dict]: """ diff --git a/litellm/proxy/guardrails/usage_endpoints.py b/litellm/proxy/guardrails/usage_endpoints.py index 6259efb6654..556b6a4e919 100644 --- a/litellm/proxy/guardrails/usage_endpoints.py +++ b/litellm/proxy/guardrails/usage_endpoints.py @@ -42,7 +42,7 @@ if TYPE_CHECKING: router: Final = APIRouter() _EMPTY_UNITS: Final[Mapping[str, int]] = MappingProxyType({}) -_ACTION_SEVERITY: Final[Mapping[str, int]] = MappingProxyType({"passed": 0, "flagged": 1, "blocked": 2}) +_ACTION_SEVERITY: Final[Mapping[str, int]] = MappingProxyType({"not_run": 0, "passed": 1, "flagged": 2, "blocked": 3}) _T = TypeVar("_T") @@ -325,7 +325,7 @@ class UsageDetailResponse(BaseModel): class UsageLogEntry(BaseModel): id: str timestamp: str - action: str # blocked | passed | flagged + action: str # blocked | passed | flagged | not_run score: float | None latency_ms: float | None model: str | None diff --git a/litellm/proxy/guardrails/usage_tracking.py b/litellm/proxy/guardrails/usage_tracking.py index 797323794d2..7e11b69108b 100644 --- a/litellm/proxy/guardrails/usage_tracking.py +++ b/litellm/proxy/guardrails/usage_tracking.py @@ -193,10 +193,12 @@ async def _upsert_rows_with_retry( def guardrail_status_to_action(status: str | None) -> str: - """Map StandardLogging guardrail_status to blocked/passed/flagged.""" + """Map StandardLogging guardrail_status to blocked/passed/flagged/not_run.""" if not status: return "passed" s: Final = (status or "").lower() + if s == "not_run": + return "not_run" if "intervened" in s or "block" in s: return "blocked" if "flagged" in s or "fail" in s or "error" in s: @@ -354,37 +356,49 @@ async def process_spend_logs_guardrail_usage( "flagged_count": 0, } ) - index_rows: Final[list[dict[str, object]]] = [] + index_rows_by_key: Final[dict[tuple[str, str], dict[str, object]]] = {} for payload in logs_to_process: request_id = payload.get("request_id") start_time = _parse_payload_start_time(payload) - if not request_id or start_time is None: + if not isinstance(request_id, str) or not request_id or start_time is None: continue date_key = _date_str(start_time) - for entry in _parse_guardrail_info_from_payload(payload): - guardrail_id = entry.get("guardrail_id") or entry.get("guardrail_name") or "" - if not guardrail_id: + entries = _parse_guardrail_info_from_payload(payload) + ids_by_name = MappingProxyType( + { + e["guardrail_name"]: e["guardrail_id"] + for e in entries + if e.get("guardrail_id") and isinstance(e.get("guardrail_name"), str) and e["guardrail_name"] + } + ) + for entry in entries: + raw_name = entry.get("guardrail_name") + guardrail_name = raw_name if isinstance(raw_name, str) else "" + guardrail_id = entry.get("guardrail_id") or ids_by_name.get(guardrail_name) or guardrail_name + if not isinstance(guardrail_id, str) or not guardrail_id: continue - key = _MetricsKey(guardrail_id, date_key) - daily_guardrail[key]["requests_evaluated"] += 1 action = guardrail_status_to_action(entry.get("guardrail_status")) - if action == "passed": - daily_guardrail[key]["passed_count"] += 1 - elif action == "blocked": - daily_guardrail[key]["blocked_count"] += 1 - else: - daily_guardrail[key]["flagged_count"] += 1 + if action != "not_run": + key = _MetricsKey(guardrail_id, date_key) + daily_guardrail[key]["requests_evaluated"] += 1 + if action == "passed": + daily_guardrail[key]["passed_count"] += 1 + elif action == "blocked": + daily_guardrail[key]["blocked_count"] += 1 + else: + daily_guardrail[key]["flagged_count"] += 1 policy_id = entry.get("policy_id") - index_rows.append( - { + prior = index_rows_by_key.get((request_id, guardrail_id)) + if prior is None or (prior["policy_id"] is None and policy_id is not None): + index_rows_by_key[(request_id, guardrail_id)] = { "request_id": request_id, "guardrail_id": guardrail_id, "policy_id": policy_id, "start_time": start_time, } - ) + index_rows: Final = tuple(index_rows_by_key.values()) async with pending.lock: pending_metrics: Final = pending.metrics diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 8ee0e982aa0..cb884fb7cc1 100644 --- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1893,6 +1893,140 @@ class TestScanOnlyToolResults: assert data["messages"][4]["content"] == "and then?" +class TestNoScannableContentRecordsNotRun: + """LIT-6314: a guardrail whose scoping leaves nothing to scan must still persist an evaluation record""" + + def _system_only_data(self) -> dict: + return {"messages": [{"role": "system", "content": "SYSTEM-PROMPT"}]} + + def _recorded_entries(self, data: dict) -> list: + metadata = data.get("metadata") or data.get("litellm_metadata") or {} + return metadata.get("standard_logging_guardrail_information") or [] + + @pytest.mark.asyncio + async def test_skipped_scan_records_not_run_entry(self): + handler = OpenAIChatCompletionsHandler() + guardrail = MockGuardrail(guardrail_name="skip-system-guardrail") + guardrail.skip_system_message_in_guardrail = True + data = self._system_only_data() + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert guardrail.last_inputs is None, "nothing survived scoping, apply_guardrail must not run" + entries = self._recorded_entries(data) + assert len(entries) == 1 + assert entries[0]["guardrail_name"] == "skip-system-guardrail" + assert entries[0]["guardrail_status"] == "not_run" + assert entries[0]["guardrail_response"] == "no scannable content after message scoping" + + @pytest.mark.asyncio + @pytest.mark.parametrize("skip_system", [False, True]) + async def test_empty_content_does_not_blame_scoping(self, skip_system: bool): + handler = OpenAIChatCompletionsHandler() + guardrail = MockGuardrail(guardrail_name="unscoped-guardrail") + guardrail.skip_system_message_in_guardrail = skip_system + data = {"messages": [{"role": "user", "content": None}]} + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert guardrail.last_inputs is None + entries = self._recorded_entries(data) + assert len(entries) == 1 + assert entries[0]["guardrail_status"] == "not_run" + assert entries[0]["guardrail_response"] == "no scannable content" + + @pytest.mark.asyncio + async def test_self_recording_guardrail_is_left_alone(self): + handler = OpenAIChatCompletionsHandler() + guardrail = MockGuardrail(guardrail_name="self-recording-guardrail") + guardrail.skip_system_message_in_guardrail = True + guardrail.records_own_guardrail_information = True + data = self._system_only_data() + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert guardrail.last_inputs is None + assert self._recorded_entries(data) == [] + + @pytest.mark.asyncio + async def test_scannable_content_records_no_extra_entry(self): + handler = OpenAIChatCompletionsHandler() + guardrail = MockGuardrail(guardrail_name="normal-guardrail") + data = {"messages": [{"role": "user", "content": "hello"}]} + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert guardrail.last_inputs is not None + assert all(e.get("guardrail_status") != "not_run" for e in self._recorded_entries(data)) + + @pytest.mark.asyncio + async def test_image_only_content_is_not_reported_as_not_run(self): + """Images are only scanned alongside text, so an image-only request is a + pre-existing scan gap, not a message-scoping skip, and must not be labelled one""" + handler = OpenAIChatCompletionsHandler() + guardrail = MockGuardrail(guardrail_name="image-guardrail") + guardrail.skip_system_message_in_guardrail = True + data = { + "messages": [ + {"role": "system", "content": "SYSTEM-PROMPT"}, + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}], + }, + ] + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert self._recorded_entries(data) == [] + + @pytest.mark.asyncio + async def test_scoped_out_image_only_message_is_not_reported_as_not_run(self): + """An image in a skipped role must behave like any other image-only request""" + handler = OpenAIChatCompletionsHandler() + guardrail = MockGuardrail(guardrail_name="image-guardrail") + guardrail.skip_system_message_in_guardrail = True + data = { + "messages": [ + { + "role": "system", + "content": [{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}], + }, + ] + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert guardrail.last_inputs is None + assert self._recorded_entries(data) == [] + + @pytest.mark.asyncio + async def test_scoped_out_text_with_image_records_not_run(self): + """Scoping removed text too, so the skip is recorded even though an image sat beside it""" + handler = OpenAIChatCompletionsHandler() + guardrail = MockGuardrail(guardrail_name="image-guardrail") + guardrail.skip_system_message_in_guardrail = True + data = { + "messages": [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Describe this picture."}, + {"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}, + ], + }, + ] + } + + await handler.process_input_messages(data=data, guardrail_to_apply=guardrail) + + assert guardrail.last_inputs is None + entries = self._recorded_entries(data) + assert len(entries) == 1 + assert entries[0]["guardrail_status"] == "not_run" + assert entries[0]["guardrail_response"] == "no scannable content after message scoping" + + class ToolDroppingTextGuardrail(CustomGuardrail): """Answers one text per non-tool message it saw, the way a guardrail that filters tool rows out before scanning does, and hands back only texts.""" diff --git a/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py b/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py index 644b213d3a3..db87e12ac88 100644 --- a/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py +++ b/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py @@ -682,3 +682,67 @@ async def test_detail_prev_trend_query_is_bounded(): prev_wheres = [w for w in wheres if "lt" in w.get("date", {})] assert prev_wheres assert all("gte" in w["date"] for w in prev_wheres) + + +@pytest.mark.asyncio +async def test_logs_report_not_run_entries_as_not_run_not_passed(): + """LIT-6314: a guardrail that never scanned must not be reported as a pass in the drill-down.""" + index_row = MagicMock() + index_row.request_id = "req-nr" + index_row.guardrail_id = "db-1" + index_row.start_time = datetime(2026, 4, 22) + spend_log = MagicMock() + spend_log.request_id = "req-nr" + spend_log.model = "gpt-4o-mini" + spend_log.startTime = datetime(2026, 4, 22) + spend_log.metadata = { + "guardrail_information": [ + {"guardrail_name": "db-1", "guardrail_status": "not_run", "duration": 0.0}, + ] + } + prisma = _prisma(find_unique=_db_row(), index_find_many=[index_row]) + prisma.db.litellm_spendlogs.find_many = AsyncMock(return_value=[spend_log]) + handler = _config_handler() + p1, p2 = _patches(prisma, handler) + with p1, p2: + resp = await guardrails_usage_logs( + guardrail_id="db-1", + policy_id=None, + page=1, + page_size=50, + action=None, + start_date=START, + end_date=END, + user_api_key_dict=ADMIN, + ) + assert [log.action for log in resp.logs] == ["not_run"] + + +@pytest.mark.asyncio +async def test_logs_action_passed_filter_excludes_not_run_entries(): + """LIT-6314: filtering the drill-down for passes must not return unscanned requests.""" + index_row = MagicMock() + index_row.request_id = "req-nr" + index_row.guardrail_id = "db-1" + index_row.start_time = datetime(2026, 4, 22) + spend_log = MagicMock() + spend_log.request_id = "req-nr" + spend_log.model = "gpt-4o-mini" + spend_log.startTime = datetime(2026, 4, 22) + spend_log.metadata = {"guardrail_information": [{"guardrail_name": "db-1", "guardrail_status": "not_run"}]} + prisma = _prisma(find_unique=_db_row(), index_find_many=[index_row]) + prisma.db.litellm_spendlogs.find_many = AsyncMock(return_value=[spend_log]) + handler = _config_handler() + p1, p2 = _patches(prisma, handler) + with p1, p2: + resp = await guardrails_usage_logs( + guardrail_id="db-1", + policy_id=None, + page=1, + page_size=50, + action="passed", + start_date=START, + end_date=END, + user_api_key_dict=ADMIN, + ) + assert resp.logs == [] diff --git a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py index 85f22e1f307..69ec098b840 100644 --- a/tests/test_litellm/proxy/guardrails/test_usage_tracking.py +++ b/tests/test_litellm/proxy/guardrails/test_usage_tracking.py @@ -349,6 +349,95 @@ async def test_zero_and_non_int_usage_counters_are_skipped(): } +@pytest.mark.asyncio +async def test_not_run_entries_are_indexed_but_not_counted_as_evaluations(): + """ + LIT-6314 records a not_run entry when message scoping leaves a guardrail + nothing to scan. The guardrail never evaluated the request, so counting it + as a passed evaluation would inflate daily pass rates; it still gets an + index row so per-request drill-down finds the spend log. + """ + prisma = _prisma() + logs = [_payload("r1", guardrail_status="not_run"), _payload("r2")] + + await process_spend_logs_guardrail_usage(prisma, logs) + + metrics_create = prisma.db.litellm_dailyguardrailmetrics.upsert.call_args.kwargs["data"]["create"] + assert metrics_create["requests_evaluated"] == 1 + assert metrics_create["passed_count"] == 1 + index_rows = prisma.db.litellm_spendlogguardrailindex.create_many.call_args.kwargs["data"] + assert sorted(row["request_id"] for row in index_rows) == ["r1", "r2"] + + +@pytest.mark.asyncio +async def test_not_run_entry_shares_index_key_with_evaluated_sibling_of_same_name(): + """ + The not_run entry from the shared base guardrail carries only guardrail_name, + while the evaluated entry from the same guardrail (e.g. content filter on the + output of a logging_only run) carries its guardrail_id. Keying them differently + lists one request twice in the monitor, once as not_run and once as passed. + """ + prisma = _prisma() + payload = _payload("r1") + payload["metadata"] = json.dumps( + { + "guardrail_information": [ + {"guardrail_name": "cf", "guardrail_status": "not_run"}, + { + "guardrail_name": "cf", + "guardrail_id": "cf-uuid", + "policy_id": "pol-1", + "guardrail_status": "success", + }, + {"guardrail_name": "other", "guardrail_status": "not_run"}, + ] + } + ) + + await process_spend_logs_guardrail_usage(prisma, [payload]) + + index_rows = prisma.db.litellm_spendlogguardrailindex.create_many.call_args.kwargs["data"] + assert sorted((row["guardrail_id"], row["policy_id"]) for row in index_rows) == [ + ("cf-uuid", "pol-1"), + ("other", None), + ] + metrics_create = prisma.db.litellm_dailyguardrailmetrics.upsert.call_args.kwargs["data"]["create"] + assert (metrics_create["guardrail_id"], metrics_create["requests_evaluated"]) == ("cf-uuid", 1) + + +@pytest.mark.asyncio +async def test_malformed_not_run_entry_does_not_drop_the_batch(): + prisma = _prisma() + payload = _payload("r1") + payload["metadata"] = json.dumps( + { + "guardrail_information": [ + {"guardrail_name": ["not", "a", "string"], "guardrail_status": "success"}, + {"guardrail_name": "", "guardrail_id": "cf-uuid", "guardrail_status": "success"}, + {"guardrail_status": "success"}, + ] + } + ) + + await process_spend_logs_guardrail_usage(prisma, [payload]) + + index_rows = prisma.db.litellm_spendlogguardrailindex.create_many.call_args.kwargs["data"] + assert [row["guardrail_id"] for row in index_rows] == ["cf-uuid"] + metrics_create = prisma.db.litellm_dailyguardrailmetrics.upsert.call_args.kwargs["data"]["create"] + assert (metrics_create["guardrail_id"], metrics_create["requests_evaluated"]) == ("cf-uuid", 1) + + +@pytest.mark.asyncio +async def test_batch_of_only_not_run_entries_writes_no_metrics_row(): + prisma = _prisma() + + await process_spend_logs_guardrail_usage(prisma, [_payload("r1", guardrail_status="not_run")]) + + assert prisma.db.litellm_dailyguardrailmetrics.upsert.call_count == 0 + index_rows = prisma.db.litellm_spendlogguardrailindex.create_many.call_args.kwargs["data"] + assert [row["request_id"] for row in index_rows] == ["r1"] + + @pytest.mark.asyncio async def test_payload_without_request_id_is_skipped_like_the_metrics_path(): prisma = _prisma() diff --git a/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py index dcbe515d5de..8382a5ada96 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_compliance_endpoints.py @@ -2,10 +2,8 @@ Unit tests for compliance check endpoints (EU AI Act and GDPR). """ - import pytest - from litellm.proxy.compliance_checks import ComplianceChecker from litellm.types.proxy.compliance_endpoints import ComplianceCheckRequest @@ -591,3 +589,37 @@ class TestModeMatching: continue if matched: assert mode in _guaranteed_modes(g_mode), (g_mode, mode) + + +class TestNotRunGuardrails: + """LIT-6314 logs a not_run entry for a guardrail that message scoping left nothing to scan.""" + + def test_not_run_alone_never_evidences_compliance(self): + data = ComplianceCheckRequest( + request_id="req-601", + user_id="user-1", + model="gpt-4", + timestamp="2026-02-17T00:00:00Z", + guardrail_information=[ + {"guardrail_name": "pii_detection", "guardrail_status": "not_run", "guardrail_mode": "pre_call"}, + ], + ) + results = {c.check_name: c.passed for c in ComplianceChecker(data).check_eu_ai_act()} + assert results["Guardrails applied"] is False + assert results["Content screened before LLM"] is False + assert results["Audit record complete"] is False + + def test_not_run_sibling_does_not_fail_a_passing_request(self): + data = ComplianceCheckRequest( + request_id="req-602", + user_id="user-1", + model="gpt-4", + timestamp="2026-02-17T00:00:00Z", + pii_detected=True, + guardrail_information=[ + {"guardrail_name": "pii_detection", "guardrail_status": "success", "guardrail_mode": "pre_call"}, + {"guardrail_name": "system_only", "guardrail_status": "not_run", "guardrail_mode": "pre_call"}, + ], + ) + results = {c.check_name: c.passed for c in ComplianceChecker(data).check_gdpr()} + assert results["Sensitive data protected"] is True diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index e2a08a40bcb..773854d29e6 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -2322,7 +2322,7 @@ }, "src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx": { "no-nested-ternary": { - "count": 4 + "count": 3 } }, "src/components/view_logs/LogDetailsDrawer/LogDetailsDrawer.tsx": { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.tsx index 81c39258f67..86b596d4bcd 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.tsx @@ -57,7 +57,7 @@ export function GuardrailDetail({ guardrailId, onBack, accessToken = null, start return list.map((l: Record) => ({ id: l.id as string, timestamp: l.timestamp as string, - action: l.action as "blocked" | "passed" | "flagged", + action: l.action as LogEntry["action"], score: l.score as number | undefined, model: l.model as string | undefined, input_snippet: l.input_snippet as string | undefined, diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.test.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.test.tsx index ab91e10c2fd..083b1e5f3e2 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.test.tsx +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.test.tsx @@ -2,7 +2,7 @@ import userEvent from "@testing-library/user-event"; import React from "react"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import { renderWithProviders, screen, testQueryClient, waitFor } from "../../../tests/test-utils"; +import { renderWithProviders, screen, testQueryClient, waitFor, within } from "../../../tests/test-utils"; import type { LogEntry as SpendLogEntry } from "@/components/view_logs/columns"; import { LogViewer } from "./LogViewer"; @@ -95,3 +95,16 @@ describe("GuardrailsMonitor LogViewer drawer", () => { }); }); }); + +describe("GuardrailsMonitor LogViewer not_run rows", () => { + it("renders a not_run log as a neutral Not run badge instead of a pass or failure", () => { + renderWithProviders( + , + ); + + const row = screen.getByRole("button", { name: /system prompt only/ }); + expect(within(row).getByText("Not run")).toHaveClass("text-muted-foreground"); + expect(within(row).queryByText("Passed")).not.toBeInTheDocument(); + expect(within(row).queryByText("Blocked")).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.tsx index 0703c94c2ed..2abd699ba86 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.tsx +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.tsx @@ -1,4 +1,4 @@ -import { CircleCheck, ChevronDown, TriangleAlert, X } from "lucide-react"; +import { CircleCheck, ChevronDown, MinusCircle, TriangleAlert, X } from "lucide-react"; import { useQuery } from "@tanstack/react-query"; import moment from "moment"; import React, { useState } from "react"; @@ -10,9 +10,16 @@ import type { LogEntry as ViewLogsLogEntry } from "@/components/view_logs/column import type { LogEntry } from "./mockData"; const actionConfig: Record< - "blocked" | "passed" | "flagged", + "blocked" | "passed" | "flagged" | "not_run", { icon: React.ElementType; color: string; bg: string; border: string; label: string } > = { + not_run: { + icon: MinusCircle, + color: "text-muted-foreground", + bg: "bg-muted", + border: "border-border", + label: "Not run", + }, blocked: { icon: X, color: "text-destructive", diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts b/ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts index 2b42f7907f1..591d5cd3edd 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts @@ -10,7 +10,7 @@ export interface LogEntry { input_snippet?: string; output_snippet?: string; score?: number; - action: "blocked" | "passed" | "flagged"; + action: "blocked" | "passed" | "flagged" | "not_run"; model?: string; reason?: string; latency_ms?: number; diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx index aabac50a661..7f343211596 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.test.tsx @@ -3,6 +3,7 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import userEvent from "@testing-library/user-event"; import { renderWithProviders, screen, waitFor } from "../../../../tests/test-utils"; import { + GuardrailInformation, makeBedrockResponse, makeEntity, makeGuardrailInformation, @@ -14,6 +15,24 @@ import GuardrailViewer from "@/components/view_logs/GuardrailViewer/GuardrailVie const PresidioPath = "@/components/view_logs/GuardrailViewer/PresidioDetectedEntities"; const BedrockPath = "@/components/view_logs/GuardrailViewer/BedrockGuardrailDetails"; +const skippedPreCall: Partial = { + guardrail_status: "not_run", + guardrail_mode: "pre_call", + guardrail_response: "no scannable content after message scoping", + start_time: null, + end_time: null, + duration: null, +}; + +const ranPostCall: Partial = { + guardrail_name: "ran-rail", + guardrail_status: "success", + guardrail_mode: "post_call", + start_time: 1_700_000_000, + end_time: 1_700_000_000.25, + duration: 0.25, +}; + describe("GuardrailViewer", () => { beforeEach(() => { vi.resetModules(); @@ -49,6 +68,36 @@ describe("GuardrailViewer", () => { expect(screen.queryByText("FAILED")).not.toBeInTheDocument(); }); + it("renders not_run as NOT RUN (muted) and keeps it out of the evaluated and passed counts", async () => { + const user = userEvent.setup(); + const data = makeGuardrailInformation(skippedPreCall); + renderWithProviders(); + + expect(screen.getByText(/0 guardrails evaluated/)).toBeInTheDocument(); + expect(screen.getByText(/0 Passed/)).toHaveClass("text-muted-foreground"); + expect(screen.getByText(/1 Not run/)).toBeInTheDocument(); + const badge = screen.getByText("NOT RUN"); + expect(badge).toHaveClass("text-muted-foreground"); + expect(screen.queryByText("FAILED")).not.toBeInTheDocument(); + expect(screen.queryByText(/^T\+/)).not.toBeInTheDocument(); + + await user.click(screen.getByText("pii-rail")); + expect(screen.getByText("no scannable content after message scoping")).toBeInTheDocument(); + }); + + it("anchors the lifecycle timeline on timed entries when an untimed not_run entry sorts first", () => { + const skipped = makeGuardrailInformation({ ...skippedPreCall, guardrail_name: "skipped-rail" }); + const ran = makeGuardrailInformation(ranPostCall); + renderWithProviders(); + + expect(screen.getByText(/1 guardrail evaluated/)).toBeInTheDocument(); + expect(screen.getByText("Request received").parentElement).toHaveTextContent("T+0ms"); + expect(screen.getByText(/Post-call guardrail: ran-rail/).parentElement).toHaveTextContent("T+250ms"); + expect(screen.getByText("Response returned").parentElement).toHaveTextContent("T+251ms"); + expect(screen.queryByText(/Pre-call guardrail: skipped-rail/)).not.toBeInTheDocument(); + expect(screen.getByText("—")).toBeInTheDocument(); + }); + it("calculates and displays masked entity totals", async () => { const user = userEvent.setup(); const data = makeGuardrailInformation({ diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx index 271b8f6ce05..1de0e3878b2 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/GuardrailViewer.tsx @@ -38,9 +38,9 @@ interface MatchDetail { } interface GuardrailInformation { - duration: number; - end_time: number; - start_time: number; + duration: number | null; + end_time: number | null; + start_time: number | null; guardrail_mode: string | string[] | Record | null; guardrail_name: string; guardrail_status: string; @@ -121,7 +121,8 @@ const formatMode = (mode: GuardrailInformation["guardrail_mode"]): string => { return s.replace(/_/g, "-").toUpperCase(); }; -const formatDurationMs = (seconds: number): string => { +const formatDurationMs = (seconds: number | null): string => { + if (seconds == null) return "—"; const ms = Math.round(seconds * 1000); return `${ms}ms`; }; @@ -133,12 +134,13 @@ const getTotalMasked = (entry: GuardrailInformation): number => { ); }; -type EntryOutcome = "passed" | "flagged" | "failed"; +type EntryOutcome = "passed" | "flagged" | "failed" | "not_run"; const getEntryOutcome = (entry: GuardrailInformation): EntryOutcome => { const status = (entry.guardrail_status ?? "").toLowerCase(); if (status === "success") return "passed"; if (status === "guardrail_flagged") return "flagged"; + if (status === "not_run") return "not_run"; return "failed"; }; @@ -148,12 +150,21 @@ const OUTCOME_LABEL: Record = { passed: "PASSED", flagged: "FLAGGED", failed: "FAILED", + not_run: "NOT RUN", }; const OUTCOME_BADGE_CLASS: Record = { passed: "bg-success/15 text-success border border-success/20", flagged: "bg-warning/15 text-warning border border-warning/20", failed: "bg-destructive/15 text-destructive border border-destructive/20", + not_run: "bg-muted text-muted-foreground border border-border", +}; + +const getHeaderOutcome = (counts: { evaluated: number; passed: number; flagged: number }): EntryOutcome => { + if (counts.evaluated === 0) return "not_run"; + if (counts.passed === counts.evaluated) return "passed"; + if (counts.passed + counts.flagged === counts.evaluated) return "flagged"; + return "failed"; }; const getRiskColor = (score: number): string => { @@ -231,6 +242,7 @@ const FlagCircleIcon = ({ className }: { className?: string }) => ( const OutcomeIcon = ({ outcome }: { outcome: EntryOutcome }) => { if (outcome === "passed") return ; if (outcome === "flagged") return ; + if (outcome === "not_run") return ; return ; }; @@ -353,8 +365,13 @@ interface TimelineEntry { outcome?: EntryOutcome; } +type TimedGuardrailInformation = GuardrailInformation & { start_time: number; end_time: number }; + +const isTimed = (e: GuardrailInformation): e is TimedGuardrailInformation => + typeof e.start_time === "number" && typeof e.end_time === "number"; + const RequestLifecycle = ({ entries }: { entries: GuardrailInformation[] }) => { - const sorted = useMemo(() => [...entries].sort((a, b) => (a.start_time ?? 0) - (b.start_time ?? 0)), [entries]); + const sorted = useMemo(() => entries.filter(isTimed).sort((a, b) => a.start_time - b.start_time), [entries]); const timeline = useMemo(() => { if (sorted.length === 0) return []; @@ -658,6 +675,10 @@ const EvaluationCard = ({ entry }: { entry: GuardrailInformation }) => { )} + {outcome === "not_run" && typeof guardrailResponse === "string" && ( +

{guardrailResponse}

+ )} + {/* Provider-specific details */} {guardrailProvider === "presidio" && presidioEntities.length > 0 && (
@@ -696,12 +717,10 @@ const GuardrailViewer = ({ data, accessToken, logEntry }: GuardrailViewerProps) const passedCount = guardrailEntries.filter(isEntrySuccess).length; const flaggedCount = guardrailEntries.filter((e) => getEntryOutcome(e) === "flagged").length; - const allPassed = passedCount === guardrailEntries.length; - const headerOutcome: EntryOutcome = allPassed - ? "passed" - : passedCount + flaggedCount === guardrailEntries.length - ? "flagged" - : "failed"; + const notRunCount = guardrailEntries.filter((e) => getEntryOutcome(e) === "not_run").length; + const evaluatedCount = guardrailEntries.length - notRunCount; + const allPassed = evaluatedCount > 0 && passedCount === evaluatedCount; + const headerOutcome = getHeaderOutcome({ evaluated: evaluatedCount, passed: passedCount, flagged: flaggedCount }); const totalOverheadMs = useMemo(() => { return Math.round(guardrailEntries.reduce((sum, e) => sum + (e.duration ?? 0), 0) * 1000); @@ -733,7 +752,7 @@ const GuardrailViewer = ({ data, accessToken, logEntry }: GuardrailViewerProps)

Guardrails & Policy Compliance

- {guardrailEntries.length} guardrail{guardrailEntries.length !== 1 ? "s" : ""} evaluated + {evaluatedCount} guardrail{evaluatedCount !== 1 ? "s" : ""} evaluated | )} + {notRunCount > 0 && ( + + {notRunCount} Not run + + )}
diff --git a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/__tests__/fixtures.ts b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/__tests__/fixtures.ts index fe27428283d..ab121adf6b4 100644 --- a/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/__tests__/fixtures.ts +++ b/ui/litellm-dashboard/src/components/view_logs/GuardrailViewer/__tests__/fixtures.ts @@ -20,13 +20,13 @@ export interface GuardrailEntity { } export interface GuardrailInformation { - duration: number; - end_time: number; - start_time: number; + duration: number | null; + end_time: number | null; + start_time: number | null; guardrail_mode: string | string[] | Record | null; guardrail_name: string; guardrail_status: string; - guardrail_response: GuardrailEntity[] | BedrockGuardrailResponse; + guardrail_response: GuardrailEntity[] | BedrockGuardrailResponse | string; masked_entity_count: Record; guardrail_usage?: Record; guardrail_cost?: number; diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.integration.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.integration.test.tsx index 721525268b3..3637b6c55a3 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.integration.test.tsx @@ -635,4 +635,23 @@ describe("GuardrailJumpLink", () => { expect(pill).toHaveClass(expectedClass); expect(pill).toHaveTextContent(glyph); }); + + it.each([ + [["success", "not_run"], "text-success", "\u2713"], + [["guardrail_intervened", "not_run"], "text-destructive", "\u2717"], + ])("ignores not_run when styling %j as %s", (statuses, expectedClass, glyph) => { + render( ({ guardrail_status: s }))} />); + + const pill = screen.getByText(/1 guardrail evaluated, 1 not run/); + expect(pill).toHaveClass(expectedClass); + expect(pill).toHaveTextContent(glyph); + }); + + it("renders an all not_run request as neutral rather than passed", () => { + render(); + + const pill = screen.getByText(/0 guardrails evaluated, 1 not run/); + expect(pill).toHaveClass("text-muted-foreground"); + expect(pill).not.toHaveTextContent("\u2713"); + }); }); diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx index b5a3412a96f..30cd96935f7 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx @@ -700,20 +700,25 @@ const GUARDRAIL_JUMP_LINK_STYLE = { passed: { className: "border border-success/20 bg-success/10 text-success", glyph: "\u2713" }, flagged: { className: "border border-warning/20 bg-warning/10 text-warning", glyph: "\u26A0" }, failed: { className: "border border-destructive/20 bg-destructive/10 text-destructive", glyph: "\u2717" }, + not_run: { className: "border border-border bg-muted text-muted-foreground", glyph: "\u2013" }, } as const; const isPassedStatus = (status: unknown) => status === "pass" || status === "passed" || status === "success"; const isFlaggedStatus = (status: unknown) => status === "flagged" || status === "guardrail_flagged"; +const isNotRunStatus = (status: unknown) => status === "not_run"; -const guardrailJumpLinkOutcome = (statuses: unknown[]): keyof typeof GUARDRAIL_JUMP_LINK_STYLE => { - if (statuses.every(isPassedStatus)) return "passed"; - if (statuses.every((s) => isPassedStatus(s) || isFlaggedStatus(s))) return "flagged"; +const guardrailJumpLinkOutcome = (evaluated: unknown[]): keyof typeof GUARDRAIL_JUMP_LINK_STYLE => { + if (evaluated.length === 0) return "not_run"; + if (evaluated.every(isPassedStatus)) return "passed"; + if (evaluated.every((s) => isPassedStatus(s) || isFlaggedStatus(s))) return "flagged"; return "failed"; }; export function GuardrailJumpLink({ guardrailEntries }: { guardrailEntries: any[] }) { - const outcome = guardrailJumpLinkOutcome(guardrailEntries.map((e) => e?.guardrail_status || e?.status)); - const { className, glyph } = GUARDRAIL_JUMP_LINK_STYLE[outcome]; + const statuses = guardrailEntries.map((e) => e?.guardrail_status || e?.status); + const evaluated = statuses.filter((s) => !isNotRunStatus(s)); + const notRunCount = statuses.length - evaluated.length; + const { className, glyph } = GUARDRAIL_JUMP_LINK_STYLE[guardrailJumpLinkOutcome(evaluated)]; const handleClick = () => { const el = document.getElementById("guardrail-section"); @@ -736,8 +741,9 @@ export function GuardrailJumpLink({ guardrailEntries }: { guardrailEntries: any[ fontWeight: 500, }} > - {glyph} {guardrailEntries.length} guardrail - {guardrailEntries.length !== 1 ? "s" : ""} evaluated + {glyph} {evaluated.length} guardrail + {evaluated.length !== 1 ? "s" : ""} evaluated + {notRunCount > 0 ? `, ${notRunCount} not run` : ""} {"\u2193"}