From 2dc8dad29dedc2d33866efbefb53cadac50cbb70 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 13 Jul 2026 21:32:01 -0700 Subject: [PATCH] feat(guardrails): type the guardrails-monitor usage API end to end The three /guardrails/usage/* routes already carried a response_model, but five fields were declared loosely: status, trend and action were bare `str`, and chart / time_series were `List[Dict[str, Any]]`. Every producer only ever emits a closed set (the models said so in comments), so the generated OpenAPI types came out as `string` and `{[key: string]: unknown}[]` and the dashboard had to re-narrow them by hand. Narrow the models to Literal unions and give chart / time_series real UsageChartPoint / UsageTimeSeriesPoint models. This writes down what the code already does; it is not a behavior change. The same models back /policies/usage/overview, so both snapshot fragments that describe them are regenerated. On the dashboard, move the three call sites onto the typed $api client via useGuardrailsUsageOverview / useGuardrailUsageDetail / useGuardrailUsageLogs and delete the hand-written types in mockData.ts, which carried three fields the backend never returned (p95Latency, falsePositiveRate, falseNegativeRate) plus a dead GuardrailDetailRecord. The now-orphaned networking.tsx wrappers go with them. This also fixes a live bug: the logs query key omitted startDate/endDate while the fetcher closed over them, so the logs table served stale rows after a date-range change. $api derives the key from the request, so the whole class of bug is gone. --- litellm/proxy/_lazy_openapi_snapshot.json | 125 ++++++++- litellm/proxy/guardrails/usage_endpoints.py | 69 +++-- .../proxy/guardrails/test_usage_endpoints.py | 258 ++++++++++++++++++ ui/litellm-dashboard/eslint-suppressions.json | 2 +- .../_components/GuardrailDetail.tsx | 85 ++---- .../GuardrailsMonitorView.test.tsx | 92 +++++-- .../_components/GuardrailsOverview.tsx | 77 ++---- .../_components/ScoreChart.tsx | 3 +- .../useGuardrailUsageDetail.test.ts | 50 ++++ .../useGuardrailUsageDetail.ts | 21 ++ .../useGuardrailUsageLogs.test.ts | 79 ++++++ .../useGuardrailUsageLogs.ts | 42 +++ .../useGuardrailsUsageOverview.test.ts | 44 +++ .../useGuardrailsUsageOverview.ts | 19 ++ .../GuardrailsMonitor/LogViewer.tsx | 22 +- .../components/GuardrailsMonitor/mockData.ts | 50 ---- .../src/components/ToolDetail.tsx | 5 +- .../src/components/networking.tsx | 99 ------- ui/litellm-dashboard/src/lib/http/schema.d.ts | 63 +++-- 19 files changed, 864 insertions(+), 341 deletions(-) create mode 100644 tests/test_litellm/proxy/guardrails/test_usage_endpoints.py create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrailsMonitor/useGuardrailUsageDetail.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrailsMonitor/useGuardrailUsageDetail.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrailsMonitor/useGuardrailUsageLogs.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrailsMonitor/useGuardrailUsageLogs.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrailsMonitor/useGuardrailsUsageOverview.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrailsMonitor/useGuardrailsUsageOverview.ts delete mode 100644 ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 27cdc483d4a..e068d761aba 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -11070,6 +11070,29 @@ "title": "UpdateGuardrailRequest", "type": "object" }, + "UsageChartPoint": { + "properties": { + "blocked": { + "title": "Blocked", + "type": "integer" + }, + "date": { + "title": "Date", + "type": "string" + }, + "passed": { + "title": "Passed", + "type": "integer" + } + }, + "required": [ + "date", + "passed", + "blocked" + ], + "title": "UsageChartPoint", + "type": "object" + }, "UsageDetailResponse": { "properties": { "avgLatency": { @@ -11126,18 +11149,27 @@ "type": "integer" }, "status": { + "enum": [ + "healthy", + "warning", + "critical" + ], "title": "Status", "type": "string" }, "time_series": { "items": { - "additionalProperties": true, - "type": "object" + "$ref": "#/components/schemas/UsageTimeSeriesPoint" }, "title": "Time Series", "type": "array" }, "trend": { + "enum": [ + "up", + "down", + "stable" + ], "title": "Trend", "type": "string" }, @@ -11166,6 +11198,11 @@ "UsageLogEntry": { "properties": { "action": { + "enum": [ + "passed", + "blocked", + "flagged" + ], "title": "Action", "type": "string" }, @@ -11293,8 +11330,7 @@ "properties": { "chart": { "items": { - "additionalProperties": true, - "type": "object" + "$ref": "#/components/schemas/UsageChartPoint" }, "title": "Chart", "type": "array" @@ -11374,10 +11410,20 @@ "type": "integer" }, "status": { + "enum": [ + "healthy", + "warning", + "critical" + ], "title": "Status", "type": "string" }, "trend": { + "enum": [ + "up", + "down", + "stable" + ], "title": "Trend", "type": "string" }, @@ -11401,6 +11447,41 @@ "title": "UsageOverviewRow", "type": "object" }, + "UsageTimeSeriesPoint": { + "properties": { + "blocked": { + "title": "Blocked", + "type": "integer" + }, + "date": { + "title": "Date", + "type": "string" + }, + "passed": { + "title": "Passed", + "type": "integer" + }, + "score": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Score" + } + }, + "required": [ + "date", + "passed", + "blocked", + "score" + ], + "title": "UsageTimeSeriesPoint", + "type": "object" + }, "ValidationError": { "properties": { "loc": { @@ -21117,12 +21198,34 @@ "title": "PolicyVersionStatusUpdateRequest", "type": "object" }, + "UsageChartPoint": { + "properties": { + "blocked": { + "title": "Blocked", + "type": "integer" + }, + "date": { + "title": "Date", + "type": "string" + }, + "passed": { + "title": "Passed", + "type": "integer" + } + }, + "required": [ + "date", + "passed", + "blocked" + ], + "title": "UsageChartPoint", + "type": "object" + }, "UsageOverviewResponse": { "properties": { "chart": { "items": { - "additionalProperties": true, - "type": "object" + "$ref": "#/components/schemas/UsageChartPoint" }, "title": "Chart", "type": "array" @@ -21202,10 +21305,20 @@ "type": "integer" }, "status": { + "enum": [ + "healthy", + "warning", + "critical" + ], "title": "Status", "type": "string" }, "trend": { + "enum": [ + "up", + "down", + "stable" + ], "title": "Trend", "type": "string" }, diff --git a/litellm/proxy/guardrails/usage_endpoints.py b/litellm/proxy/guardrails/usage_endpoints.py index e03bdbb95d2..1272331d9b0 100644 --- a/litellm/proxy/guardrails/usage_endpoints.py +++ b/litellm/proxy/guardrails/usage_endpoints.py @@ -5,7 +5,7 @@ GET /guardrails/usage/overview, /guardrails/usage/detail/:id, /guardrails/usage/ import json from datetime import datetime, timedelta, timezone -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Literal, Optional from fastapi import APIRouter, Depends, Query from pydantic import BaseModel @@ -26,6 +26,20 @@ router = APIRouter() # --- Response models --- +GuardrailStatus = Literal["healthy", "warning", "critical"] +GuardrailTrend = Literal["up", "down", "stable"] +GuardrailAction = Literal["passed", "blocked", "flagged"] + + +class UsageChartPoint(BaseModel): + date: str + passed: int + blocked: int + + +class UsageTimeSeriesPoint(UsageChartPoint): + score: float | None + class UsageOverviewRow(BaseModel): id: str @@ -36,13 +50,13 @@ class UsageOverviewRow(BaseModel): failRate: float avgScore: Optional[float] avgLatency: Optional[float] - status: str # healthy | warning | critical - trend: str # up | down | stable + status: GuardrailStatus + trend: GuardrailTrend class UsageOverviewResponse(BaseModel): rows: List[UsageOverviewRow] - chart: List[Dict[str, Any]] # [{ date, passed, blocked }] + chart: List[UsageChartPoint] totalRequests: int totalBlocked: int passRate: float @@ -57,16 +71,16 @@ class UsageDetailResponse(BaseModel): failRate: float avgScore: Optional[float] avgLatency: Optional[float] - status: str - trend: str + status: GuardrailStatus + trend: GuardrailTrend description: Optional[str] - time_series: List[Dict[str, Any]] + time_series: List[UsageTimeSeriesPoint] class UsageLogEntry(BaseModel): id: str timestamp: str - action: str # blocked | passed | flagged + action: GuardrailAction score: Optional[float] latency_ms: Optional[float] model: Optional[str] @@ -82,7 +96,7 @@ class UsageLogsResponse(BaseModel): page_size: int -def _status_from_fail_rate(fail_rate: float) -> str: +def _status_from_fail_rate(fail_rate: float) -> GuardrailStatus: if fail_rate > 15: return "critical" if fail_rate > 5: @@ -90,7 +104,7 @@ def _status_from_fail_rate(fail_rate: float) -> str: return "healthy" -def _trend_from_comparison(current_fail: float, previous_fail: float) -> str: +def _trend_from_comparison(current_fail: float, previous_fail: float) -> GuardrailTrend: if previous_fail <= 0: return "stable" diff = current_fail - previous_fail @@ -126,7 +140,7 @@ def _prev_fail_rates(metrics_prev: Any, id_attr: str) -> Dict[str, float]: return {gid: (100.0 * v["blocked"] / v["req"]) if v["req"] else 0.0 for gid, v in prev_agg_raw.items()} -def _chart_from_metrics(metrics: Any) -> List[Dict[str, Any]]: +def _chart_from_metrics(metrics: Any) -> List[UsageChartPoint]: chart_by_date: Dict[str, Dict[str, int]] = {} for m in metrics: d = m.date @@ -134,7 +148,7 @@ def _chart_from_metrics(metrics: Any) -> List[Dict[str, Any]]: chart_by_date[d] = {"passed": 0, "blocked": 0} chart_by_date[d]["passed"] += int(m.passed_count or 0) chart_by_date[d]["blocked"] += int(m.blocked_count or 0) - return [{"date": d, "passed": v["passed"], "blocked": v["blocked"]} for d, v in sorted(chart_by_date.items())] + return [UsageChartPoint(date=d, passed=v["passed"], blocked=v["blocked"]) for d, v in sorted(chart_by_date.items())] def _get_guardrail_attrs(g: Any) -> tuple[Any, str]: @@ -356,16 +370,9 @@ async def guardrails_usage_detail( trend = _trend_from_comparison(fail_rate, prev_fail) # Aggregate by date in case metrics exist under both UUID and logical name - ts_by_date: Dict[str, Dict[str, Any]] = {} - for m in metrics: - d = m.date - if d not in ts_by_date: - ts_by_date[d] = {"passed": 0, "blocked": 0} - ts_by_date[d]["passed"] += int(m.passed_count or 0) - ts_by_date[d]["blocked"] += int(m.blocked_count or 0) time_series = [ - {"date": d, "passed": v["passed"], "blocked": v["blocked"], "score": None} - for d, v in sorted(ts_by_date.items()) + UsageTimeSeriesPoint(date=p.date, passed=p.passed, blocked=p.blocked, score=None) + for p in _chart_from_metrics(metrics) ] _litellm_params = getattr(guardrail, "litellm_params", None) or ( guardrail.get("litellm_params") if isinstance(guardrail, dict) else None @@ -422,7 +429,18 @@ def _build_usage_logs_where( return where -def _usage_log_entry_from_row(r: Any, sl: Any, action_filter: Optional[str]) -> Optional[UsageLogEntry]: +def _action_from_guardrail_entry(entry: dict[str, Any] | None) -> GuardrailAction: + if not entry: + return "passed" + st = (entry.get("guardrail_status") or "").lower() + if "intervened" in st or "block" in st: + return "blocked" + if "fail" in st or "error" in st: + return "flagged" + return "passed" + + +def _usage_log_entry_from_row(r: Any, sl: Any, action_filter: str | None) -> UsageLogEntry | None: meta = sl.metadata if isinstance(meta, str): try: @@ -435,16 +453,11 @@ def _usage_log_entry_from_row(r: Any, sl: Any, action_filter: Optional[str]) -> if (gi.get("guardrail_id") or gi.get("guardrail_name")) == r.guardrail_id: entry_for_guardrail = gi break - action_val = "passed" + action_val = _action_from_guardrail_entry(entry_for_guardrail) score_val = None latency_val = None reason_val = None if entry_for_guardrail: - st = (entry_for_guardrail.get("guardrail_status") or "").lower() - if "intervened" in st or "block" in st: - action_val = "blocked" - elif "fail" in st or "error" in st: - action_val = "flagged" duration = entry_for_guardrail.get("duration") if duration is not None: latency_val = round(float(duration) * 1000, 0) diff --git a/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py b/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py new file mode 100644 index 00000000000..7fe0393abbc --- /dev/null +++ b/tests/test_litellm/proxy/guardrails/test_usage_endpoints.py @@ -0,0 +1,258 @@ +from types import SimpleNamespace + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from pydantic import ValidationError + +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.guardrails.usage_endpoints import ( + UsageChartPoint, + UsageDetailResponse, + UsageLogEntry, + UsageOverviewRow, + UsageTimeSeriesPoint, + _action_from_guardrail_entry, + _chart_from_metrics, + _status_from_fail_rate, + _trend_from_comparison, + router, +) + + +def _metric(date: str, passed: int, blocked: int) -> SimpleNamespace: + return SimpleNamespace(date=date, passed_count=passed, blocked_count=blocked, requests_evaluated=passed + blocked) + + +class TestStatusFromFailRate: + @pytest.mark.parametrize( + "fail_rate, expected", + [ + (0.0, "healthy"), + (5.0, "healthy"), + (5.1, "warning"), + (15.0, "warning"), + (15.1, "critical"), + (100.0, "critical"), + ], + ) + def test_boundaries(self, fail_rate: float, expected: str) -> None: + assert _status_from_fail_rate(fail_rate) == expected + + +class TestTrendFromComparison: + @pytest.mark.parametrize( + "current, previous, expected", + [ + (10.0, 0.0, "stable"), + (10.0, -1.0, "stable"), + (10.0, 9.5, "stable"), + (10.0, 9.4, "up"), + (10.0, 10.5, "stable"), + (10.0, 10.6, "down"), + ], + ) + def test_boundaries(self, current: float, previous: float, expected: str) -> None: + assert _trend_from_comparison(current, previous) == expected + + +class TestActionFromGuardrailEntry: + @pytest.mark.parametrize( + "status, expected", + [ + ("guardrail_intervened", "blocked"), + ("BLOCKED", "blocked"), + ("guardrail_failed_to_run", "flagged"), + ("ERROR", "flagged"), + ("success", "passed"), + ("", "passed"), + ], + ) + def test_maps_guardrail_status(self, status: str, expected: str) -> None: + assert _action_from_guardrail_entry({"guardrail_status": status}) == expected + + def test_missing_entry_is_passed(self) -> None: + assert _action_from_guardrail_entry(None) == "passed" + + def test_null_status_is_passed(self) -> None: + assert _action_from_guardrail_entry({"guardrail_status": None}) == "passed" + + def test_blocked_takes_precedence_over_failed(self) -> None: + assert _action_from_guardrail_entry({"guardrail_status": "blocked_after_error"}) == "blocked" + + +class TestChartFromMetrics: + def test_aggregates_duplicate_dates_and_sorts(self) -> None: + chart = _chart_from_metrics( + [ + _metric("2026-07-03", passed=1, blocked=2), + _metric("2026-07-01", passed=3, blocked=4), + _metric("2026-07-01", passed=5, blocked=6), + ] + ) + + assert chart == [ + UsageChartPoint(date="2026-07-01", passed=8, blocked=10), + UsageChartPoint(date="2026-07-03", passed=1, blocked=2), + ] + + def test_null_counts_are_zero(self) -> None: + metric = SimpleNamespace(date="2026-07-01", passed_count=None, blocked_count=None, requests_evaluated=None) + + assert _chart_from_metrics([metric]) == [UsageChartPoint(date="2026-07-01", passed=0, blocked=0)] + + def test_empty_metrics(self) -> None: + assert _chart_from_metrics([]) == [] + + +class TestResponseContractIsNarrow: + """The dashboard indexes record lookups by these literals, so the schema must not widen back to `str`.""" + + def test_overview_row_rejects_unknown_status(self) -> None: + with pytest.raises(ValidationError): + UsageOverviewRow( + id="g1", + name="g1", + type="Guardrail", + provider="Custom", + requestsEvaluated=1, + failRate=0.0, + avgScore=None, + avgLatency=None, + status="degraded", + trend="stable", + ) + + def test_overview_row_rejects_unknown_trend(self) -> None: + with pytest.raises(ValidationError): + UsageOverviewRow( + id="g1", + name="g1", + type="Guardrail", + provider="Custom", + requestsEvaluated=1, + failRate=0.0, + avgScore=None, + avgLatency=None, + status="healthy", + trend="sideways", + ) + + def test_log_entry_rejects_unknown_action(self) -> None: + with pytest.raises(ValidationError): + UsageLogEntry( + id="req-1", + timestamp="2026-07-01T00:00:00+00:00", + action="allowed", + score=None, + latency_ms=None, + model=None, + input_snippet=None, + output_snippet=None, + reason=None, + ) + + def test_detail_time_series_rejects_unshaped_point(self) -> None: + with pytest.raises(ValidationError): + UsageDetailResponse( + guardrail_id="g1", + guardrail_name="g1", + type="Guardrail", + provider="Custom", + requestsEvaluated=0, + failRate=0.0, + avgScore=None, + avgLatency=None, + status="healthy", + trend="stable", + description=None, + time_series=[{"day": "2026-07-01"}], + ) + + def test_time_series_point_carries_chart_fields_plus_score(self) -> None: + point = UsageTimeSeriesPoint(date="2026-07-01", passed=2, blocked=1, score=None) + + assert point.model_dump() == {"date": "2026-07-01", "passed": 2, "blocked": 1, "score": None} + + +class _FakeTable: + def __init__(self, rows: tuple[SimpleNamespace, ...]) -> None: + self._rows = rows + + async def find_many(self, **_kwargs: object) -> list[SimpleNamespace]: + return list(self._rows) + + +class _FakeDb: + def __init__(self, guardrails: tuple[SimpleNamespace, ...], metrics: tuple[SimpleNamespace, ...]) -> None: + self.litellm_guardrailstable = _FakeTable(guardrails) + self.litellm_dailyguardrailmetrics = _FakeTable(metrics) + + +class _FakePrisma: + def __init__(self, guardrails: tuple[SimpleNamespace, ...], metrics: tuple[SimpleNamespace, ...]) -> None: + self.db = _FakeDb(guardrails, metrics) + + +@pytest.fixture +def overview_client(monkeypatch: pytest.MonkeyPatch) -> TestClient: + guardrail = SimpleNamespace( + guardrail_id="g-1", + guardrail_name="pii-detector", + litellm_params={"guardrail": "presidio"}, + guardrail_info={"type": "PII"}, + ) + metrics = ( + SimpleNamespace( + guardrail_id="pii-detector", + date="2026-07-01", + requests_evaluated=100, + passed_count=80, + blocked_count=20, + flagged_count=0, + ), + ) + + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "prisma_client", _FakePrisma((guardrail,), metrics), raising=False) + + app = FastAPI() + app.include_router(router) + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(api_key="sk-test") + return TestClient(app) + + +class TestOverviewEndpointContract: + """Exercises FastAPI's response_model serialization, not just the helpers.""" + + def test_serializes_a_row_through_the_narrowed_response_model(self, overview_client: TestClient) -> None: + response = overview_client.get("/guardrails/usage/overview?start_date=2026-07-01&end_date=2026-07-08") + + assert response.status_code == 200 + body = response.json() + assert body["rows"] == [ + { + "id": "g-1", + "name": "pii-detector", + "type": "PII", + "provider": "presidio", + "requestsEvaluated": 100, + "failRate": 20.0, + "avgScore": None, + "avgLatency": None, + "status": "critical", + "trend": "stable", + } + ] + + def test_chart_serializes_as_objects_not_opaque_dicts(self, overview_client: TestClient) -> None: + response = overview_client.get("/guardrails/usage/overview?start_date=2026-07-01&end_date=2026-07-08") + + assert response.json()["chart"] == [{"date": "2026-07-01", "passed": 80, "blocked": 20}] + + def test_totals_are_computed_from_metrics(self, overview_client: TestClient) -> None: + body = overview_client.get("/guardrails/usage/overview").json() + + assert (body["totalRequests"], body["totalBlocked"], body["passRate"]) == (100, 20, 80.0) diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index fa2287bdaef..0af09e21538 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -2133,7 +2133,7 @@ "count": 5 }, "no-restricted-syntax": { - "count": 154 + "count": 151 } }, "src/components/object_permissions_view.tsx": { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.tsx index 1d959ccca95..123abc0c3a2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailDetail.tsx @@ -1,12 +1,11 @@ import { ArrowLeftOutlined, SafetyOutlined, SettingOutlined, WarningOutlined } from "@ant-design/icons"; -import { useQuery } from "@tanstack/react-query"; import { Button, Col, Row, Spin, Tabs } from "antd"; -import React, { useMemo, useState } from "react"; -import { getGuardrailsUsageDetail, getGuardrailsUsageLogs } from "@/components/networking"; +import React, { useState } from "react"; import { EvaluationSettingsModal } from "./EvaluationSettingsModal"; import { LogViewer } from "@/components/GuardrailsMonitor/LogViewer"; import { MetricCard } from "@/components/GuardrailsMonitor/MetricCard"; -import type { LogEntry } from "@/components/GuardrailsMonitor/mockData"; +import { useGuardrailUsageDetail } from "@/app/(dashboard)/hooks/guardrailsMonitor/useGuardrailUsageDetail"; +import { useGuardrailUsageLogs } from "@/app/(dashboard)/hooks/guardrailsMonitor/useGuardrailUsageLogs"; interface GuardrailDetailProps { guardrailId: string; @@ -22,71 +21,37 @@ const statusColors: Record = critical: { bg: "bg-red-50", text: "text-red-700", dot: "bg-red-500" }, }; +const LOGS_PAGE = 1; +const LOGS_PAGE_SIZE = 50; + export function GuardrailDetail({ guardrailId, onBack, accessToken = null, startDate, endDate }: GuardrailDetailProps) { const [activeTab, setActiveTab] = useState("overview"); const [evaluationModalOpen, setEvaluationModalOpen] = useState(false); - const [logsPage, setLogsPage] = useState(1); - const logsPageSize = 50; const { data: detailData, isLoading: detailLoading, error: detailError, - } = useQuery({ - queryKey: ["guardrails-usage-detail", guardrailId, startDate, endDate], - queryFn: () => getGuardrailsUsageDetail(accessToken!, guardrailId, startDate, endDate), - enabled: !!accessToken && !!guardrailId, - }); - const { data: logsData, isLoading: logsLoading } = useQuery({ - queryKey: ["guardrails-usage-logs", guardrailId, logsPage, logsPageSize], - queryFn: () => - getGuardrailsUsageLogs(accessToken!, { - guardrailId, - page: logsPage, - pageSize: logsPageSize, - startDate, - endDate, - }), - enabled: !!accessToken && !!guardrailId, - }); + } = useGuardrailUsageDetail(guardrailId, startDate, endDate); + const logsParams = { + guardrailId, + page: LOGS_PAGE, + pageSize: LOGS_PAGE_SIZE, + startDate, + endDate, + }; + const { data: logsData, isLoading: logsLoading } = useGuardrailUsageLogs(logsParams); - const logs: LogEntry[] = useMemo(() => { - const list = logsData?.logs ?? []; - return list.map((l: Record) => ({ - id: l.id as string, - timestamp: l.timestamp as string, - action: l.action as "blocked" | "passed" | "flagged", - score: l.score as number | undefined, - model: l.model as string | undefined, - input_snippet: l.input_snippet as string | undefined, - output_snippet: l.output_snippet as string | undefined, - reason: l.reason as string | undefined, - })); - }, [logsData?.logs]); - - const data = detailData - ? { - name: detailData.guardrail_name, - description: detailData.description ?? "", - status: detailData.status, - provider: detailData.provider, - type: detailData.type, - requestsEvaluated: detailData.requestsEvaluated, - failRate: detailData.failRate, - avgScore: detailData.avgScore, - avgLatency: detailData.avgLatency, - } - : { - name: guardrailId, - description: "", - status: "healthy", - provider: "—", - type: "—", - requestsEvaluated: 0, - failRate: 0, - avgScore: undefined as number | undefined, - avgLatency: undefined as number | undefined, - }; + const logs = logsData?.logs ?? []; + const data = { + name: detailData?.guardrail_name ?? guardrailId, + description: detailData?.description ?? "", + status: detailData?.status ?? "healthy", + provider: detailData?.provider ?? "—", + requestsEvaluated: detailData?.requestsEvaluated ?? 0, + failRate: detailData?.failRate ?? 0, + avgLatency: detailData?.avgLatency ?? null, + }; const statusStyle = statusColors[data.status] ?? statusColors.healthy; if (detailLoading && !detailData) { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.test.tsx index b946749bf65..31f5ec8e6ac 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.test.tsx @@ -1,45 +1,95 @@ import { render, screen, waitFor } from "@testing-library/react"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; -import { describe, expect, it, vi } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; import GuardrailsMonitorView from "./GuardrailsMonitorView"; -import * as networking from "@/components/networking"; +import type { GuardrailUsageOverview } from "@/app/(dashboard)/hooks/guardrailsMonitor/useGuardrailsUsageOverview"; -vi.mock("@/components/networking", () => ({ - getGuardrailsUsageOverview: vi.fn(), - formatDate: vi.fn((d: Date) => d.toISOString().slice(0, 10)), +const useQueryMock = vi.fn(); +vi.mock("@/lib/http/api", () => ({ + $api: { useQuery: (...args: unknown[]) => useQueryMock(...args) }, })); -const mockGetGuardrailsUsageOverview = vi.mocked(networking.getGuardrailsUsageOverview); +const mockUseAuthorized = vi.fn(); +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: () => mockUseAuthorized(), +})); + +vi.mock("@/components/networking", () => ({ + formatDate: vi.fn((d: Date) => d.toISOString().slice(0, 10)), + fetchAvailableModels: vi.fn(async () => []), + uiSpendLogsCall: vi.fn(async () => ({ data: [], total: 0 })), +})); + +const overview: GuardrailUsageOverview = { + rows: [ + { + id: "guardrail-1", + name: "PII Detector", + type: "Guardrail", + provider: "Custom", + requestsEvaluated: 120, + failRate: 20, + avgScore: null, + avgLatency: null, + status: "critical", + trend: "up", + }, + ], + chart: [{ date: "2026-07-01", passed: 96, blocked: 24 }], + totalRequests: 120, + totalBlocked: 24, + passRate: 80, +}; function wrapper({ children }: { children: React.ReactNode }) { - const queryClient = new QueryClient({ - defaultOptions: { - queries: { retry: false }, - }, - }); + const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); return {children}; } describe("GuardrailsMonitorView", () => { - it("should render overview and fetch guardrails usage when accessToken is provided", async () => { - mockGetGuardrailsUsageOverview.mockResolvedValue({ - rows: [], - chart: [], - totalRequests: 0, - totalBlocked: 0, - passRate: 100, - }); + beforeEach(() => { + vi.clearAllMocks(); + mockUseAuthorized.mockReturnValue({ accessToken: "test-token", userRole: "Admin" }); + useQueryMock.mockReturnValue({ data: overview, isLoading: false, error: null }); + }); + it("requests the guardrails usage overview for the selected date range", async () => { render(, { wrapper }); expect(await screen.findByRole("heading", { name: /Guardrails Monitor/i })).toBeDefined(); await waitFor(() => { - expect(mockGetGuardrailsUsageOverview).toHaveBeenCalled(); + expect(useQueryMock).toHaveBeenCalledWith( + "get", + "/guardrails/usage/overview", + expect.objectContaining({ + params: { query: { start_date: expect.any(String), end_date: expect.any(String) } }, + }), + expect.objectContaining({ enabled: true }), + ); }); }); - it("should render without crashing when accessToken is null", async () => { + it("renders the typed overview rows and totals", async () => { + render(, { wrapper }); + + expect(await screen.findByText("PII Detector")).toBeDefined(); + expect(screen.getByText("20%")).toBeDefined(); + expect(screen.getByText("80%")).toBeDefined(); + expect(screen.getByText("critical")).toBeDefined(); + }); + + it("disables the query when there is no access token", async () => { + mockUseAuthorized.mockReturnValue({ accessToken: null, userRole: "Admin" }); + useQueryMock.mockReturnValue({ data: undefined, isLoading: false, error: null }); + render(, { wrapper }); + expect(await screen.findByRole("heading", { name: /Guardrails Monitor/i })).toBeDefined(); + expect(useQueryMock).toHaveBeenCalledWith( + "get", + "/guardrails/usage/overview", + expect.anything(), + expect.objectContaining({ enabled: false }), + ); }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx index 9d57f36f93f..555458c3c5b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx @@ -1,13 +1,15 @@ import { DownloadOutlined, RiseOutlined, SafetyOutlined, SettingOutlined, WarningOutlined } from "@ant-design/icons"; -import { useQuery } from "@tanstack/react-query"; import { Button, Card, Col, Row, Spin, Table, Typography } from "antd"; import type { ColumnsType } from "antd/es/table"; import React, { useMemo, useState } from "react"; -import { getGuardrailsUsageOverview } from "@/components/networking"; -import { type PerformanceRow } from "@/components/GuardrailsMonitor/mockData"; import { EvaluationSettingsModal } from "./EvaluationSettingsModal"; import { MetricCard } from "@/components/GuardrailsMonitor/MetricCard"; import { ScoreChart } from "./ScoreChart"; +import { + useGuardrailsUsageOverview, + type GuardrailStatus, + type GuardrailUsageRow, +} from "@/app/(dashboard)/hooks/guardrailsMonitor/useGuardrailsUsageOverview"; interface GuardrailsOverviewProps { accessToken?: string | null; @@ -16,7 +18,9 @@ interface GuardrailsOverviewProps { onSelectGuardrail: (id: string) => void; } -type SortKey = "failRate" | "requestsEvaluated" | "avgLatency" | "falsePositiveRate" | "falseNegativeRate"; +type SortKey = "failRate" | "requestsEvaluated" | "avgLatency"; + +const SORTABLE_KEYS: readonly SortKey[] = ["failRate", "requestsEvaluated", "avgLatency"]; const providerColors: Record = { Bedrock: "bg-orange-100 text-orange-700 border-orange-200", @@ -25,16 +29,6 @@ const providerColors: Record = { Custom: "bg-gray-100 text-gray-600 border-gray-200", }; -function computeMetricsFromRows(data: PerformanceRow[]) { - const totalRequests = data.reduce((sum, r) => sum + r.requestsEvaluated, 0); - const totalBlocked = data.reduce((sum, r) => sum + Math.round((r.requestsEvaluated * r.failRate) / 100), 0); - const passRate = totalRequests > 0 ? ((1 - totalBlocked / totalRequests) * 100).toFixed(1) : "0"; - const withLat = data.filter((r) => r.avgLatency != null); - const avgLatency = - withLat.length > 0 ? Math.round(withLat.reduce((sum, r) => sum + (r.avgLatency ?? 0), 0) / withLat.length) : 0; - return { totalRequests, totalBlocked, passRate, avgLatency, count: data.length }; -} - export function GuardrailsOverview({ accessToken = null, startDate, @@ -45,44 +39,30 @@ export function GuardrailsOverview({ const [sortDir, setSortDir] = useState<"asc" | "desc">("desc"); const [evaluationModalOpen, setEvaluationModalOpen] = useState(false); - const { - data: guardrailsData, - isLoading: guardrailsLoading, - error: guardrailsError, - } = useQuery({ - queryKey: ["guardrails-usage-overview", startDate, endDate], - queryFn: () => getGuardrailsUsageOverview(accessToken!, startDate, endDate), - enabled: !!accessToken, - }); + const { data: guardrailsData, isLoading, error } = useGuardrailsUsageOverview(startDate, endDate); - const activeData: PerformanceRow[] = guardrailsData?.rows ?? []; - const metrics = useMemo(() => { - if (guardrailsData) { - return { - totalRequests: guardrailsData.totalRequests ?? 0, - totalBlocked: guardrailsData.totalBlocked ?? 0, - passRate: String(guardrailsData.passRate ?? 0), - avgLatency: activeData.length - ? Math.round(activeData.reduce((s, r) => s + (r.avgLatency ?? 0), 0) / activeData.length) - : 0, - count: activeData.length, - }; - } - return computeMetricsFromRows(activeData); - }, [guardrailsData, activeData]); + const activeData = useMemo(() => guardrailsData?.rows ?? [], [guardrailsData]); + const metrics = useMemo( + () => ({ + totalRequests: guardrailsData?.totalRequests ?? 0, + totalBlocked: guardrailsData?.totalBlocked ?? 0, + passRate: String(guardrailsData?.passRate ?? 0), + avgLatency: activeData.length + ? Math.round(activeData.reduce((s, r) => s + (r.avgLatency ?? 0), 0) / activeData.length) + : 0, + count: activeData.length, + }), + [guardrailsData, activeData], + ); const chartData = guardrailsData?.chart; const sorted = useMemo(() => { return [...activeData].sort((a, b) => { const mult = sortDir === "desc" ? -1 : 1; - const aVal = a[sortBy] ?? 0; - const bVal = b[sortBy] ?? 0; - return (Number(aVal) - Number(bVal)) * mult; + return ((a[sortBy] ?? 0) - (b[sortBy] ?? 0)) * mult; }); }, [activeData, sortBy, sortDir]); - const isLoading = guardrailsLoading; - const error = guardrailsError; - const columns: ColumnsType = [ + const columns: ColumnsType = [ { title: "Guardrail", dataIndex: "name", @@ -141,7 +121,7 @@ export function GuardrailsOverview({ align: "right", sorter: true, sortOrder: sortBy === "avgLatency" ? (sortDir === "desc" ? "descend" : "ascend") : null, - render: (v?: number) => ( + render: (v: number | null) => ( 150 ? "text-red-600" : v > 50 ? "text-amber-600" : "text-green-600" @@ -156,7 +136,7 @@ export function GuardrailsOverview({ dataIndex: "status", key: "status", align: "center", - render: (status: string) => ( + render: (status: GuardrailStatus) => ( { - const s = sorter as { field?: keyof PerformanceRow; order?: string }; - if (s?.field && sortableKeys.includes(s.field as SortKey)) { + const s = sorter as { field?: keyof GuardrailUsageRow; order?: string }; + if (s?.field && SORTABLE_KEYS.includes(s.field as SortKey)) { setSortBy(s.field as SortKey); setSortDir(s.order === "ascend" ? "asc" : "desc"); } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/ScoreChart.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/ScoreChart.tsx index bc11a6fd3e0..119c7c82133 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/ScoreChart.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/ScoreChart.tsx @@ -1,13 +1,14 @@ import React from "react"; import { BarChart } from "@/components/shared/charts"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import type { GuardrailUsageChartPoint } from "@/app/(dashboard)/hooks/guardrailsMonitor/useGuardrailsUsageOverview"; /** * Overview chart: Request Outcomes Over Time (passed vs blocked). * Stacked bar chart. Data from usage/overview API (chart array). */ interface ScoreChartProps { - data?: Array<{ date: string; passed: number; blocked: number }>; + data?: GuardrailUsageChartPoint[]; } export function ScoreChart({ data }: ScoreChartProps) { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrailsMonitor/useGuardrailUsageDetail.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrailsMonitor/useGuardrailUsageDetail.test.ts new file mode 100644 index 00000000000..42bf6389bf5 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrailsMonitor/useGuardrailUsageDetail.test.ts @@ -0,0 +1,50 @@ +import { renderHook } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { useGuardrailUsageDetail } from "./useGuardrailUsageDetail"; + +const useQueryMock = vi.fn(); +vi.mock("@/lib/http/api", () => ({ + $api: { useQuery: (...args: unknown[]) => useQueryMock(...args) }, +})); + +const mockUseAuthorized = vi.fn(); +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: () => mockUseAuthorized(), +})); + +const lastCall = () => useQueryMock.mock.calls[useQueryMock.mock.calls.length - 1]; +const lastInit = () => lastCall()[2] as { params: { path: Record; query: Record } }; +const lastOptions = () => lastCall()[3] as { enabled: boolean }; + +describe("useGuardrailUsageDetail", () => { + beforeEach(() => { + vi.clearAllMocks(); + useQueryMock.mockReturnValue({ data: undefined }); + mockUseAuthorized.mockReturnValue({ accessToken: "test-access-token", userRole: "Admin" }); + }); + + it("queries the guardrail detail route with the id as a path param", () => { + renderHook(() => useGuardrailUsageDetail("guardrail-1", "2026-07-01", "2026-07-08")); + + expect(lastCall()[0]).toBe("get"); + expect(lastCall()[1]).toBe("/guardrails/usage/detail/{guardrail_id}"); + expect(lastInit().params.path).toEqual({ guardrail_id: "guardrail-1" }); + expect(lastInit().params.query).toEqual({ start_date: "2026-07-01", end_date: "2026-07-08" }); + }); + + it("is enabled for an authorized user with a guardrail selected", () => { + renderHook(() => useGuardrailUsageDetail("guardrail-1", "2026-07-01", "2026-07-08")); + expect(lastOptions().enabled).toBe(true); + }); + + it("is disabled without an access token", () => { + mockUseAuthorized.mockReturnValue({ accessToken: null, userRole: "Admin" }); + renderHook(() => useGuardrailUsageDetail("guardrail-1", "2026-07-01", "2026-07-08")); + expect(lastOptions().enabled).toBe(false); + }); + + it("is disabled without a guardrail id", () => { + renderHook(() => useGuardrailUsageDetail("", "2026-07-01", "2026-07-08")); + expect(lastOptions().enabled).toBe(false); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrailsMonitor/useGuardrailUsageDetail.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrailsMonitor/useGuardrailUsageDetail.ts new file mode 100644 index 00000000000..3acfd460e34 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrailsMonitor/useGuardrailUsageDetail.ts @@ -0,0 +1,21 @@ +import { $api } from "@/lib/http/api"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import type { components } from "@/lib/http/schema"; + +export type GuardrailUsageDetail = components["schemas"]["UsageDetailResponse"]; + +export const useGuardrailUsageDetail = (guardrailId: string, startDate: string, endDate: string) => { + const { accessToken } = useAuthorized(); + + return $api.useQuery( + "get", + "/guardrails/usage/detail/{guardrail_id}", + { + params: { + path: { guardrail_id: guardrailId }, + query: { start_date: startDate, end_date: endDate }, + }, + }, + { enabled: Boolean(accessToken) && Boolean(guardrailId) }, + ); +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrailsMonitor/useGuardrailUsageLogs.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrailsMonitor/useGuardrailUsageLogs.test.ts new file mode 100644 index 00000000000..432c6fa466b --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrailsMonitor/useGuardrailUsageLogs.test.ts @@ -0,0 +1,79 @@ +import { renderHook } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { useGuardrailUsageLogs } from "./useGuardrailUsageLogs"; + +const useQueryMock = vi.fn(); +vi.mock("@/lib/http/api", () => ({ + $api: { useQuery: (...args: unknown[]) => useQueryMock(...args) }, +})); + +const mockUseAuthorized = vi.fn(); +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: () => mockUseAuthorized(), +})); + +const params = { + guardrailId: "guardrail-1", + page: 1, + pageSize: 50, + startDate: "2026-07-01", + endDate: "2026-07-08", +}; + +const lastCall = () => useQueryMock.mock.calls[useQueryMock.mock.calls.length - 1]; +const lastInit = () => lastCall()[2] as { params: { query: Record } }; +const lastOptions = () => lastCall()[3] as { enabled: boolean }; + +describe("useGuardrailUsageLogs", () => { + beforeEach(() => { + vi.clearAllMocks(); + useQueryMock.mockReturnValue({ data: undefined }); + mockUseAuthorized.mockReturnValue({ accessToken: "test-access-token", userRole: "Admin" }); + }); + + it("queries GET /guardrails/usage/logs", () => { + renderHook(() => useGuardrailUsageLogs(params)); + expect(lastCall()[0]).toBe("get"); + expect(lastCall()[1]).toBe("/guardrails/usage/logs"); + }); + + it("sends the date range so the derived query key refetches when it changes", () => { + const expectedQuery = { + guardrail_id: "guardrail-1", + page: 1, + page_size: 50, + start_date: "2026-07-01", + end_date: "2026-07-08", + }; + + renderHook(() => useGuardrailUsageLogs(params)); + + expect(lastInit().params.query).toEqual(expectedQuery); + }); + + it("passes a changed date range through to the request", () => { + const nextParams = { ...params, startDate: "2026-06-01", endDate: "2026-06-30" }; + const { rerender } = renderHook((p: typeof params) => useGuardrailUsageLogs(p), { initialProps: params }); + + rerender(nextParams); + + expect(lastInit().params.query.start_date).toBe("2026-06-01"); + expect(lastInit().params.query.end_date).toBe("2026-06-30"); + }); + + it("is enabled for an authorized user with a guardrail selected", () => { + renderHook(() => useGuardrailUsageLogs(params)); + expect(lastOptions().enabled).toBe(true); + }); + + it("is disabled without an access token", () => { + mockUseAuthorized.mockReturnValue({ accessToken: null, userRole: "Admin" }); + renderHook(() => useGuardrailUsageLogs(params)); + expect(lastOptions().enabled).toBe(false); + }); + + it("is disabled without a guardrail id", () => { + renderHook(() => useGuardrailUsageLogs({ ...params, guardrailId: "" })); + expect(lastOptions().enabled).toBe(false); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrailsMonitor/useGuardrailUsageLogs.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrailsMonitor/useGuardrailUsageLogs.ts new file mode 100644 index 00000000000..475bffd14f5 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrailsMonitor/useGuardrailUsageLogs.ts @@ -0,0 +1,42 @@ +import { $api } from "@/lib/http/api"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import type { components } from "@/lib/http/schema"; + +export type GuardrailUsageLogs = components["schemas"]["UsageLogsResponse"]; +export type GuardrailUsageLogEntry = components["schemas"]["UsageLogEntry"]; +export type GuardrailLogAction = GuardrailUsageLogEntry["action"]; + +interface UseGuardrailUsageLogsParams { + guardrailId: string; + page: number; + pageSize: number; + startDate: string; + endDate: string; +} + +export const useGuardrailUsageLogs = ({ + guardrailId, + page, + pageSize, + startDate, + endDate, +}: UseGuardrailUsageLogsParams) => { + const { accessToken } = useAuthorized(); + + return $api.useQuery( + "get", + "/guardrails/usage/logs", + { + params: { + query: { + guardrail_id: guardrailId, + page, + page_size: pageSize, + start_date: startDate, + end_date: endDate, + }, + }, + }, + { enabled: Boolean(accessToken) && Boolean(guardrailId) }, + ); +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrailsMonitor/useGuardrailsUsageOverview.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrailsMonitor/useGuardrailsUsageOverview.test.ts new file mode 100644 index 00000000000..767aa0887d3 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrailsMonitor/useGuardrailsUsageOverview.test.ts @@ -0,0 +1,44 @@ +import { renderHook } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { useGuardrailsUsageOverview } from "./useGuardrailsUsageOverview"; + +const useQueryMock = vi.fn(); +vi.mock("@/lib/http/api", () => ({ + $api: { useQuery: (...args: unknown[]) => useQueryMock(...args) }, +})); + +const mockUseAuthorized = vi.fn(); +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: () => mockUseAuthorized(), +})); + +const lastCall = () => useQueryMock.mock.calls[useQueryMock.mock.calls.length - 1]; +const lastInit = () => lastCall()[2] as { params: { query: Record } }; +const lastOptions = () => lastCall()[3] as { enabled: boolean }; + +describe("useGuardrailsUsageOverview", () => { + beforeEach(() => { + vi.clearAllMocks(); + useQueryMock.mockReturnValue({ data: undefined }); + mockUseAuthorized.mockReturnValue({ accessToken: "test-access-token", userRole: "Admin" }); + }); + + it("queries GET /guardrails/usage/overview with the selected date range", () => { + renderHook(() => useGuardrailsUsageOverview("2026-07-01", "2026-07-08")); + + expect(lastCall()[0]).toBe("get"); + expect(lastCall()[1]).toBe("/guardrails/usage/overview"); + expect(lastInit().params.query).toEqual({ start_date: "2026-07-01", end_date: "2026-07-08" }); + }); + + it("is enabled once an access token is present", () => { + renderHook(() => useGuardrailsUsageOverview("2026-07-01", "2026-07-08")); + expect(lastOptions().enabled).toBe(true); + }); + + it("is disabled without an access token", () => { + mockUseAuthorized.mockReturnValue({ accessToken: null, userRole: "Admin" }); + renderHook(() => useGuardrailsUsageOverview("2026-07-01", "2026-07-08")); + expect(lastOptions().enabled).toBe(false); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrailsMonitor/useGuardrailsUsageOverview.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrailsMonitor/useGuardrailsUsageOverview.ts new file mode 100644 index 00000000000..1a80cab5fe5 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/guardrailsMonitor/useGuardrailsUsageOverview.ts @@ -0,0 +1,19 @@ +import { $api } from "@/lib/http/api"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import type { components } from "@/lib/http/schema"; + +export type GuardrailUsageOverview = components["schemas"]["UsageOverviewResponse"]; +export type GuardrailUsageRow = components["schemas"]["UsageOverviewRow"]; +export type GuardrailUsageChartPoint = components["schemas"]["UsageChartPoint"]; +export type GuardrailStatus = GuardrailUsageRow["status"]; + +export const useGuardrailsUsageOverview = (startDate: string, endDate: string) => { + const { accessToken } = useAuthorized(); + + return $api.useQuery( + "get", + "/guardrails/usage/overview", + { params: { query: { start_date: startDate, end_date: endDate } } }, + { enabled: Boolean(accessToken) }, + ); +}; diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.tsx b/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.tsx index c3671fc9e2c..76f93ef6371 100644 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.tsx +++ b/ui/litellm-dashboard/src/components/GuardrailsMonitor/LogViewer.tsx @@ -6,10 +6,18 @@ import React, { useState } from "react"; import { uiSpendLogsCall } from "@/components/networking"; import { LogDetailsDrawer } from "@/components/view_logs/LogDetailsDrawer"; import type { LogEntry as ViewLogsLogEntry } from "@/components/view_logs/columns"; -import type { LogEntry } from "./mockData"; +import type { GuardrailLogAction } from "@/app/(dashboard)/hooks/guardrailsMonitor/useGuardrailUsageLogs"; + +export interface LogViewerEntry { + id: string; + timestamp: string; + action: GuardrailLogAction; + model?: string | null; + input_snippet?: string | null; +} const actionConfig: Record< - "blocked" | "passed" | "flagged", + GuardrailLogAction, { icon: React.ElementType; color: string; bg: string; border: string; label: string } > = { blocked: { @@ -37,8 +45,8 @@ const actionConfig: Record< interface LogViewerProps { guardrailName?: string; - filterAction?: "all" | "blocked" | "passed" | "flagged"; - logs?: LogEntry[]; + filterAction?: "all" | GuardrailLogAction; + logs?: LogViewerEntry[]; logsLoading?: boolean; totalLogs?: number; accessToken?: string | null; @@ -65,7 +73,7 @@ export function LogViewer({ const displayLogs = filteredLogs.slice(0, sampleSize); const total = totalLogs ?? logs.length; const sampleSizes = [10, 50, 100]; - const filters: Array<"all" | "blocked" | "flagged" | "passed"> = ["all", "blocked", "flagged", "passed"]; + const filters: Array<"all" | GuardrailLogAction> = ["all", "blocked", "flagged", "passed"]; const startTime = startDate ? moment(startDate).utc().format("YYYY-MM-DD HH:mm:ss") @@ -93,7 +101,7 @@ export function LogViewer({ const selectedLog: ViewLogsLogEntry | null = fullLogResponse?.data?.[0] ?? null; - const handleLogClick = (log: LogEntry) => { + const handleLogClick = (log: LogViewerEntry) => { setSelectedRequestId(log.id); setDrawerOpen(true); }; @@ -184,7 +192,7 @@ export function LogViewer({ · {log.model && {log.model}} -

{log.input_snippet ?? log.input ?? "—"}

+

{log.input_snippet ?? "—"}

diff --git a/ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts b/ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts deleted file mode 100644 index 7d99ebe7c44..00000000000 --- a/ui/litellm-dashboard/src/components/GuardrailsMonitor/mockData.ts +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Types for Guardrails Monitor dashboard (data from usage API). - */ - -export interface PerformanceRow { - id: string; - name: string; - type: string; - provider: string; - requestsEvaluated: number; - failRate: number; - avgScore?: number; - avgLatency?: number; - p95Latency?: number; - falsePositiveRate?: number; - falseNegativeRate?: number; - status: "healthy" | "warning" | "critical"; - trend: "up" | "down" | "stable"; -} - -export interface GuardrailDetailRecord { - name: string; - type: string; - provider: string; - requestsEvaluated: number; - failRate: number; - avgScore?: number; - avgLatency?: number; - p95Latency?: number; - falsePositiveRate?: number; - falsePositiveCount?: number; - falseNegativeRate?: number; - falseNegativeCount?: number; - status: string; - description: string; -} - -export interface LogEntry { - id: string; - timestamp: string; - input?: string; - output?: string; - input_snippet?: string; - output_snippet?: string; - score?: number; - action: "blocked" | "passed" | "flagged"; - model?: string; - reason?: string; - latency_ms?: number; -} diff --git a/ui/litellm-dashboard/src/components/ToolDetail.tsx b/ui/litellm-dashboard/src/components/ToolDetail.tsx index f5ef938fb3a..6ae4e44ead1 100644 --- a/ui/litellm-dashboard/src/components/ToolDetail.tsx +++ b/ui/litellm-dashboard/src/components/ToolDetail.tsx @@ -5,8 +5,7 @@ import { useQuery, useQueryClient } from "@tanstack/react-query"; import { Button, Select, Spin } from "antd"; import React, { useCallback, useEffect, useMemo, useState } from "react"; import TeamDropdown from "@/components/common_components/team_dropdown"; -import { LogViewer } from "@/components/GuardrailsMonitor/LogViewer"; -import type { LogEntry } from "@/components/GuardrailsMonitor/mockData"; +import { LogViewer, type LogViewerEntry } from "@/components/GuardrailsMonitor/LogViewer"; import { PolicySelect } from "@/components/ToolPolicies/PolicySelect"; import { deleteToolPolicyOverride, @@ -96,7 +95,7 @@ export function ToolDetail({ toolName, onBack, accessToken }: ToolDetailProps) { enabled: !!accessToken && !!toolName, }); - const logs: LogEntry[] = useMemo(() => { + const logs: LogViewerEntry[] = useMemo(() => { const list = logsData?.logs ?? []; return list.map((l) => ({ id: l.id, diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 7e3af46b931..cc8f89a449a 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -3805,105 +3805,6 @@ export const rejectGuardrailSubmission = async ( ); }; -// Guardrails / Policies usage (dashboard) -export const getGuardrailsUsageOverview = async (accessToken: string, startDate?: string, endDate?: string) => { - try { - let url = proxyBaseUrl ? `${proxyBaseUrl}/guardrails/usage/overview` : `/guardrails/usage/overview`; - const params = new URLSearchParams(); - if (startDate) params.append("start_date", startDate); - if (endDate) params.append("end_date", endDate); - if (params.toString()) url += `?${params.toString()}`; - const response = await fetch(url, { - method: "GET", - headers: { - [globalLitellmHeaderName]: `Bearer ${accessToken}`, - "Content-Type": "application/json", - }, - }); - if (!response.ok) { - const errorData = await response.json(); - throw new Error(deriveErrorMessage(errorData)); - } - return response.json(); - } catch (error) { - console.error("Failed to get guardrails usage overview:", error); - throw error; - } -}; - -export const getGuardrailsUsageDetail = async ( - accessToken: string, - guardrailId: string, - startDate?: string, - endDate?: string, -) => { - try { - let url = proxyBaseUrl - ? `${proxyBaseUrl}/guardrails/usage/detail/${encodeURIComponent(guardrailId)}` - : `/guardrails/usage/detail/${encodeURIComponent(guardrailId)}`; - const params = new URLSearchParams(); - if (startDate) params.append("start_date", startDate); - if (endDate) params.append("end_date", endDate); - if (params.toString()) url += `?${params.toString()}`; - const response = await fetch(url, { - method: "GET", - headers: { - [globalLitellmHeaderName]: `Bearer ${accessToken}`, - "Content-Type": "application/json", - }, - }); - if (!response.ok) { - const errorData = await response.json(); - throw new Error(deriveErrorMessage(errorData)); - } - return response.json(); - } catch (error) { - console.error("Failed to get guardrails usage detail:", error); - throw error; - } -}; - -export const getGuardrailsUsageLogs = async ( - accessToken: string, - options: { - guardrailId?: string; - policyId?: string; - page?: number; - pageSize?: number; - action?: string; - startDate?: string; - endDate?: string; - }, -) => { - try { - let url = proxyBaseUrl ? `${proxyBaseUrl}/guardrails/usage/logs` : `/guardrails/usage/logs`; - const params = new URLSearchParams(); - if (options.guardrailId) params.append("guardrail_id", options.guardrailId); - if (options.policyId) params.append("policy_id", options.policyId); - if (options.page != null) params.append("page", String(options.page)); - if (options.pageSize != null) params.append("page_size", String(options.pageSize)); - if (options.action) params.append("action", options.action); - if (options.startDate) params.append("start_date", options.startDate); - if (options.endDate) params.append("end_date", options.endDate); - if (params.toString()) url += `?${params.toString()}`; - const response = await fetch(url, { - method: "GET", - headers: { - [globalLitellmHeaderName]: `Bearer ${accessToken}`, - "Content-Type": "application/json", - }, - }); - if (!response.ok) { - const errorData = await response.json(); - throw new Error(deriveErrorMessage(errorData)); - } - return response.json(); - } catch (error) { - console.error("Failed to get guardrails usage logs:", error); - throw error; - } -}; - // ───────────────────────────────────────────────────────────────────────────── // Policy CRUD API Calls // ───────────────────────────────────────────────────────────────────────────── diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 1d620e9d153..ca2e6722e28 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -32622,6 +32622,15 @@ export interface components { */ model?: string | null; }; + /** UsageChartPoint */ + UsageChartPoint: { + /** Blocked */ + blocked: number; + /** Date */ + date: string; + /** Passed */ + passed: number; + }; /** UsageDetailResponse */ UsageDetailResponse: { /** Avglatency */ @@ -32640,21 +32649,28 @@ export interface components { provider: string; /** Requestsevaluated */ requestsEvaluated: number; - /** Status */ - status: string; + /** + * Status + * @enum {string} + */ + status: "healthy" | "warning" | "critical"; /** Time Series */ - time_series: { - [key: string]: unknown; - }[]; - /** Trend */ - trend: string; + time_series: components["schemas"]["UsageTimeSeriesPoint"][]; + /** + * Trend + * @enum {string} + */ + trend: "up" | "down" | "stable"; /** Type */ type: string; }; /** UsageLogEntry */ UsageLogEntry: { - /** Action */ - action: string; + /** + * Action + * @enum {string} + */ + action: "passed" | "blocked" | "flagged"; /** Id */ id: string; /** Input Snippet */ @@ -32686,9 +32702,7 @@ export interface components { /** UsageOverviewResponse */ UsageOverviewResponse: { /** Chart */ - chart: { - [key: string]: unknown; - }[]; + chart: components["schemas"]["UsageChartPoint"][]; /** Passrate */ passRate: number; /** Rows */ @@ -32714,13 +32728,30 @@ export interface components { provider: string; /** Requestsevaluated */ requestsEvaluated: number; - /** Status */ - status: string; - /** Trend */ - trend: string; + /** + * Status + * @enum {string} + */ + status: "healthy" | "warning" | "critical"; + /** + * Trend + * @enum {string} + */ + trend: "up" | "down" | "stable"; /** Type */ type: string; }; + /** UsageTimeSeriesPoint */ + UsageTimeSeriesPoint: { + /** Blocked */ + blocked: number; + /** Date */ + date: string; + /** Passed */ + passed: number; + /** Score */ + score: number | null; + }; /** * UserAPIKeyAuth * @description Return the row in the db