mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-16 23:41:43 +00:00
refactor(guardrails): rename scoped-out evaluation status from not_run to skipped
The per-guardrail status a scoped-out evaluation records is now skipped, matching the skip_*_in_guardrail settings that cause it. Request-level rollup still maps it to not_run so the StandardLoggingPayload status contract is unchanged Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yucheng
parent
33fb6625ad
commit
b37ce94075
18 changed files with 87 additions and 73 deletions
|
|
@ -6021,6 +6021,7 @@ def _get_status_fields(
|
|||
"failure": "guardrail_failed_to_respond", # legacy
|
||||
"guardrail_failed_to_respond": "guardrail_failed_to_respond", # direct
|
||||
"not_run": "not_run",
|
||||
"skipped": "not_run",
|
||||
}
|
||||
|
||||
# Set LLM API status
|
||||
|
|
|
|||
|
|
@ -214,7 +214,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
guardrail_to_apply.add_standard_logging_guardrail_information_to_request_data(
|
||||
guardrail_json_response="no scannable content after message scoping",
|
||||
request_data=data,
|
||||
guardrail_status="not_run",
|
||||
guardrail_status="skipped",
|
||||
)
|
||||
|
||||
verbose_proxy_logger.debug(
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ class ComplianceChecker:
|
|||
|
||||
def __init__(self, data: ComplianceCheckRequest):
|
||||
self.data = data
|
||||
self.guardrails = [g for g in (data.guardrail_information or []) if g.get("guardrail_status") != "not_run"]
|
||||
self.guardrails = [g for g in (data.guardrail_information or []) if g.get("guardrail_status") != "skipped"]
|
||||
|
||||
def _get_guardrails_by_mode(self, mode: str) -> list[dict]:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ if TYPE_CHECKING:
|
|||
router: Final = APIRouter()
|
||||
|
||||
_EMPTY_UNITS: Final[Mapping[str, int]] = MappingProxyType({})
|
||||
_ACTION_SEVERITY: Final[Mapping[str, int]] = MappingProxyType({"not_run": 0, "passed": 1, "flagged": 2, "blocked": 3})
|
||||
_ACTION_SEVERITY: Final[Mapping[str, int]] = MappingProxyType({"skipped": 0, "passed": 1, "flagged": 2, "blocked": 3})
|
||||
|
||||
_T = TypeVar("_T")
|
||||
|
||||
|
|
@ -325,7 +325,7 @@ class UsageDetailResponse(BaseModel):
|
|||
class UsageLogEntry(BaseModel):
|
||||
id: str
|
||||
timestamp: str
|
||||
action: str # blocked | passed | flagged | not_run
|
||||
action: str # blocked | passed | flagged | skipped
|
||||
score: float | None
|
||||
latency_ms: float | None
|
||||
model: str | None
|
||||
|
|
|
|||
|
|
@ -193,12 +193,12 @@ async def _upsert_rows_with_retry(
|
|||
|
||||
|
||||
def guardrail_status_to_action(status: str | None) -> str:
|
||||
"""Map StandardLogging guardrail_status to blocked/passed/flagged/not_run."""
|
||||
"""Map StandardLogging guardrail_status to blocked/passed/flagged/skipped."""
|
||||
if not status:
|
||||
return "passed"
|
||||
s: Final = (status or "").lower()
|
||||
if s == "not_run":
|
||||
return "not_run"
|
||||
if s == "skipped":
|
||||
return "skipped"
|
||||
if "intervened" in s or "block" in s:
|
||||
return "blocked"
|
||||
if "flagged" in s or "fail" in s or "error" in s:
|
||||
|
|
@ -380,7 +380,7 @@ async def process_spend_logs_guardrail_usage(
|
|||
if not isinstance(guardrail_id, str) or not guardrail_id:
|
||||
continue
|
||||
action = guardrail_status_to_action(entry.get("guardrail_status"))
|
||||
if action != "not_run":
|
||||
if action != "skipped":
|
||||
key = _MetricsKey(guardrail_id, date_key)
|
||||
daily_guardrail[key]["requests_evaluated"] += 1
|
||||
if action == "passed":
|
||||
|
|
|
|||
|
|
@ -3120,7 +3120,7 @@ class GuardrailMode(TypedDict, total=False):
|
|||
|
||||
|
||||
GuardrailStatus = Literal[
|
||||
"success", "guardrail_flagged", "guardrail_intervened", "guardrail_failed_to_respond", "not_run"
|
||||
"success", "guardrail_flagged", "guardrail_intervened", "guardrail_failed_to_respond", "not_run", "skipped"
|
||||
]
|
||||
|
||||
# Fields on a guardrail record whose values can quote the caller's prompt: the payload sent to the
|
||||
|
|
@ -3367,6 +3367,7 @@ class StandardLoggingPayloadStatusFields(TypedDict, total=False):
|
|||
- 'guardrail_intervened': Guardrail blocked or modified content
|
||||
- 'guardrail_failed_to_respond': Guardrail had technical failure
|
||||
- 'not_run': No guardrail was run
|
||||
- 'skipped': Only used per guardrail entry, message scoping left the guardrail nothing to scan
|
||||
"""
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -6926,6 +6926,18 @@ def test_get_status_fields_ranks_guardrail_flagged_between_success_and_intervene
|
|||
)["guardrail_status"] == "guardrail_intervened"
|
||||
|
||||
|
||||
def test_get_status_fields_rolls_skipped_entries_up_to_not_run():
|
||||
"""LIT-6314: a guardrail that message scoping left nothing to scan records a
|
||||
skipped entry. At request level that means no guardrail ran, and a skipped
|
||||
entry must never outrank a sibling that did evaluate."""
|
||||
skipped = {"guardrail_status": "skipped"}
|
||||
|
||||
assert _get_status_fields("success", [skipped], None)["guardrail_status"] == "not_run"
|
||||
assert _get_status_fields(
|
||||
"success", [skipped, {"guardrail_status": "success"}], None
|
||||
)["guardrail_status"] == "success"
|
||||
|
||||
|
||||
def test_get_error_information_redacts_provider_key_from_upstream_url():
|
||||
"""A pass-through upstream failure logs the httpx traceback, whose message
|
||||
quotes the upstream URL with the provider key in its query string. That
|
||||
|
|
|
|||
|
|
@ -1893,7 +1893,7 @@ class TestScanOnlyToolResults:
|
|||
assert data["messages"][4]["content"] == "and then?"
|
||||
|
||||
|
||||
class TestNoScannableContentRecordsNotRun:
|
||||
class TestNoScannableContentRecordsSkipped:
|
||||
"""LIT-6314: a guardrail whose scoping leaves nothing to scan must still persist an evaluation record"""
|
||||
|
||||
def _system_only_data(self) -> dict:
|
||||
|
|
@ -1904,7 +1904,7 @@ class TestNoScannableContentRecordsNotRun:
|
|||
return metadata.get("standard_logging_guardrail_information") or []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_skipped_scan_records_not_run_entry(self):
|
||||
async def test_skipped_scan_records_skipped_entry(self):
|
||||
handler = OpenAIChatCompletionsHandler()
|
||||
guardrail = MockGuardrail(guardrail_name="skip-system-guardrail")
|
||||
guardrail.skip_system_message_in_guardrail = True
|
||||
|
|
@ -1916,7 +1916,7 @@ class TestNoScannableContentRecordsNotRun:
|
|||
entries = self._recorded_entries(data)
|
||||
assert len(entries) == 1
|
||||
assert entries[0]["guardrail_name"] == "skip-system-guardrail"
|
||||
assert entries[0]["guardrail_status"] == "not_run"
|
||||
assert entries[0]["guardrail_status"] == "skipped"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_self_recording_guardrail_is_left_alone(self):
|
||||
|
|
@ -1940,7 +1940,7 @@ class TestNoScannableContentRecordsNotRun:
|
|||
await handler.process_input_messages(data=data, guardrail_to_apply=guardrail)
|
||||
|
||||
assert guardrail.last_inputs is not None
|
||||
assert all(e.get("guardrail_status") != "not_run" for e in self._recorded_entries(data))
|
||||
assert all(e.get("guardrail_status") != "skipped" for e in self._recorded_entries(data))
|
||||
|
||||
|
||||
class TestBuildBlockSseChunks:
|
||||
|
|
|
|||
|
|
@ -685,7 +685,7 @@ async def test_detail_prev_trend_query_is_bounded():
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_logs_report_not_run_entries_as_not_run_not_passed():
|
||||
async def test_logs_report_skipped_entries_as_skipped_not_passed():
|
||||
"""LIT-6314: a guardrail that never scanned must not be reported as a pass in the drill-down."""
|
||||
index_row = MagicMock()
|
||||
index_row.request_id = "req-nr"
|
||||
|
|
@ -697,7 +697,7 @@ async def test_logs_report_not_run_entries_as_not_run_not_passed():
|
|||
spend_log.startTime = datetime(2026, 4, 22)
|
||||
spend_log.metadata = {
|
||||
"guardrail_information": [
|
||||
{"guardrail_name": "db-1", "guardrail_status": "not_run", "duration": 0.0},
|
||||
{"guardrail_name": "db-1", "guardrail_status": "skipped", "duration": 0.0},
|
||||
]
|
||||
}
|
||||
prisma = _prisma(find_unique=_db_row(), index_find_many=[index_row])
|
||||
|
|
@ -715,11 +715,11 @@ async def test_logs_report_not_run_entries_as_not_run_not_passed():
|
|||
end_date=END,
|
||||
user_api_key_dict=ADMIN,
|
||||
)
|
||||
assert [log.action for log in resp.logs] == ["not_run"]
|
||||
assert [log.action for log in resp.logs] == ["skipped"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_logs_action_passed_filter_excludes_not_run_entries():
|
||||
async def test_logs_action_passed_filter_excludes_skipped_entries():
|
||||
"""LIT-6314: filtering the drill-down for passes must not return unscanned requests."""
|
||||
index_row = MagicMock()
|
||||
index_row.request_id = "req-nr"
|
||||
|
|
@ -729,7 +729,7 @@ async def test_logs_action_passed_filter_excludes_not_run_entries():
|
|||
spend_log.request_id = "req-nr"
|
||||
spend_log.model = "gpt-4o-mini"
|
||||
spend_log.startTime = datetime(2026, 4, 22)
|
||||
spend_log.metadata = {"guardrail_information": [{"guardrail_name": "db-1", "guardrail_status": "not_run"}]}
|
||||
spend_log.metadata = {"guardrail_information": [{"guardrail_name": "db-1", "guardrail_status": "skipped"}]}
|
||||
prisma = _prisma(find_unique=_db_row(), index_find_many=[index_row])
|
||||
prisma.db.litellm_spendlogs.find_many = AsyncMock(return_value=[spend_log])
|
||||
handler = _config_handler()
|
||||
|
|
|
|||
|
|
@ -350,15 +350,15 @@ async def test_zero_and_non_int_usage_counters_are_skipped():
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_not_run_entries_are_indexed_but_not_counted_as_evaluations():
|
||||
async def test_skipped_entries_are_indexed_but_not_counted_as_evaluations():
|
||||
"""
|
||||
LIT-6314 records a not_run entry when message scoping leaves a guardrail
|
||||
LIT-6314 records a skipped entry when message scoping leaves a guardrail
|
||||
nothing to scan. The guardrail never evaluated the request, so counting it
|
||||
as a passed evaluation would inflate daily pass rates; it still gets an
|
||||
index row so per-request drill-down finds the spend log.
|
||||
"""
|
||||
prisma = _prisma()
|
||||
logs = [_payload("r1", guardrail_status="not_run"), _payload("r2")]
|
||||
logs = [_payload("r1", guardrail_status="skipped"), _payload("r2")]
|
||||
|
||||
await process_spend_logs_guardrail_usage(prisma, logs)
|
||||
|
||||
|
|
@ -370,26 +370,26 @@ async def test_not_run_entries_are_indexed_but_not_counted_as_evaluations():
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_not_run_entry_shares_index_key_with_evaluated_sibling_of_same_name():
|
||||
async def test_skipped_entry_shares_index_key_with_evaluated_sibling_of_same_name():
|
||||
"""
|
||||
The not_run entry from the shared base guardrail carries only guardrail_name,
|
||||
The skipped entry from the shared base guardrail carries only guardrail_name,
|
||||
while the evaluated entry from the same guardrail (e.g. content filter on the
|
||||
output of a logging_only run) carries its guardrail_id. Keying them differently
|
||||
lists one request twice in the monitor, once as not_run and once as passed.
|
||||
lists one request twice in the monitor, once as skipped and once as passed.
|
||||
"""
|
||||
prisma = _prisma()
|
||||
payload = _payload("r1")
|
||||
payload["metadata"] = json.dumps(
|
||||
{
|
||||
"guardrail_information": [
|
||||
{"guardrail_name": "cf", "guardrail_status": "not_run"},
|
||||
{"guardrail_name": "cf", "guardrail_status": "skipped"},
|
||||
{
|
||||
"guardrail_name": "cf",
|
||||
"guardrail_id": "cf-uuid",
|
||||
"policy_id": "pol-1",
|
||||
"guardrail_status": "success",
|
||||
},
|
||||
{"guardrail_name": "other", "guardrail_status": "not_run"},
|
||||
{"guardrail_name": "other", "guardrail_status": "skipped"},
|
||||
]
|
||||
}
|
||||
)
|
||||
|
|
@ -406,7 +406,7 @@ async def test_not_run_entry_shares_index_key_with_evaluated_sibling_of_same_nam
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_malformed_not_run_entry_does_not_drop_the_batch():
|
||||
async def test_malformed_skipped_entry_does_not_drop_the_batch():
|
||||
prisma = _prisma()
|
||||
payload = _payload("r1")
|
||||
payload["metadata"] = json.dumps(
|
||||
|
|
@ -428,10 +428,10 @@ async def test_malformed_not_run_entry_does_not_drop_the_batch():
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_batch_of_only_not_run_entries_writes_no_metrics_row():
|
||||
async def test_batch_of_only_skipped_entries_writes_no_metrics_row():
|
||||
prisma = _prisma()
|
||||
|
||||
await process_spend_logs_guardrail_usage(prisma, [_payload("r1", guardrail_status="not_run")])
|
||||
await process_spend_logs_guardrail_usage(prisma, [_payload("r1", guardrail_status="skipped")])
|
||||
|
||||
assert prisma.db.litellm_dailyguardrailmetrics.upsert.call_count == 0
|
||||
index_rows = prisma.db.litellm_spendlogguardrailindex.create_many.call_args.kwargs["data"]
|
||||
|
|
|
|||
|
|
@ -591,17 +591,17 @@ class TestModeMatching:
|
|||
assert mode in _guaranteed_modes(g_mode), (g_mode, mode)
|
||||
|
||||
|
||||
class TestNotRunGuardrails:
|
||||
"""LIT-6314 logs a not_run entry for a guardrail that message scoping left nothing to scan."""
|
||||
class TestSkippedGuardrails:
|
||||
"""LIT-6314 logs a skipped entry for a guardrail that message scoping left nothing to scan."""
|
||||
|
||||
def test_not_run_alone_never_evidences_compliance(self):
|
||||
def test_skipped_alone_never_evidences_compliance(self):
|
||||
data = ComplianceCheckRequest(
|
||||
request_id="req-601",
|
||||
user_id="user-1",
|
||||
model="gpt-4",
|
||||
timestamp="2026-02-17T00:00:00Z",
|
||||
guardrail_information=[
|
||||
{"guardrail_name": "pii_detection", "guardrail_status": "not_run", "guardrail_mode": "pre_call"},
|
||||
{"guardrail_name": "pii_detection", "guardrail_status": "skipped", "guardrail_mode": "pre_call"},
|
||||
],
|
||||
)
|
||||
results = {c.check_name: c.passed for c in ComplianceChecker(data).check_eu_ai_act()}
|
||||
|
|
@ -609,7 +609,7 @@ class TestNotRunGuardrails:
|
|||
assert results["Content screened before LLM"] is False
|
||||
assert results["Audit record complete"] is False
|
||||
|
||||
def test_not_run_sibling_does_not_fail_a_passing_request(self):
|
||||
def test_skipped_sibling_does_not_fail_a_passing_request(self):
|
||||
data = ComplianceCheckRequest(
|
||||
request_id="req-602",
|
||||
user_id="user-1",
|
||||
|
|
@ -618,7 +618,7 @@ class TestNotRunGuardrails:
|
|||
pii_detected=True,
|
||||
guardrail_information=[
|
||||
{"guardrail_name": "pii_detection", "guardrail_status": "success", "guardrail_mode": "pre_call"},
|
||||
{"guardrail_name": "system_only", "guardrail_status": "not_run", "guardrail_mode": "pre_call"},
|
||||
{"guardrail_name": "system_only", "guardrail_status": "skipped", "guardrail_mode": "pre_call"},
|
||||
],
|
||||
)
|
||||
results = {c.check_name: c.passed for c in ComplianceChecker(data).check_gdpr()}
|
||||
|
|
|
|||
|
|
@ -96,14 +96,14 @@ describe("GuardrailsMonitor LogViewer drawer", () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe("GuardrailsMonitor LogViewer not_run rows", () => {
|
||||
it("renders a not_run log as a neutral Not run badge instead of a pass or failure", () => {
|
||||
describe("GuardrailsMonitor LogViewer skipped rows", () => {
|
||||
it("renders a skipped log as a neutral Skipped badge instead of a pass or failure", () => {
|
||||
renderWithProviders(
|
||||
<LogViewer logs={[{ ...guardrailLog, action: "not_run", input_snippet: "system prompt only" }]} />,
|
||||
<LogViewer logs={[{ ...guardrailLog, action: "skipped", input_snippet: "system prompt only" }]} />,
|
||||
);
|
||||
|
||||
const row = screen.getByRole("button", { name: /system prompt only/ });
|
||||
expect(within(row).getByText("Not run")).toHaveClass("text-muted-foreground");
|
||||
expect(within(row).getByText("Skipped")).toHaveClass("text-muted-foreground");
|
||||
expect(within(row).queryByText("Passed")).not.toBeInTheDocument();
|
||||
expect(within(row).queryByText("Blocked")).not.toBeInTheDocument();
|
||||
});
|
||||
|
|
|
|||
|
|
@ -10,15 +10,15 @@ import type { LogEntry as ViewLogsLogEntry } from "@/components/view_logs/column
|
|||
import type { LogEntry } from "./mockData";
|
||||
|
||||
const actionConfig: Record<
|
||||
"blocked" | "passed" | "flagged" | "not_run",
|
||||
"blocked" | "passed" | "flagged" | "skipped",
|
||||
{ icon: React.ElementType; color: string; bg: string; border: string; label: string }
|
||||
> = {
|
||||
not_run: {
|
||||
skipped: {
|
||||
icon: MinusCircle,
|
||||
color: "text-muted-foreground",
|
||||
bg: "bg-muted",
|
||||
border: "border-border",
|
||||
label: "Not run",
|
||||
label: "Skipped",
|
||||
},
|
||||
blocked: {
|
||||
icon: X,
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ export interface LogEntry {
|
|||
input_snippet?: string;
|
||||
output_snippet?: string;
|
||||
score?: number;
|
||||
action: "blocked" | "passed" | "flagged" | "not_run";
|
||||
action: "blocked" | "passed" | "flagged" | "skipped";
|
||||
model?: string;
|
||||
reason?: string;
|
||||
latency_ms?: number;
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ const PresidioPath = "@/components/view_logs/GuardrailViewer/PresidioDetectedEnt
|
|||
const BedrockPath = "@/components/view_logs/GuardrailViewer/BedrockGuardrailDetails";
|
||||
|
||||
const skippedPreCall: Partial<GuardrailInformation> = {
|
||||
guardrail_status: "not_run",
|
||||
guardrail_status: "skipped",
|
||||
guardrail_mode: "pre_call",
|
||||
guardrail_response: "no scannable content after message scoping",
|
||||
start_time: null,
|
||||
|
|
@ -68,15 +68,15 @@ describe("GuardrailViewer", () => {
|
|||
expect(screen.queryByText("FAILED")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("renders not_run as NOT RUN (muted) and keeps it out of the evaluated and passed counts", async () => {
|
||||
it("renders skipped as SKIPPED (muted) and keeps it out of the evaluated and passed counts", async () => {
|
||||
const user = userEvent.setup();
|
||||
const data = makeGuardrailInformation(skippedPreCall);
|
||||
renderWithProviders(<GuardrailViewer data={data} />);
|
||||
|
||||
expect(screen.getByText(/0 guardrails evaluated/)).toBeInTheDocument();
|
||||
expect(screen.getByText(/0 Passed/)).toHaveClass("text-muted-foreground");
|
||||
expect(screen.getByText(/1 Not run/)).toBeInTheDocument();
|
||||
const badge = screen.getByText("NOT RUN");
|
||||
expect(screen.getByText(/1 Skipped/)).toBeInTheDocument();
|
||||
const badge = screen.getByText("SKIPPED");
|
||||
expect(badge).toHaveClass("text-muted-foreground");
|
||||
expect(screen.queryByText("FAILED")).not.toBeInTheDocument();
|
||||
expect(screen.queryByText(/^T\+/)).not.toBeInTheDocument();
|
||||
|
|
@ -85,7 +85,7 @@ describe("GuardrailViewer", () => {
|
|||
expect(screen.getByText("no scannable content after message scoping")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("anchors the lifecycle timeline on timed entries when an untimed not_run entry sorts first", () => {
|
||||
it("anchors the lifecycle timeline on timed entries when an untimed skipped entry sorts first", () => {
|
||||
const skipped = makeGuardrailInformation({ ...skippedPreCall, guardrail_name: "skipped-rail" });
|
||||
const ran = makeGuardrailInformation(ranPostCall);
|
||||
renderWithProviders(<GuardrailViewer data={[skipped, ran]} />);
|
||||
|
|
|
|||
|
|
@ -134,13 +134,13 @@ const getTotalMasked = (entry: GuardrailInformation): number => {
|
|||
);
|
||||
};
|
||||
|
||||
type EntryOutcome = "passed" | "flagged" | "failed" | "not_run";
|
||||
type EntryOutcome = "passed" | "flagged" | "failed" | "skipped";
|
||||
|
||||
const getEntryOutcome = (entry: GuardrailInformation): EntryOutcome => {
|
||||
const status = (entry.guardrail_status ?? "").toLowerCase();
|
||||
if (status === "success") return "passed";
|
||||
if (status === "guardrail_flagged") return "flagged";
|
||||
if (status === "not_run") return "not_run";
|
||||
if (status === "skipped") return "skipped";
|
||||
return "failed";
|
||||
};
|
||||
|
||||
|
|
@ -150,18 +150,18 @@ const OUTCOME_LABEL: Record<EntryOutcome, string> = {
|
|||
passed: "PASSED",
|
||||
flagged: "FLAGGED",
|
||||
failed: "FAILED",
|
||||
not_run: "NOT RUN",
|
||||
skipped: "SKIPPED",
|
||||
};
|
||||
|
||||
const OUTCOME_BADGE_CLASS: Record<EntryOutcome, string> = {
|
||||
passed: "bg-success/15 text-success border border-success/20",
|
||||
flagged: "bg-warning/15 text-warning border border-warning/20",
|
||||
failed: "bg-destructive/15 text-destructive border border-destructive/20",
|
||||
not_run: "bg-muted text-muted-foreground border border-border",
|
||||
skipped: "bg-muted text-muted-foreground border border-border",
|
||||
};
|
||||
|
||||
const getHeaderOutcome = (counts: { evaluated: number; passed: number; flagged: number }): EntryOutcome => {
|
||||
if (counts.evaluated === 0) return "not_run";
|
||||
if (counts.evaluated === 0) return "skipped";
|
||||
if (counts.passed === counts.evaluated) return "passed";
|
||||
if (counts.passed + counts.flagged === counts.evaluated) return "flagged";
|
||||
return "failed";
|
||||
|
|
@ -242,7 +242,7 @@ const FlagCircleIcon = ({ className }: { className?: string }) => (
|
|||
const OutcomeIcon = ({ outcome }: { outcome: EntryOutcome }) => {
|
||||
if (outcome === "passed") return <CheckCircleIcon />;
|
||||
if (outcome === "flagged") return <FlagCircleIcon />;
|
||||
if (outcome === "not_run") return <GrayDotIcon />;
|
||||
if (outcome === "skipped") return <GrayDotIcon />;
|
||||
return <FailCircleIcon />;
|
||||
};
|
||||
|
||||
|
|
@ -675,7 +675,7 @@ const EvaluationCard = ({ entry }: { entry: GuardrailInformation }) => {
|
|||
</div>
|
||||
)}
|
||||
|
||||
{outcome === "not_run" && typeof guardrailResponse === "string" && (
|
||||
{outcome === "skipped" && typeof guardrailResponse === "string" && (
|
||||
<p className="text-sm text-muted-foreground">{guardrailResponse}</p>
|
||||
)}
|
||||
|
||||
|
|
@ -717,8 +717,8 @@ const GuardrailViewer = ({ data, accessToken, logEntry }: GuardrailViewerProps)
|
|||
|
||||
const passedCount = guardrailEntries.filter(isEntrySuccess).length;
|
||||
const flaggedCount = guardrailEntries.filter((e) => getEntryOutcome(e) === "flagged").length;
|
||||
const notRunCount = guardrailEntries.filter((e) => getEntryOutcome(e) === "not_run").length;
|
||||
const evaluatedCount = guardrailEntries.length - notRunCount;
|
||||
const skippedCount = guardrailEntries.filter((e) => getEntryOutcome(e) === "skipped").length;
|
||||
const evaluatedCount = guardrailEntries.length - skippedCount;
|
||||
const allPassed = evaluatedCount > 0 && passedCount === evaluatedCount;
|
||||
const headerOutcome = getHeaderOutcome({ evaluated: evaluatedCount, passed: passedCount, flagged: flaggedCount });
|
||||
|
||||
|
|
@ -778,11 +778,11 @@ const GuardrailViewer = ({ data, accessToken, logEntry }: GuardrailViewerProps)
|
|||
{flaggedCount} Flagged
|
||||
</span>
|
||||
)}
|
||||
{notRunCount > 0 && (
|
||||
{skippedCount > 0 && (
|
||||
<span
|
||||
className={`inline-flex items-center px-2 py-0.5 rounded-full text-xs font-semibold ${OUTCOME_BADGE_CLASS.not_run}`}
|
||||
className={`inline-flex items-center px-2 py-0.5 rounded-full text-xs font-semibold ${OUTCOME_BADGE_CLASS.skipped}`}
|
||||
>
|
||||
{notRunCount} Not run
|
||||
{skippedCount} Skipped
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -637,20 +637,20 @@ describe("GuardrailJumpLink", () => {
|
|||
});
|
||||
|
||||
it.each([
|
||||
[["success", "not_run"], "text-success", "\u2713"],
|
||||
[["guardrail_intervened", "not_run"], "text-destructive", "\u2717"],
|
||||
])("ignores not_run when styling %j as %s", (statuses, expectedClass, glyph) => {
|
||||
[["success", "skipped"], "text-success", "\u2713"],
|
||||
[["guardrail_intervened", "skipped"], "text-destructive", "\u2717"],
|
||||
])("ignores skipped when styling %j as %s", (statuses, expectedClass, glyph) => {
|
||||
render(<GuardrailJumpLink guardrailEntries={statuses.map((s) => ({ guardrail_status: s }))} />);
|
||||
|
||||
const pill = screen.getByText(/1 guardrail evaluated, 1 not run/);
|
||||
const pill = screen.getByText(/1 guardrail evaluated, 1 skipped/);
|
||||
expect(pill).toHaveClass(expectedClass);
|
||||
expect(pill).toHaveTextContent(glyph);
|
||||
});
|
||||
|
||||
it("renders an all not_run request as neutral rather than passed", () => {
|
||||
render(<GuardrailJumpLink guardrailEntries={[{ guardrail_status: "not_run" }]} />);
|
||||
it("renders an all skipped request as neutral rather than passed", () => {
|
||||
render(<GuardrailJumpLink guardrailEntries={[{ guardrail_status: "skipped" }]} />);
|
||||
|
||||
const pill = screen.getByText(/0 guardrails evaluated, 1 not run/);
|
||||
const pill = screen.getByText(/0 guardrails evaluated, 1 skipped/);
|
||||
expect(pill).toHaveClass("text-muted-foreground");
|
||||
expect(pill).not.toHaveTextContent("\u2713");
|
||||
});
|
||||
|
|
|
|||
|
|
@ -700,15 +700,15 @@ const GUARDRAIL_JUMP_LINK_STYLE = {
|
|||
passed: { className: "border border-success/20 bg-success/10 text-success", glyph: "\u2713" },
|
||||
flagged: { className: "border border-warning/20 bg-warning/10 text-warning", glyph: "\u26A0" },
|
||||
failed: { className: "border border-destructive/20 bg-destructive/10 text-destructive", glyph: "\u2717" },
|
||||
not_run: { className: "border border-border bg-muted text-muted-foreground", glyph: "\u2013" },
|
||||
skipped: { className: "border border-border bg-muted text-muted-foreground", glyph: "\u2013" },
|
||||
} as const;
|
||||
|
||||
const isPassedStatus = (status: unknown) => status === "pass" || status === "passed" || status === "success";
|
||||
const isFlaggedStatus = (status: unknown) => status === "flagged" || status === "guardrail_flagged";
|
||||
const isNotRunStatus = (status: unknown) => status === "not_run";
|
||||
const isSkippedStatus = (status: unknown) => status === "skipped";
|
||||
|
||||
const guardrailJumpLinkOutcome = (evaluated: unknown[]): keyof typeof GUARDRAIL_JUMP_LINK_STYLE => {
|
||||
if (evaluated.length === 0) return "not_run";
|
||||
if (evaluated.length === 0) return "skipped";
|
||||
if (evaluated.every(isPassedStatus)) return "passed";
|
||||
if (evaluated.every((s) => isPassedStatus(s) || isFlaggedStatus(s))) return "flagged";
|
||||
return "failed";
|
||||
|
|
@ -716,8 +716,8 @@ const guardrailJumpLinkOutcome = (evaluated: unknown[]): keyof typeof GUARDRAIL_
|
|||
|
||||
export function GuardrailJumpLink({ guardrailEntries }: { guardrailEntries: any[] }) {
|
||||
const statuses = guardrailEntries.map((e) => e?.guardrail_status || e?.status);
|
||||
const evaluated = statuses.filter((s) => !isNotRunStatus(s));
|
||||
const notRunCount = statuses.length - evaluated.length;
|
||||
const evaluated = statuses.filter((s) => !isSkippedStatus(s));
|
||||
const skippedCount = statuses.length - evaluated.length;
|
||||
const { className, glyph } = GUARDRAIL_JUMP_LINK_STYLE[guardrailJumpLinkOutcome(evaluated)];
|
||||
|
||||
const handleClick = () => {
|
||||
|
|
@ -743,7 +743,7 @@ export function GuardrailJumpLink({ guardrailEntries }: { guardrailEntries: any[
|
|||
>
|
||||
{glyph} {evaluated.length} guardrail
|
||||
{evaluated.length !== 1 ? "s" : ""} evaluated
|
||||
{notRunCount > 0 ? `, ${notRunCount} not run` : ""}
|
||||
{skippedCount > 0 ? `, ${skippedCount} skipped` : ""}
|
||||
<span style={{ fontSize: 11, opacity: 0.7 }}>{"\u2193"}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue