mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-16 23:41:43 +00:00
fix(guardrails): key not_run index rows by the sibling evaluation's guardrail_id
A not_run entry from the base guardrail only carries guardrail_name, while the content filter's evaluated entry carries guardrail_id. Keyed apart, one request listed twice in the monitor for a logging_only guardrail (Not run and Passed). Resolve the id from a same-name sibling in the payload so the severity pick applies Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
80725addb4
commit
c3f52fe0d5
2 changed files with 39 additions and 2 deletions
|
|
@ -365,8 +365,17 @@ async def process_spend_logs_guardrail_usage(
|
|||
continue
|
||||
date_key = _date_str(start_time)
|
||||
|
||||
for entry in _parse_guardrail_info_from_payload(payload):
|
||||
guardrail_id = entry.get("guardrail_id") or entry.get("guardrail_name") or ""
|
||||
entries = _parse_guardrail_info_from_payload(payload)
|
||||
ids_by_name = MappingProxyType(
|
||||
{
|
||||
e["guardrail_name"]: e["guardrail_id"]
|
||||
for e in entries
|
||||
if e.get("guardrail_id") and e.get("guardrail_name")
|
||||
}
|
||||
)
|
||||
for entry in entries:
|
||||
guardrail_name = entry.get("guardrail_name") or ""
|
||||
guardrail_id = entry.get("guardrail_id") or ids_by_name.get(guardrail_name) or guardrail_name
|
||||
if not guardrail_id:
|
||||
continue
|
||||
action = guardrail_status_to_action(entry.get("guardrail_status"))
|
||||
|
|
|
|||
|
|
@ -369,6 +369,34 @@ async def test_not_run_entries_are_indexed_but_not_counted_as_evaluations():
|
|||
assert sorted(row["request_id"] for row in index_rows) == ["r1", "r2"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_not_run_entry_shares_index_key_with_evaluated_sibling_of_same_name():
|
||||
"""
|
||||
The not_run entry from the shared base guardrail carries only guardrail_name,
|
||||
while the evaluated entry from the same guardrail (e.g. content filter on the
|
||||
output of a logging_only run) carries its guardrail_id. Keying them differently
|
||||
lists one request twice in the monitor, once as not_run and once as passed.
|
||||
"""
|
||||
prisma = _prisma()
|
||||
payload = _payload("r1")
|
||||
payload["metadata"] = json.dumps(
|
||||
{
|
||||
"guardrail_information": [
|
||||
{"guardrail_name": "cf", "guardrail_status": "not_run"},
|
||||
{"guardrail_name": "cf", "guardrail_id": "cf-uuid", "guardrail_status": "success"},
|
||||
{"guardrail_name": "other", "guardrail_status": "not_run"},
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
await process_spend_logs_guardrail_usage(prisma, [payload])
|
||||
|
||||
index_rows = prisma.db.litellm_spendlogguardrailindex.create_many.call_args.kwargs["data"]
|
||||
assert sorted(row["guardrail_id"] for row in index_rows) == ["cf-uuid", "cf-uuid", "other"]
|
||||
metrics_create = prisma.db.litellm_dailyguardrailmetrics.upsert.call_args.kwargs["data"]["create"]
|
||||
assert (metrics_create["guardrail_id"], metrics_create["requests_evaluated"]) == ("cf-uuid", 1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_batch_of_only_not_run_entries_writes_no_metrics_row():
|
||||
prisma = _prisma()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue