feat(shadow-eval): add optional rubric grades (#52321)

* feat(shadow-eval): add optional rubric grades

* chore(shadow-eval): generate rubric API types

* test(shadow-eval): cover unavailable rubric result storage

---------

Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com>
This commit is contained in:
joshua-berri 2026-11-17 12:04:31 -08:00 • committed by GitHub
parent 57560551a0
commit 45115cbfa2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 1467 additions and 27 deletions

View file

@ -12,6 +12,7 @@ across pods or stop races; the hook reads active jobs through a short-TTL cache.
import asyncio
import hashlib
import json
import os
import random
import traceback
from collections.abc import Awaitable, Callable, Mapping, Sequence
@ -44,9 +45,10 @@ from litellm.llms.base_llm.base_utils import type_to_response_format_param
from litellm.redis.stores.spend_counters import (
shadow_eval_counter,
)
from litellm.secret_managers.main import str_to_bool
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.management_endpoints.auto_router_endpoints import ShadowEvalDirection
from litellm.types.utils import SHADOW_EVAL_JUDGE_CALL_ORIGIN, SHADOW_EVAL_ROUTER_CALL_ORIGIN
from litellm.types.management_endpoints.auto_router_endpoints import SHADOW_EVAL_RUBRIC_VERSION, ShadowEvalDirection
from litellm.types.utils import SHADOW_EVAL_JUDGE_CALL_ORIGIN, SHADOW_EVAL_ROUTER_CALL_ORIGIN, ModelResponse
if TYPE_CHECKING:
from litellm.proxy.db.shadow_eval_funnel import ShadowEvalFunnelStage
@ -485,6 +487,76 @@ class PairwiseVerdict(LiteLLMBaseModel):
PAIRWISE_JUDGE_RESPONSE_FORMAT: Final = type_to_response_format_param(PairwiseVerdict)
_RUBRIC_JUDGE_SYSTEM_PROMPT_TEMPLATE: Final = """You are an impartial quality judge grading two responses to the same conversation
against an operator's rubric.
The responses are labeled A and B in random order. You do not know which system produced which.
A response may be prose, or a tool call shown as `[tool call] name(arguments)` if the
assistant chose to act instead of answering directly. A tool call is not a defect: judge
whether calling that tool was the right response to the conversation, the same as you
would judge prose.
Rubric:
<rubric>
{rubric}
</rubric>
Grade each response on its own against the rubric with an integer from 1 to 5:
1 = fails the rubric's criteria
2 = meets few of the criteria
3 = meets about half of the criteria
4 = meets most of the criteria
5 = fully meets the criteria
Then say which response is better overall. Equal grades may still have a better response.
Return ONLY valid JSON in this exact format, no other text:
{{
"preference": "A" | "B" | "tie",
"confidence": <0.0 to 1.0>,
"grade_a": <integer 1 to 5>,
"grade_b": <integer 1 to 5>
}}"""
def rubric_judge_system_prompt(rubric: str) -> str:
return _RUBRIC_JUDGE_SYSTEM_PROMPT_TEMPLATE.format(rubric=rubric)
class RubricVerdict(LiteLLMBaseModel):
"""The rubric judge's blind verdict: the pairwise preference plus one grade per label.
A grade is exactly a JSON integer from 1 to 5; booleans, floats, strings, null, and
out-of-range values invalidate the whole verdict rather than being coerced."""
model_config = ConfigDict(frozen=True)
preference: Literal["A", "B", "tie"]
confidence: float
grade_a: Literal[1, 2, 3, 4, 5]
grade_b: Literal[1, 2, 3, 4, 5]
@field_validator("grade_a", "grade_b", mode="before")
@classmethod
def _exact_integer(cls, value: object) -> object:
if type(value) is not int:
raise ValueError("grade must be a JSON integer")
return value
RUBRIC_JUDGE_RESPONSE_FORMAT: Final = _CHAT_REQUEST_ADAPTER.validate_python(
type_to_response_format_param(RubricVerdict)
)
SHADOW_EVAL_RUBRIC_ENABLED_ENV_VAR: Final = "LITELLM_SHADOW_EVAL_RUBRIC_ENABLED"
def is_shadow_eval_rubric_enabled() -> bool:
"""Whether this process may start and sample rubric shadow evals. Off by default: a
worker without rubric support would sample a rubric job as a plain preference job, so
operators turn this on only once every worker sharing the database runs rubric support."""
return str_to_bool(os.environ.get(SHADOW_EVAL_RUBRIC_ENABLED_ENV_VAR)) is True
def _sample_hits(request_id: str, job_id: str, percentage: float) -> bool:
"""Deterministically decide whether a request falls in the shadowed slice: hash-based
@ -516,6 +588,30 @@ def _judge_reply_shape(response: object) -> str:
return f"finish_reason={_chat_finish_reason(response)}, content={body}, model={served}"
def _legacy_judge_call_failure(e: Exception) -> str:
return f"judge call failed: {e}"
def _rubric_judge_call_failure(e: Exception) -> str:
"""Provider errors can echo the request or response, which carry the rubric and the
sampled turns, so a rubric row keeps only the exception type and HTTP status."""
status: Final = getattr(e, "status_code", None)
detail: Final = f"type={type(e).__name__}" + (f", status={status}" if isinstance(status, int) else "")
return f"rubric judge call failed; {detail}"
def _invalid_verdict_detail(e: Exception) -> str:
"""Which verdict fields failed and how, from validation metadata alone, so the row
never carries the judge's reply text."""
if not isinstance(e, ValidationError):
return f"invalid=verdict:{type(e).__name__}"
fields: Final = ",".join(
f"{'.'.join(str(part) for part in error['loc']) or 'verdict'}:{error['type']}"
for error in e.errors(include_url=False, include_context=False, include_input=False)
)
return f"invalid={fields}"
def _call_cost(response: object) -> float:
"""Price one eval-arm call with the figure the spend pipeline bills: the router client
stamps _hidden_params.response_cost from the deployment's own pricing, which the public
@ -537,6 +633,10 @@ def _price_map_cost(response: object) -> float:
return 0.0
def _coin_flip() -> bool:
return random.random() < 0.5
def _unmask_preference(raw_preference: str, real_is_a: bool) -> str:
"""Map the judge's blind A/B/tie verdict back to real/shadow/tie."""
normalized: Final = raw_preference.strip().lower()
@ -737,6 +837,23 @@ class _JudgeVerdict:
cost: float
@dataclass(frozen=True, slots=True)
class _RubricJudgeVerdict:
"""A parsed rubric verdict, preference and grades unmasked back to real/shadow."""
preference: str
confidence: float
cost: float
real_grade: int
shadow_grade: int
@dataclass(frozen=True, slots=True)
class _JudgeReply:
response: ModelResponse
real_is_a: bool
class ActiveShadowEvalJob(LiteLLMBaseModel):
"""One active job as the sampling path needs it, validated straight off the untyped
job row: immutable config plus the attempt count as of the cache fill (the turn
@ -757,9 +874,21 @@ class ActiveShadowEvalJob(LiteLLMBaseModel):
max_turns: int
max_budget: float | None = None
ends_at: datetime
rubric: str | None = None
rubric_version: int | None = None
attempts: int = 0
spend: float = 0.0
@model_validator(mode="after")
def _rubric_runs_under_a_known_contract(self) -> "ActiveShadowEvalJob":
"""A rubric row this worker cannot grade under the contract it was created with
fails closed rather than sampling as a preference-only job."""
if self.rubric is None and self.rubric_version is None:
return self
if self.rubric is None or self.rubric_version != SHADOW_EVAL_RUBRIC_VERSION:
raise ValueError("a rubric job needs its rubric and a grading version this worker implements")
return self
@field_validator("ends_at")
@classmethod
def _as_utc(cls, value: datetime) -> datetime:
@ -916,6 +1045,8 @@ class ShadowEvalLogger(CustomLogger):
job_spend_reader: Callable[[str, float, float], Awaitable[float]] | None = None,
job_spend_writer: Callable[[str, float], Awaitable[None]] | None = None,
funnel_recorder: Callable[[str, "ShadowEvalFunnelStage"], None] | None = None,
real_is_a_draw: Callable[[], bool] | None = None,
rubric_enabled: Callable[[], bool] | None = None,
) -> None:
"""Providers are callables so the proxy's lazily-initialized globals are resolved
at call time, not at logger construction. The spend reader and writer wrap the
@ -926,6 +1057,8 @@ class ShadowEvalLogger(CustomLogger):
self._read_job_spend = job_spend_reader or _job_spend_from_counter
self._write_job_spend = job_spend_writer or _add_job_spend_to_counter
self._record_funnel = funnel_recorder or _record_funnel_event
self._draw_real_is_a = real_is_a_draw or _coin_flip
self._rubric_enabled = rubric_enabled or is_shadow_eval_rubric_enabled
self._inflight_shadow_tasks: int = 0
# Starts per job since the last cache fill, never decremented within a
# generation; the refill absorbs written rows and resets.
@ -1006,9 +1139,11 @@ class ShadowEvalLogger(CustomLogger):
eligible: list[ActiveShadowEvalJob] = [] # mutable-ok: bucketed per-job admission
now: Final = datetime.now(timezone.utc)
router: Final = self._router_provider()
rubric_enabled: Final = self._rubric_enabled()
for job in active_jobs:
if (
now >= job.ends_at
(job.rubric is not None and not rubric_enabled)
or now >= job.ends_at
or job.attempts + self._job_starts.get(job.id, 0) >= job.max_turns
or (job.max_budget is not None and job.spend >= job.max_budget)
or not _direction_admits(request_metadata, job)
@ -1243,13 +1378,25 @@ class ShadowEvalLogger(CustomLogger):
return
# From here the shadow call has billed, so every exit records its cost.
try:
verdict: Final = await self._call_judge(
judge_model=job.judge_model,
messages=messages,
real_text=real_text,
shadow_text=shadow.text,
tools=shadow_params.get("tools"),
parent_metadata=parent_metadata,
verdict: Final = (
await self._call_judge(
judge_model=job.judge_model,
messages=messages,
real_text=real_text,
shadow_text=shadow.text,
tools=shadow_params.get("tools"),
parent_metadata=parent_metadata,
)
if job.rubric is None
else await self._call_rubric_judge(
judge_model=job.judge_model,
rubric=job.rubric,
messages=messages,
real_text=real_text,
shadow_text=shadow.text,
tools=shadow_params.get("tools"),
parent_metadata=parent_metadata,
)
)
if isinstance(verdict, _CallFailure):
await self._record_attempt(
@ -1279,6 +1426,9 @@ class ShadowEvalLogger(CustomLogger):
shadow=shadow,
real_model=real_model,
confidence=verdict.confidence,
grades=(
(verdict.real_grade, verdict.shadow_grade) if isinstance(verdict, _RubricJudgeVerdict) else None
),
judge_cost=verdict.cost,
shadow_cost=shadow.cost,
shadow_classifier_cost=shadow.classifier_cost,
@ -1323,6 +1473,7 @@ class ShadowEvalLogger(CustomLogger):
shadow_cost: float = 0.0,
shadow_classifier_cost: float = 0.0,
error: str | None = None,
grades: tuple[int, int] | None = None,
) -> None:
eval_spend: Final = judge_cost + shadow_cost + shadow_classifier_cost
if eval_spend > 0:
@ -1347,6 +1498,7 @@ class ShadowEvalLogger(CustomLogger):
"real_classifier_cost": real_classifier_cost,
"real_cache_hit": real_cache_hit,
"error": error[:_MAX_ERROR_CHARS] if error else None,
**({"real_grade": grades[0], "shadow_grade": grades[1]} if grades is not None else {}),
}
)
except Exception as e: # noqa: BLE001 # a lost row degrades sample size, nothing can disagree with it
@ -1402,19 +1554,22 @@ class ShadowEvalLogger(CustomLogger):
classifier_cost=_decision_classifier_cost(shadow_metadata),
)
async def _call_judge(
async def _judge_reply(
self,
judge_model: str,
system_prompt: str,
response_format: Mapping[str, object] | None,
messages: Sequence[Mapping[str, object]],
real_text: str,
shadow_text: str,
tools: object,
parent_metadata: Mapping[str, object],
) -> "_JudgeVerdict | _CallFailure":
"""Blind pairwise judge with A/B labels randomized to cancel position bias. Both
call_failure: Callable[[Exception], str],
) -> "_JudgeReply | _CallFailure":
"""One blind judge call with A/B labels randomized to cancel position bias. Both
arms were offered the same tools, so the judge is shown their definitions too: a
tool call is only assessable against what else was available to call instead."""
real_is_a: Final = random.random() < 0.5
real_is_a: Final = self._draw_real_is_a()
response_a: Final = real_text if real_is_a else shadow_text
response_b: Final = shadow_text if real_is_a else real_text
@ -1425,7 +1580,7 @@ class ShadowEvalLogger(CustomLogger):
)
judge_metadata: Final = sanitized_forwardable_call_metadata(parent_metadata, SHADOW_EVAL_JUDGE_CALL_ORIGIN)
judge_messages: Final = [
{"role": "system", "content": PAIRWISE_JUDGE_SYSTEM_PROMPT},
{"role": "system", "content": system_prompt},
{
"role": "user",
"content": _judge_user_prompt(conversation, response_a, response_b, _tool_definitions_text(tools)),
@ -1439,12 +1594,37 @@ class ShadowEvalLogger(CustomLogger):
team_id=_forwarded_team_id(parent_metadata),
temperature=0,
max_tokens=JUDGE_MAX_OUTPUT_TOKENS,
response_format=PAIRWISE_JUDGE_RESPONSE_FORMAT,
response_format=response_format,
metadata=judge_metadata,
)
except Exception as e: # noqa: BLE001 # judge outages become error rows, not crashes
verbose_logger.debug("shadow_eval: judge call failed: %s", e)
return _CallFailure(f"judge call failed: {e}")
return _CallFailure(call_failure(e))
return _JudgeReply(response=response, real_is_a=real_is_a)
async def _call_judge(
self,
judge_model: str,
messages: Sequence[Mapping[str, object]],
real_text: str,
shadow_text: str,
tools: object,
parent_metadata: Mapping[str, object],
) -> "_JudgeVerdict | _CallFailure":
reply: Final = await self._judge_reply(
judge_model,
PAIRWISE_JUDGE_SYSTEM_PROMPT,
PAIRWISE_JUDGE_RESPONSE_FORMAT,
messages,
real_text,
shadow_text,
tools,
parent_metadata,
_legacy_judge_call_failure,
)
if isinstance(reply, _CallFailure):
return reply
response: Final = reply.response
try:
raw: Final = response["choices"][0]["message"]["content"] or ""
verdict: Final = PairwiseVerdict.model_validate(parse_json_verdict(raw))
@ -1454,11 +1634,56 @@ class ShadowEvalLogger(CustomLogger):
f"unparseable judge verdict: {e}; {_judge_reply_shape(response)}", cost=_call_cost(response)
)
return _JudgeVerdict(
preference=_unmask_preference(verdict.preference, real_is_a),
preference=_unmask_preference(verdict.preference, reply.real_is_a),
confidence=max(0.0, min(1.0, verdict.confidence)),
cost=_call_cost(response),
)
async def _call_rubric_judge(
self,
judge_model: str,
rubric: str,
messages: Sequence[Mapping[str, object]],
real_text: str,
shadow_text: str,
tools: object,
parent_metadata: Mapping[str, object],
) -> "_RubricJudgeVerdict | _CallFailure":
"""The pairwise judge plus a grade per response. The whole verdict validates before
anything counts, so a malformed grade is an error row carrying the judge's cost,
never a preference-only or defaulted verdict. Errors name the invalid fields and the
reply's shape, never the reply or the rubric."""
reply: Final = await self._judge_reply(
judge_model,
rubric_judge_system_prompt(rubric),
RUBRIC_JUDGE_RESPONSE_FORMAT,
messages,
real_text,
shadow_text,
tools,
parent_metadata,
_rubric_judge_call_failure,
)
if isinstance(reply, _CallFailure):
return reply
response: Final = reply.response
try:
raw: Final = response["choices"][0]["message"]["content"] or ""
verdict: Final = RubricVerdict.model_validate(parse_json_verdict(raw))
except Exception as e: # noqa: BLE001 # malformed verdicts become error rows
verbose_logger.debug("shadow_eval: unparseable rubric verdict: %s", type(e).__name__)
return _CallFailure(
f"unparseable rubric verdict; {_invalid_verdict_detail(e)}, {_judge_reply_shape(response)}",
cost=_call_cost(response),
)
return _RubricJudgeVerdict(
preference=_unmask_preference(verdict.preference, reply.real_is_a),
confidence=max(0.0, min(1.0, verdict.confidence)),
cost=_call_cost(response),
real_grade=verdict.grade_a if reply.real_is_a else verdict.grade_b,
shadow_grade=verdict.grade_b if reply.real_is_a else verdict.grade_a,
)
_EMPTY_JOBS: Final[Mapping[tuple[str, str], tuple[ActiveShadowEvalJob, ...]]] = MappingProxyType({})

View file

@ -0,0 +1,13 @@
SET LOCAL lock_timeout = '10s';
-- AlterTable
ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN IF NOT EXISTS "rubric" TEXT;
-- AlterTable
ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN IF NOT EXISTS "rubric_version" INTEGER;
-- AlterTable
ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN IF NOT EXISTS "real_grade" INTEGER;
-- AlterTable
ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN IF NOT EXISTS "shadow_grade" INTEGER;

View file

@ -2091,6 +2091,8 @@ model LiteLLM_ShadowEvalJob {
ends_at DateTime
stopped_at DateTime?
stopped_by String? // operator who stopped it early; null when it ended on its own
rubric String? // criteria both responses are graded against; null on preference-only jobs
rubric_version Int? // grading contract the rubric job runs under; set exactly when rubric is
@@index([group_id])
@@index([target_type, target_id])
@ -2114,6 +2116,8 @@ model LiteLLM_ShadowEvalAttempt {
real_classifier_cost Float @default(0)
shadow_classifier_cost Float @default(0)
real_cache_hit Boolean @default(false)
real_grade Int? // rubric jobs: 1-5 grade of the response the caller received; null = ungraded
shadow_grade Int? // rubric jobs: 1-5 grade of the shadow response; null = ungraded
error String?
created_at DateTime @default(now())

View file

@ -73,6 +73,7 @@ from litellm.router_strategy.complexity_router.request_models import (
)
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.management_endpoints.auto_router_endpoints import (
SHADOW_EVAL_RUBRIC_VERSION,
SHADOW_EVAL_TURN_VALVE,
AutoRouterAvailabilityRequest,
AutoRouterAvailabilityResponse,
@ -86,12 +87,17 @@ from litellm.types.management_endpoints.auto_router_endpoints import (
ComplexityRouterConfigValidationRequest,
ComplexityRouterConfigValidationResponse,
ShadowEvalDirection,
ShadowEvalGradePair,
ShadowEvalJobResponse,
ShadowEvalJobTargetResponse,
ShadowEvalResult,
ShadowEvalRubricJobResponse,
ShadowEvalRubricResultResponse,
ShadowEvalRubricSlice,
ShadowEvalSlice,
ShadowEvalTargetType,
StartShadowEvalRequest,
StartShadowEvalRubricRequest,
)
if TYPE_CHECKING:
@ -1462,6 +1468,51 @@ class _AttemptTotalsRow(LiteLLMBaseModel):
_ATTEMPT_TOTALS_ROWS: Final = TypeAdapter(list[_AttemptTotalsRow])
# Grades are counted per arm on the same rows the preference results read (errors
# excluded); a verdict row without grades is counted as ungraded, never as a grade.
_RUBRIC_GRADE_COUNTS_SQL: Final = """
SELECT COALESCE(a.router_name, j.router_name) AS grp, a.real_grade, a.shadow_grade, COUNT(*)::int AS turn_count
FROM "LiteLLM_ShadowEvalAttempt" a
JOIN "LiteLLM_ShadowEvalJob" j ON j.id = a.job_id
WHERE a.job_id = ANY($1::text[]) AND a.outcome != 'error'
GROUP BY 1, 2, 3
"""
class _GradeCountRow(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
grp: str
real_grade: int | None
shadow_grade: int | None
turn_count: int
@property
def grade_pair(self) -> tuple[int, int] | None:
if self.real_grade is None or self.shadow_grade is None:
return None
return self.real_grade, self.shadow_grade
_GRADE_COUNT_ROWS: Final = TypeAdapter(list[_GradeCountRow])
def _rubric_slice(group: str, rows: Sequence[_GradeCountRow]) -> ShadowEvalRubricSlice:
pairs: Final = sorted(frozenset(pair for row in rows if (pair := row.grade_pair) is not None))
return ShadowEvalRubricSlice(
group=group,
graded_count=sum(row.turn_count for row in rows if row.grade_pair is not None),
ungraded_count=sum(row.turn_count for row in rows if row.grade_pair is None),
grade_distribution=tuple(
ShadowEvalGradePair(
real_grade=real_grade,
shadow_grade=shadow_grade,
turn_count=sum(row.turn_count for row in rows if row.grade_pair == (real_grade, shadow_grade)),
)
for real_grade, shadow_grade in pairs
),
)
def _pct_of(numerator: int, denominator: int) -> float:
return _pct(numerator, denominator)
@ -1514,6 +1565,8 @@ class _LegRow(LiteLLMBaseModel):
ends_at: datetime
stopped_at: datetime | None = None
stopped_by: str | None = None
rubric: str | None = None
rubric_version: int | None = None
@property
def arm_router_names(self) -> tuple[str, ...]:
@ -1756,9 +1809,17 @@ async def start_shadow_eval(
seconds. Shadow and judge calls bill to the sampled request's own identity but are
excluded from request counts and auto-router adoption metrics.
"""
_require_admin_writer(user_api_key_dict, "start a shadow eval")
return await _create_shadow_eval_job(data, user_api_key_dict, rubric=None)
async def _create_shadow_eval_job(
data: StartShadowEvalRequest, user_api_key_dict: UserAPIKeyAuth, rubric: str | None
) -> ShadowEvalJobResponse:
"""Validate and claim a job for an authorized writer. The rubric columns are written
only for a rubric job, so a preference-only job inserts exactly what it always did."""
from litellm.proxy.proxy_server import llm_router, prisma_client
_require_admin_writer(user_api_key_dict, "start a shadow eval")
if prisma_client is None:
raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
unconfigured: Final = tuple(
@ -1878,6 +1939,7 @@ async def start_shadow_eval(
"created_by": user_api_key_dict.user_id,
"created_at": now,
"ends_at": ends_at,
**({"rubric": rubric, "rubric_version": SHADOW_EVAL_RUBRIC_VERSION} if rubric is not None else {}),
}
try:
# Leg ids are minted here rather than by the DB default so the funnel seed below
@ -1941,6 +2003,99 @@ async def start_shadow_eval(
)
@router.post(
"/auto_router/shadow_eval/rubric/start",
tags=["auto router"],
dependencies=(Depends(user_api_key_auth),),
response_model=ShadowEvalRubricJobResponse,
status_code=status.HTTP_201_CREATED,
)
async def start_shadow_eval_rubric(
data: StartShadowEvalRubricRequest,
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
) -> ShadowEvalRubricJobResponse:
"""
Start a shadow eval whose judge, besides picking the better response, grades each
response from 1 (fails the rubric's criteria) to 5 (fully meets them). Everything
else matches /auto_router/shadow_eval/start, and the job's preference results stay
on the regular job endpoints. Read grades from
/auto_router/shadow_eval/{job_id}/rubric_results.
Disabled unless LITELLM_SHADOW_EVAL_RUBRIC_ENABLED=true. Each worker process reads
its own environment, so a change applies only once every worker is redeployed or
restarted with it. A worker without rubric support ignores the setting and samples a
rubric job as a preference-only job, so enable it only after every API and sampling
worker sharing the database runs a version with this route. Workers with the setting
off neither start nor sample rubric jobs.
Rolling back to a version without rubric support is not zero-downtime for rubric
jobs. Before any older worker starts: apply the setting off to every upgraded worker,
stop every running rubric job with /auto_router/shadow_eval/{job_id}/stop, and wait
until every upgraded worker's in-flight evaluations have finished, for example by
draining or stopping those workers. The roughly 10 second job cache expiry alone does
not guarantee that drain.
"""
from litellm.integrations.shadow_eval_logger import (
SHADOW_EVAL_RUBRIC_ENABLED_ENV_VAR,
is_shadow_eval_rubric_enabled,
)
_require_admin_writer(user_api_key_dict, "start a shadow eval")
if not is_shadow_eval_rubric_enabled():
raise HTTPException(
status_code=400,
detail=(
f"Rubric shadow evals are disabled. Set {SHADOW_EVAL_RUBRIC_ENABLED_ENV_VAR}=true once every "
"worker sharing this database supports rubric grading"
),
)
job: Final = await _create_shadow_eval_job(data, user_api_key_dict, rubric=data.rubric)
return ShadowEvalRubricJobResponse.model_validate(
{
**job.model_dump(exclude={"router_name", "status"}),
"rubric": data.rubric,
"rubric_version": SHADOW_EVAL_RUBRIC_VERSION,
}
)
@router.get(
"/auto_router/shadow_eval/{job_id}/rubric_results",
tags=["auto router"],
dependencies=(Depends(user_api_key_auth),),
response_model=ShadowEvalRubricResultResponse,
)
async def get_shadow_eval_rubric_results(
job_id: str,
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
) -> ShadowEvalRubricResultResponse:
"""A rubric job's grades: graded and ungraded turn counts and the joint distribution
of (real_grade, shadow_grade), overall and per router arm. A job started without a
rubric is a 404 here."""
from litellm.proxy.proxy_server import prisma_client
_require_admin_viewer(user_api_key_dict, "view shadow evals")
if prisma_client is None:
raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
legs: Final = _LEG_ROWS.validate_python(
await _shadow_eval_jobs(prisma_client).find_many(where={"group_id": job_id}) or ()
)
first: Final = legs[0] if legs else None
if first is None or first.rubric is None or first.rubric_version is None:
raise HTTPException(status_code=404, detail=f"No rubric shadow eval job {job_id}")
rows: Final = _GRADE_COUNT_ROWS.validate_python(
await _query_raw(prisma_client, _RUBRIC_GRADE_COUNTS_SQL, [leg.id for leg in legs]) or ()
)
routers: Final = sorted(frozenset((*first.arm_router_names, *(row.grp for row in rows))))
return ShadowEvalRubricResultResponse(
job_id=job_id,
rubric=first.rubric,
rubric_version=first.rubric_version,
overall=_rubric_slice("all", rows),
by_router=tuple(_rubric_slice(name, tuple(row for row in rows if row.grp == name)) for name in routers),
)
@router.get(
"/auto_router/shadow_eval",
tags=("auto router",),

View file

@ -2091,6 +2091,8 @@ model LiteLLM_ShadowEvalJob {
ends_at DateTime
stopped_at DateTime?
stopped_by String? // operator who stopped it early; null when it ended on its own
rubric String? // criteria both responses are graded against; null on preference-only jobs
rubric_version Int? // grading contract the rubric job runs under; set exactly when rubric is
@@index([group_id])
@@index([target_type, target_id])
@ -2114,6 +2116,8 @@ model LiteLLM_ShadowEvalAttempt {
real_classifier_cost Float @default(0)
shadow_classifier_cost Float @default(0)
real_cache_hit Boolean @default(false)
real_grade Int? // rubric jobs: 1-5 grade of the response the caller received; null = ungraded
shadow_grade Int? // rubric jobs: 1-5 grade of the shadow response; null = ungraded
error String?
created_at DateTime @default(now())

View file

@ -2091,6 +2091,8 @@ model LiteLLM_ShadowEvalJob {
ends_at DateTime
stopped_at DateTime?
stopped_by String? // operator who stopped it early; null when it ended on its own
rubric String? // criteria both responses are graded against; null on preference-only jobs
rubric_version Int? // grading contract the rubric job runs under; set exactly when rubric is
@@index([group_id])
@@index([target_type, target_id])
@ -2114,6 +2116,8 @@ model LiteLLM_ShadowEvalAttempt {
real_classifier_cost Float @default(0)
shadow_classifier_cost Float @default(0)
real_cache_hit Boolean @default(false)
real_grade Int? // rubric jobs: 1-5 grade of the response the caller received; null = ungraded
shadow_grade Int? // rubric jobs: 1-5 grade of the shadow response; null = ungraded
error String?
created_at DateTime @default(now())

View file

@ -7,7 +7,7 @@ from collections.abc import Mapping
from datetime import datetime, timezone
from typing import Final, Literal, TypeAlias
from pydantic import Field, computed_field, field_validator, model_validator
from pydantic import ConfigDict, Field, computed_field, field_validator, model_validator
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.utils import StandardLoggingRoutingDecision
@ -246,6 +246,12 @@ SHADOW_EVAL_TURN_VALVE: Final[int] = 10_000
SHADOW_EVAL_MAX_ROUTERS: Final[int] = 4
SHADOW_EVAL_RUBRIC_MAX_CHARS: Final[int] = 4_000
# Identifies the rubric grading contract (prompt, anchors, 1-5 scale) a job was created
# under. A sampler only runs rubric jobs whose version it implements.
SHADOW_EVAL_RUBRIC_VERSION: Final[int] = 1
class StartShadowEvalRequest(LiteLLMBaseModel):
"""Start duplicating one or more targets' traffic for blind comparison against an auto-router.
@ -446,6 +452,27 @@ class StartShadowEvalRequest(LiteLLMBaseModel):
return self
class StartShadowEvalRubricRequest(StartShadowEvalRequest): # frozen-ok: parent validator assigns router_names
"""A shadow eval whose judge also grades each response from 1 to 5 against an operator rubric."""
rubric: str = Field(
min_length=1,
max_length=SHADOW_EVAL_RUBRIC_MAX_CHARS,
description=(
"The criteria the judge grades both responses against, in plain language, at most "
f"{SHADOW_EVAL_RUBRIC_MAX_CHARS} characters. Each response gets an integer grade on a fixed scale "
"where 1 fails the criteria and 5 fully meets them. Stored unchanged with the job and never edited"
),
)
@field_validator("rubric")
@classmethod
def _rubric_is_not_blank(cls, value: str) -> str:
if not value.strip():
raise ValueError("rubric must contain criteria, not only whitespace")
return value
class ShadowEvalSlice(LiteLLMBaseModel):
"""Judge outcomes for one slice of a job's verdicts: a router tier, one of the
models that served the real arm, or one scoped target (embedded on that target's
@ -697,6 +724,65 @@ class ShadowEvalJobResponse(LiteLLMBaseModel):
return "running"
class ShadowEvalRubricJobResponse(ShadowEvalJobResponse):
"""A rubric shadow eval job: the regular job view plus the rubric it grades against."""
model_config = ConfigDict(frozen=True)
rubric: str = Field(description="The criteria both responses are graded against, as stored at start")
rubric_version: int = Field(description="The grading contract (prompt, anchors, 1-5 scale) the job runs under")
class ShadowEvalGradePair(LiteLLMBaseModel):
"""How many judged turns gave the real and shadow responses this pair of grades."""
model_config = ConfigDict(frozen=True)
real_grade: int = Field(description="Grade of the response the caller actually received, 1 to 5")
shadow_grade: int = Field(description="Grade of the duplicated response nobody was served, 1 to 5")
turn_count: int
class ShadowEvalRubricSlice(LiteLLMBaseModel):
"""Rubric grades for one router arm, or for every arm together."""
model_config = ConfigDict(frozen=True)
group: str
graded_count: int = Field(description="Judged turns that carry both grades")
ungraded_count: int = Field(
description=(
"Judged turns with a preference but no grades, such as turns a worker without rubric support "
"judged. They are excluded from grade_distribution, never counted as low grades"
)
)
grade_distribution: tuple[ShadowEvalGradePair, ...] = Field(
description=(
"Joint counts of (real_grade, shadow_grade) over graded turns, ordered by real then shadow grade, "
"so turns where both responses were good or both were bad stay distinguishable"
)
)
class ShadowEvalRubricResultResponse(LiteLLMBaseModel):
"""A rubric job's grades so far. Error attempts are excluded, like the job's preference results."""
model_config = ConfigDict(frozen=True)
job_id: str
rubric: str
rubric_version: int
overall: ShadowEvalRubricSlice = Field(
description=(
"Every arm together. A judged turn is one (request, router arm) pair, so a multi-router job "
"counts each real response once per arm; compare routers with by_router"
)
)
by_router: tuple[ShadowEvalRubricSlice, ...] = Field(
description="One slice per router arm, ordered by router name, each graded against the same real responses"
)
def __getattr__(name: str) -> object:
if name in _REQUEST_MODELS_EXPORTS:
return getattr(importlib.import_module(_REQUEST_MODELS_MODULE), name)

View file

@ -1576,6 +1576,44 @@ export interface paths {
patch?: never;
trace?: never;
};
"/auto_router/shadow_eval/rubric/start": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
get?: never;
put?: never;
/**
* Start Shadow Eval Rubric
* @description Start a shadow eval whose judge, besides picking the better response, grades each
* response from 1 (fails the rubric's criteria) to 5 (fully meets them). Everything
* else matches /auto_router/shadow_eval/start, and the job's preference results stay
* on the regular job endpoints. Read grades from
* /auto_router/shadow_eval/{job_id}/rubric_results.
*
* Disabled unless LITELLM_SHADOW_EVAL_RUBRIC_ENABLED=true. Each worker process reads
* its own environment, so a change applies only once every worker is redeployed or
* restarted with it. A worker without rubric support ignores the setting and samples a
* rubric job as a preference-only job, so enable it only after every API and sampling
* worker sharing the database runs a version with this route. Workers with the setting
* off neither start nor sample rubric jobs.
*
* Rolling back to a version without rubric support is not zero-downtime for rubric
* jobs. Before any older worker starts: apply the setting off to every upgraded worker,
* stop every running rubric job with /auto_router/shadow_eval/{job_id}/stop, and wait
* until every upgraded worker's in-flight evaluations have finished, for example by
* draining or stopping those workers. The roughly 10 second job cache expiry alone does
* not guarantee that drain.
*/
post: operations["start_shadow_eval_rubric_auto_router_shadow_eval_rubric_start_post"];
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/auto_router/shadow_eval/start": {
parameters: {
query?: never;
@ -1641,6 +1679,28 @@ export interface paths {
patch?: never;
trace?: never;
};
"/auto_router/shadow_eval/{job_id}/rubric_results": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/**
* Get Shadow Eval Rubric Results
* @description A rubric job's grades: graded and ungraded turn counts and the joint distribution
* of (real_grade, shadow_grade), overall and per router arm. A job started without a
* rubric is a 404 here.
*/
get: operations["get_shadow_eval_rubric_results_auto_router_shadow_eval__job_id__rubric_results_get"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/auto_router/shadow_eval/{job_id}/stop": {
parameters: {
query?: never;
@ -48609,6 +48669,24 @@ export interface components {
/** Message */
message: string;
};
/**
* ShadowEvalGradePair
* @description How many judged turns gave the real and shadow responses this pair of grades.
*/
ShadowEvalGradePair: {
/**
* Real Grade
* @description Grade of the response the caller actually received, 1 to 5
*/
real_grade: number;
/**
* Shadow Grade
* @description Grade of the duplicated response nobody was served, 1 to 5
*/
shadow_grade: number;
/** Turn Count */
turn_count: number;
};
/**
* ShadowEvalJobResponse
* @description A shadow-eval job over one or more targets, each with its own budget and stop state;
@ -48821,6 +48899,158 @@ export interface components {
*/
withheld_count?: number | null;
};
/**
* ShadowEvalRubricJobResponse
* @description A rubric shadow eval job: the regular job view plus the rubric it grades against.
*/
ShadowEvalRubricJobResponse: {
/** Baseline Model */
baseline_model?: string | null;
/**
* Created At
* Format: date-time
*/
created_at: string;
/**
* Direction
* @default forward
* @enum {string}
*/
direction: "forward" | "reverse";
/**
* Ends At
* Format: date-time
*/
ends_at: string;
/**
* Error Count
* @description Sampled attempts that errored; detail endpoint only
*/
error_count?: number | null;
/** Job Id */
job_id: string;
/** Judge Model */
judge_model: string;
/**
* Judge Spend
* @description Judge cost so far; detail endpoint only
*/
judge_spend?: number | null;
/**
* Judged Count
* @description Verdicts recorded; detail endpoint only
*/
judged_count?: number | null;
/**
* Last Error
* @description Most recent attempt error; detail endpoint only
*/
last_error?: string | null;
/**
* Models
* @description Model groups the sampled traffic is narrowed to; empty means every model the targets use
* @default []
*/
models: string[];
/** @description Stratified verdicts; detail endpoint only */
results?: components["schemas"]["ShadowEvalResult"] | null;
/**
* Router Name
* @description The first router, kept for callers that predate router_names; derived so the
* two fields can never disagree.
*/
readonly router_name: string;
/**
* Router Names
* @description Every auto-router this job runs as a shadow arm. Multi-router jobs sample one slice of traffic and judge every arm against the same real responses
*/
router_names: string[];
/**
* Rubric
* @description The criteria both responses are graded against, as stored at start
*/
rubric: string;
/**
* Rubric Version
* @description The grading contract (prompt, anchors, 1-5 scale) the job runs under
*/
rubric_version: number;
/**
* Same Model Skipped Count
* @description Sampled requests skipped because baseline_model's own model group served them, summed over targets; detail endpoint only and reported even before any turn is judged. None when coverage counts are unknown for any target
*/
same_model_skipped_count?: number | null;
/** Shadow Percentage */
shadow_percentage: number;
/**
* Skip Same Model
* @description Whether this reverse job skips samples that baseline_model's own model group served
* @default false
*/
skip_same_model: boolean;
/**
* Status
* @description Three recorded facts, no history-guessing: a stop is stopped_by (the migration
* backfills it for every job that displayed stopped when the column arrived, so the
* pre-column population is closed), completion is the window passing or every target
* spending its budget, and anything else is running. The all-targets-stamped fallback
* covers only stops written by pre-column pods during a rolling deploy.
* @enum {string}
*/
readonly status: "running" | "completed" | "stopped";
/**
* Stopped By
* @description The operator who stopped the job early, recorded by the stop endpoint; 'unknown' backfilled by migration for jobs that displayed stopped when the column arrived; None when the job ended on its own. Its presence is what makes a job read stopped rather than completed
*/
stopped_by?: string | null;
/**
* Targets
* @description The targets whose traffic this job evaluates, and only theirs, each with its own budget
*/
targets: components["schemas"]["ShadowEvalJobTargetResponse"][];
};
/**
* ShadowEvalRubricResultResponse
* @description A rubric job's grades so far. Error attempts are excluded, like the job's preference results.
*/
ShadowEvalRubricResultResponse: {
/**
* By Router
* @description One slice per router arm, ordered by router name, each graded against the same real responses
*/
by_router: components["schemas"]["ShadowEvalRubricSlice"][];
/** Job Id */
job_id: string;
/** @description Every arm together. A judged turn is one (request, router arm) pair, so a multi-router job counts each real response once per arm; compare routers with by_router */
overall: components["schemas"]["ShadowEvalRubricSlice"];
/** Rubric */
rubric: string;
/** Rubric Version */
rubric_version: number;
};
/**
* ShadowEvalRubricSlice
* @description Rubric grades for one router arm, or for every arm together.
*/
ShadowEvalRubricSlice: {
/**
* Grade Distribution
* @description Joint counts of (real_grade, shadow_grade) over graded turns, ordered by real then shadow grade, so turns where both responses were good or both were bad stay distinguishable
*/
grade_distribution: components["schemas"]["ShadowEvalGradePair"][];
/**
* Graded Count
* @description Judged turns that carry both grades
*/
graded_count: number;
/** Group */
group: string;
/**
* Ungraded Count
* @description Judged turns with a preference but no grades, such as turns a worker without rubric support judged. They are excluded from grade_distribution, never counted as low grades
*/
ungraded_count: number;
};
/**
* ShadowEvalSlice
* @description Judge outcomes for one slice of a job's verdicts: a router tier, one of the
@ -49975,6 +50205,93 @@ export interface components {
*/
user_ids: string[];
};
/**
* StartShadowEvalRubricRequest
* @description A shadow eval whose judge also grades each response from 1 to 5 against an operator rubric.
*/
StartShadowEvalRubricRequest: {
/**
* Api Key Ids
* @description Hashed virtual keys whose traffic will be shadowed. Combined with team_ids and user_ids the job needs at least one target and at most 100, which also bounds every read the job's endpoints make. Each target carries its own max_budget spend budget, so one exhausting its budget leaves the others sampling.
* @default []
*/
api_key_ids: string[];
/**
* Baseline Model
* @description Required when direction is reverse and rejected otherwise: the fixed model the router's own responses are judged against. Must be a plain model rather than another auto-router
*/
baseline_model?: string | null;
/**
* Direction
* @description forward answers 'should this key adopt router_name': it samples the requests the key did NOT route through the router and duplicates them through it. reverse answers 'is the router still worth it for a key already on it': it samples the requests the router did serve and duplicates them against baseline_model. The response the caller received is always the real arm
* @default forward
* @enum {string}
*/
direction: "forward" | "reverse";
/**
* Duration Days
* @description How many days the job samples traffic before completing on its own
* @default 7
*/
duration_days: number;
/**
* Judge Model
* @description Model used to blindly judge real vs. shadow responses. The judge only compares two answers, so a mid-tier model (Claude Sonnet or GPT-4o class) is the sweet spot: small/nano-class models produce unreliable or malformed verdicts, while frontier reasoning models add cost without changing outcomes.
* @default anthropic/claude-sonnet-5
*/
judge_model: string;
/**
* Max Budget
* @description Per-target USD budget for the eval's own overhead, the shadow-arm and judge calls, priced with the same figures the spend pipeline bills. EACH scoped target samples until its recorded eval spend reaches this, so a job over N targets spends at most about N times max_budget; in-flight samples can overshoot the cap by one sampling cache window. Every router arm draws from the same per-target budget, so a multi-router job reaches it proportionally sooner
* @default 10
*/
max_budget: number;
/**
* Models
* @description Model groups to narrow the sampled traffic to, matched on the group the caller requested and resolved through model_group_alias, so an alias and its target are one name. Empty samples every model the targets use. This ANDs with the targets: a job over a user and one model samples that user's requests on that model across every key they own, and none of their other traffic. Forward jobs only: a reverse job samples exactly the traffic its own router served, which no other model group can name
* @default []
*/
models: string[];
/**
* Router Name
* @description The auto-router under evaluation, in either direction: the single-router spelling of router_names. Provide exactly one of the two fields
*/
router_name?: string | null;
/**
* Router Names
* @description The auto-routers under evaluation, at most 4. Every sampled request runs through every router listed and each arm is judged independently against the same real response, so routers compare head-to-head on identical traffic. More than one router requires direction 'forward'. After validation this field always carries the full deduplicated set, whichever spelling the caller used
* @default []
*/
router_names: string[];
/**
* Rubric
* @description The criteria the judge grades both responses against, in plain language, at most 4000 characters. Each response gets an integer grade on a fixed scale where 1 fails the criteria and 5 fully meets them. Stored unchanged with the job and never edited
*/
rubric: string;
/**
* Shadow Percentage
* @description Percentage of each target's requests to duplicate through the router
*/
shadow_percentage: number;
/**
* Skip Same Model
* @description Reverse jobs only. When true, a sampled request whose response came from baseline_model's own model group (aliases resolved) is skipped instead of judged, so no shadow or judge call is made or billed for it and it never counts toward win, tie, or error rates. Skips are reported as same_model_skipped_count. Different model group names stay distinct even when they share a provider model
* @default false
*/
skip_same_model: boolean;
/**
* Team Ids
* @description Teams whose traffic will be shadowed, matched on the team every authenticated request resolves to, so a team's JWT-auth and virtual-key traffic are both sampled
* @default []
*/
team_ids: string[];
/**
* User Ids
* @description Users whose traffic will be shadowed, matched on the user every authenticated request resolves to across all their teams: JWT requests carrying their subject claim and virtual keys they own
* @default []
*/
user_ids: string[];
};
/** StoredReport */
StoredReport: {
/** Id */
@ -57838,6 +58155,39 @@ export interface operations {
};
};
};
start_shadow_eval_rubric_auto_router_shadow_eval_rubric_start_post: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody: {
content: {
"application/json": components["schemas"]["StartShadowEvalRubricRequest"];
};
};
responses: {
/** @description Successful Response */
201: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["ShadowEvalRubricJobResponse"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
start_shadow_eval_auto_router_shadow_eval_start_post: {
parameters: {
query?: never;
@ -57902,6 +58252,37 @@ export interface operations {
};
};
};
get_shadow_eval_rubric_results_auto_router_shadow_eval__job_id__rubric_results_get: {
parameters: {
query?: never;
header?: never;
path: {
job_id: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["ShadowEvalRubricResultResponse"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
stop_shadow_eval_job_auto_router_shadow_eval__job_id__stop_post: {
parameters: {
query?: never;

View file

@ -30,10 +30,10 @@ litellm/proxy/management_endpoints/auto_router_endpoints.py _with_target_labels
litellm/proxy/management_endpoints/auto_router_endpoints.py _with_target_labels prisma token.in `list(tokens)` 0
litellm/proxy/management_endpoints/auto_router_endpoints.py _with_target_labels prisma user_id.in `list(user_ids)` 0
litellm/proxy/management_endpoints/auto_router_endpoints.py get_shadow_eval_job prisma job_id.in `leg_ids` 0
litellm/proxy/management_endpoints/auto_router_endpoints.py start_shadow_eval prisma target_id.in `list(ids)` 0
litellm/proxy/management_endpoints/auto_router_endpoints.py start_shadow_eval prisma team_id.in `list(data.team_ids)` 0
litellm/proxy/management_endpoints/auto_router_endpoints.py start_shadow_eval prisma token.in `list(data.api_key_ids)` 0
litellm/proxy/management_endpoints/auto_router_endpoints.py start_shadow_eval prisma user_id.in `list(data.user_ids)` 0
litellm/proxy/management_endpoints/auto_router_endpoints.py _create_shadow_eval_job prisma target_id.in `list(ids)` 0
litellm/proxy/management_endpoints/auto_router_endpoints.py _create_shadow_eval_job prisma team_id.in `list(data.team_ids)` 0
litellm/proxy/management_endpoints/auto_router_endpoints.py _create_shadow_eval_job prisma token.in `list(data.api_key_ids)` 0
litellm/proxy/management_endpoints/auto_router_endpoints.py _create_shadow_eval_job prisma user_id.in `list(data.user_ids)` 0
litellm/proxy/management_endpoints/budget_management_endpoints.py info_budget prisma budget_id.in `data.budgets` 0
litellm/proxy/management_endpoints/common_utils.py _team_admin_can_invite_user prisma team_id.in `admin_user_obj.teams` 0
litellm/proxy/management_endpoints/common_utils.py user_has_admin_privileges prisma team_id.in `user_obj.teams` 0

View file

@ -2,6 +2,7 @@
the detached pipeline's single attempt-row write, and the cache-first job lookup."""
import asyncio
import json
from collections.abc import AsyncIterator, Coroutine, Mapping
from contextlib import asynccontextmanager
from contextvars import Context
@ -20,6 +21,9 @@ from litellm.integrations.shadow_eval_logger import (
_MAX_JUDGE_PROMPT_CHARS,
JUDGE_MAX_OUTPUT_TOKENS,
PAIRWISE_JUDGE_RESPONSE_FORMAT,
PAIRWISE_JUDGE_SYSTEM_PROMPT,
RUBRIC_JUDGE_RESPONSE_FORMAT,
SHADOW_EVAL_RUBRIC_ENABLED_ENV_VAR,
ActiveShadowEvalJob,
GuardrailRequestSnapshot,
ShadowEvalLogger,
@ -27,9 +31,12 @@ from litellm.integrations.shadow_eval_logger import (
_judge_user_prompt,
_sample_hits,
_unmask_preference,
is_shadow_eval_rubric_enabled,
request_guardrail_fingerprint,
rubric_judge_system_prompt,
)
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.management_endpoints.auto_router_endpoints import SHADOW_EVAL_RUBRIC_VERSION
from litellm.types.utils import (
SHADOW_EVAL_JUDGE_CALL_ORIGIN,
SHADOW_EVAL_ROUTER_CALL_ORIGIN,
@ -79,7 +86,14 @@ def _prisma(jobs=(), attempt_counts=(), attempt_costs=()) -> MagicMock:
return prisma
def _job_record(job: ActiveShadowEvalJob, target_type="key", target_id="key-hash") -> MagicMock:
def _job_record(
job: ActiveShadowEvalJob,
target_type="key",
target_id="key-hash",
rubric: str | None = None,
rubric_version: int | None = None,
) -> MagicMock:
"""rubric and rubric_version, when given, override the job's, to pose rows the job model rejects."""
record = MagicMock()
for field, value in dict(
id=job.id,
@ -96,6 +110,8 @@ def _job_record(job: ActiveShadowEvalJob, target_type="key", target_id="key-hash
max_turns=job.max_turns,
max_budget=job.max_budget,
ends_at=job.ends_at,
rubric=rubric or job.rubric,
rubric_version=rubric_version or job.rubric_version,
).items():
setattr(record, field, value)
return record
@ -240,7 +256,15 @@ def _spend_counter(store=None):
return counter, read, write
def _logger(router=None, prisma=None, jobs=(), counter_store=None, jobs_by_target=None) -> ShadowEvalLogger:
def _logger(
router=None,
prisma=None,
jobs=(),
counter_store=None,
jobs_by_target=None,
real_is_a: bool | None = None,
rubric_enabled: bool = True,
) -> ShadowEvalLogger:
cache = InMemoryCache(max_size_in_memory=4, default_ttl=60)
counter, read, write = _spend_counter(counter_store)
funnel_events = []
@ -251,6 +275,8 @@ def _logger(router=None, prisma=None, jobs=(), counter_store=None, jobs_by_targe
job_spend_reader=read,
job_spend_writer=write,
funnel_recorder=lambda job_id, stage: funnel_events.append((job_id, stage)),
real_is_a_draw=(lambda: real_is_a) if real_is_a is not None else None,
rubric_enabled=lambda: rubric_enabled,
)
logger._test_counter = counter
logger._test_funnel = funnel_events
@ -2789,3 +2815,308 @@ class TestSameModelSkip:
router.acompletion.assert_not_called()
assert logger._test_funnel == [("job-1", "same_model")]
RUBRIC: Final = "Cites the source document and answers in under 100 words."
def _rubric_job(
id: str = "job-1",
router_names: tuple[str, ...] = (),
direction: Literal["forward", "reverse"] = "forward",
baseline_model: str | None = None,
) -> ActiveShadowEvalJob:
return _job(
id=id,
router_names=router_names,
direction=direction,
baseline_model=baseline_model,
rubric=RUBRIC,
rubric_version=SHADOW_EVAL_RUBRIC_VERSION,
)
def _rubric_reply(grade_a: object, grade_b: object, preference: str = "A") -> str:
return json.dumps({"preference": preference, "confidence": 0.8, "grade_a": grade_a, "grade_b": grade_b})
def _judge_calls(router: MagicMock) -> list[Mapping[str, object]]:
return [
call.kwargs
for call in router.acompletion.call_args_list
if call.kwargs["metadata"].get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == SHADOW_EVAL_JUDGE_CALL_ORIGIN
]
def _attempt_rows(prisma: MagicMock) -> list[Mapping[str, object]]:
return [call.kwargs["data"] for call in prisma.db.litellm_shadowevalattempt.create.await_args_list]
async def _sample_once(logger: ShadowEvalLogger, job: ActiveShadowEvalJob) -> None:
await logger._run_shadow_eval(
job=job,
request_id="req-1",
messages=({"role": "user", "content": "hi"},),
real_text="real answer",
real_model="claude-opus",
real_cost=0.0,
real_classifier_cost=0.0,
real_cache_hit=False,
control_tier=None,
shadow_params={},
parent_metadata={},
)
@pytest.mark.asyncio
class TestRubricGrades:
async def test_a_preference_job_sends_the_legacy_judge_request_and_writes_the_legacy_row(
self, monkeypatch: pytest.MonkeyPatch
):
import litellm as litellm_module
monkeypatch.setattr(litellm_module, "completion_cost", lambda completion_response: 0.005)
prisma = _prisma()
router = _router(judge_json='{"preference": "A", "confidence": 1.7, "grade_a": 5, "grade_b": 1}')
await _sample_once(_logger(router=router, prisma=prisma, real_is_a=False), _job())
(judge_call,) = _judge_calls(router)
assert judge_call["messages"][0] == {"role": "system", "content": PAIRWISE_JUDGE_SYSTEM_PROMPT}
assert judge_call["response_format"] == PAIRWISE_JUDGE_RESPONSE_FORMAT
assert "Response A:\nshadow answer\n\nResponse B:\nreal answer" in judge_call["messages"][1]["content"]
assert _attempt_rows(prisma) == [
{
"job_id": "job-1",
"request_id": "req-1",
"router_name": "my-router",
"outcome": "shadow",
"tier": "SIMPLE",
"real_model": "claude-opus",
"shadow_model": "cheap-model",
"confidence": 1.0,
"judge_cost": 0.005,
"shadow_cost": 0.005,
"shadow_classifier_cost": 0.0,
"real_cost": 0.0,
"real_classifier_cost": 0.0,
"real_cache_hit": False,
"error": None,
}
]
@pytest.mark.parametrize("direction", ["forward", "reverse"])
@pytest.mark.parametrize("real_is_a", [True, False], ids=["real-is-a", "real-is-b"])
async def test_grades_follow_each_response_through_either_label_order(self, direction: str, real_is_a: bool):
"""The same draw places the responses and unmasks the grades, and real is always the
response the caller received, including a reverse job's router response."""
prisma = _prisma()
router = _router(judge_json=_rubric_reply(5, 2))
job = (
_rubric_job()
if direction == "forward"
else _rubric_job(direction="reverse", baseline_model="baseline-model")
)
await _sample_once(_logger(router=router, prisma=prisma, real_is_a=real_is_a), job)
(judge_call,) = _judge_calls(router)
first, second = ("real answer", "shadow answer") if real_is_a else ("shadow answer", "real answer")
assert f"Response A:\n{first}\n\nResponse B:\n{second}" in judge_call["messages"][1]["content"]
assert judge_call["messages"][0] == {"role": "system", "content": rubric_judge_system_prompt(RUBRIC)}
assert RUBRIC in judge_call["messages"][0]["content"]
assert judge_call["response_format"] == RUBRIC_JUDGE_RESPONSE_FORMAT
(row,) = _attempt_rows(prisma)
expected = ("real", 5, 2) if real_is_a else ("shadow", 2, 5)
assert (row["outcome"], row["real_grade"], row["shadow_grade"]) == expected
assert (row["confidence"], row["error"]) == (0.8, None)
@pytest.mark.parametrize(
("grade_a", "grade_b", "preference", "expected"),
[
(1, 5, "B", ("shadow", 1, 5)),
(5, 1, "A", ("real", 5, 1)),
(1, 1, "tie", ("tie", 1, 1)),
(5, 5, "tie", ("tie", 5, 5)),
(3, 3, "B", ("shadow", 3, 3)),
],
ids=["lowest-vs-highest", "highest-vs-lowest", "both-bad", "both-good", "equal-grades-still-a-winner"],
)
async def test_every_valid_grade_pair_is_recorded_with_an_independent_preference(
self, grade_a: int, grade_b: int, preference: str, expected: tuple[str, int, int]
):
prisma = _prisma()
router = _router(judge_json=_rubric_reply(grade_a, grade_b, preference))
await _sample_once(_logger(router=router, prisma=prisma, real_is_a=True), _rubric_job())
(row,) = _attempt_rows(prisma)
assert (row["outcome"], row["real_grade"], row["shadow_grade"]) == expected
@pytest.mark.parametrize(
("reply", "invalid"),
[
(_rubric_reply(True, 2), "grade_a:value_error"),
(_rubric_reply("4", 2), "grade_a:value_error"),
(_rubric_reply(4.0, 2), "grade_a:value_error"),
(_rubric_reply(4.5, 2), "grade_a:value_error"),
(_rubric_reply(None, 2), "grade_a:value_error"),
(_rubric_reply(0, 2), "grade_a:literal_error"),
(_rubric_reply(6, 2), "grade_a:literal_error"),
(_rubric_reply(4, -1), "grade_b:literal_error"),
('{"preference": "A", "confidence": 0.8, "grade_a": 4}', "grade_b:missing"),
('{"preference": "A", "confidence": 0.8}', "grade_a:missing,grade_b:missing"),
(_rubric_reply(4, 2, preference="C"), "preference:literal_error"),
("I would give A a 4 and B a 2", "verdict:JSONDecodeError"),
],
ids=[
"bool",
"string",
"integral-float",
"fractional-float",
"null",
"below-range",
"above-range",
"negative",
"one-missing",
"both-missing",
"bad-preference",
"not-json",
],
)
async def test_a_malformed_verdict_is_an_error_row_that_keeps_the_billed_costs(
self, reply: str, invalid: str, monkeypatch: pytest.MonkeyPatch
):
import litellm as litellm_module
monkeypatch.setattr(litellm_module, "completion_cost", lambda completion_response: 0.007)
prisma = _prisma()
logger = _logger(router=_router(judge_json=reply), prisma=prisma, real_is_a=True)
await _sample_once(logger, _rubric_job())
(row,) = _attempt_rows(prisma)
assert row["outcome"] == "error"
assert row["error"].startswith(f"unparseable rubric verdict; invalid={invalid}, finish_reason=")
assert RUBRIC not in row["error"] and reply not in row["error"]
assert row["confidence"] is None
assert "real_grade" not in row and "shadow_grade" not in row
assert (row["judge_cost"], row["shadow_cost"]) == (0.007, 0.007)
assert sum(logger._test_counter.values()) == pytest.approx(0.014)
async def test_each_router_arm_records_the_grades_its_own_response_earned(self):
prisma = _prisma()
router = _router(sibling_router_texts={"alt-router": "alt answer"})
arms = router.acompletion.side_effect
async def acompletion(**kwargs):
if kwargs["metadata"].get(INTERNAL_CALL_ORIGIN_METADATA_KEY) != SHADOW_EVAL_JUDGE_CALL_ORIGIN:
return await arms(**kwargs)
shadow_grade = 2 if "alt answer" in kwargs["messages"][1]["content"] else 4
return {"choices": [{"message": {"content": _rubric_reply(3, shadow_grade, preference="tie")}}]}
router.acompletion = MagicMock(side_effect=acompletion)
await _sample_once(
_logger(router=router, prisma=prisma, real_is_a=True),
_rubric_job(router_names=("my-router", "alt-router")),
)
assert [(row["router_name"], row["real_grade"], row["shadow_grade"]) for row in _attempt_rows(prisma)] == [
("my-router", 3, 4),
("alt-router", 3, 2),
]
@pytest.mark.parametrize("rubric", [True, False], ids=["rubric", "preference"])
async def test_a_failed_judge_call_records_only_what_the_job_may_persist(
self, rubric: bool, monkeypatch: pytest.MonkeyPatch
):
"""A provider error can quote the request it rejected. A rubric row keeps only the
exception type and status; a preference row keeps the message exactly as before."""
import litellm as litellm_module
from litellm.exceptions import BadRequestError
monkeypatch.setattr(litellm_module, "completion_cost", lambda completion_response: 0.007)
provider_error: Final = BadRequestError(
message=f"rejected prompt containing {RUBRIC} and real answer",
model="judge-model",
llm_provider="openai",
)
router: Final = _router()
arms: Final = router.acompletion.side_effect
async def acompletion(**kwargs: object) -> object:
metadata: Final = kwargs["metadata"]
if isinstance(metadata, Mapping) and metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == (
SHADOW_EVAL_JUDGE_CALL_ORIGIN
):
raise provider_error
return await arms(**kwargs)
router.acompletion = MagicMock(side_effect=acompletion)
prisma: Final = _prisma()
await _sample_once(_logger(router=router, prisma=prisma, real_is_a=True), _rubric_job() if rubric else _job())
(row,) = _attempt_rows(prisma)
expected_error: Final = (
"rubric judge call failed; type=BadRequestError, status=400"
if rubric
else f"judge call failed: {provider_error}"[:_MAX_ERROR_CHARS]
)
assert (row["outcome"], row["error"], row["confidence"]) == ("error", expected_error, None)
assert (row["judge_cost"], row["shadow_cost"]) == (0.0, 0.007)
assert "real_grade" not in row and "shadow_grade" not in row
@pytest.mark.parametrize(("enabled", "sampled"), [(True, ["job-plain", "job-rubric"]), (False, ["job-plain"])])
async def test_a_rubric_job_samples_only_while_rubric_grading_is_enabled(self, enabled: bool, sampled: list[str]):
prisma = _prisma()
logger = _logger(
router=_router(judge_json=_rubric_reply(4, 4)),
prisma=prisma,
jobs=(_job(id="job-plain"), _rubric_job(id="job-rubric")),
real_is_a=True,
rubric_enabled=enabled,
)
await _hook(logger, _success_kwargs())
assert sorted(row["job_id"] for row in _attempt_rows(prisma)) == sampled
assert logger._test_funnel == []
async def test_a_rubric_row_this_worker_cannot_grade_is_dropped_not_run_as_preference_only(self):
records = [
_job_record(_job(id="job-plain")),
_job_record(_rubric_job(id="job-rubric")),
_job_record(_job(id="job-newer-contract"), rubric=RUBRIC, rubric_version=SHADOW_EVAL_RUBRIC_VERSION + 1),
_job_record(_job(id="job-no-version"), rubric=RUBRIC),
_job_record(_job(id="job-no-rubric"), rubric_version=SHADOW_EVAL_RUBRIC_VERSION),
]
logger = ShadowEvalLogger(
router_provider=lambda: None,
prisma_provider=lambda: _prisma(jobs=records),
jobs_cache=InMemoryCache(max_size_in_memory=4, default_ttl=60),
)
jobs = (await logger._active_jobs())[("key", "key-hash")]
assert {job.id: job.rubric for job in jobs} == {"job-plain": None, "job-rubric": RUBRIC}
def test_the_rubric_response_format_demands_both_grades_on_the_fixed_scale():
schema = RUBRIC_JUDGE_RESPONSE_FORMAT["json_schema"]["schema"]
assert schema["required"] == ["preference", "confidence", "grade_a", "grade_b"]
assert schema["properties"]["grade_a"]["enum"] == [1, 2, 3, 4, 5]
assert schema["properties"]["grade_b"]["enum"] == [1, 2, 3, 4, 5]
@pytest.mark.parametrize(("value", "enabled"), [(None, False), ("false", False), ("yes", False), ("true", True)])
def test_rubric_grading_is_off_unless_the_env_var_turns_it_on(
value: str | None, enabled: bool, monkeypatch: pytest.MonkeyPatch
):
if value is None:
monkeypatch.delenv(SHADOW_EVAL_RUBRIC_ENABLED_ENV_VAR, raising=False)
else:
monkeypatch.setenv(SHADOW_EVAL_RUBRIC_ENABLED_ENV_VAR, value)
assert is_shadow_eval_rubric_enabled() is enabled

View file

@ -1256,13 +1256,22 @@ class TestAutoRouterBenchmarks:
from datetime import datetime, timedelta, timezone
from litellm.integrations.shadow_eval_logger import SHADOW_EVAL_RUBRIC_ENABLED_ENV_VAR
from litellm.proxy.management_endpoints.auto_router_endpoints import (
get_shadow_eval_job,
get_shadow_eval_rubric_results,
list_shadow_eval_jobs,
start_shadow_eval,
start_shadow_eval_rubric,
stop_shadow_eval_job,
)
from litellm.types.management_endpoints.auto_router_endpoints import SHADOW_EVAL_TURN_VALVE, StartShadowEvalRequest
from litellm.types.management_endpoints.auto_router_endpoints import (
SHADOW_EVAL_RUBRIC_MAX_CHARS,
SHADOW_EVAL_RUBRIC_VERSION,
SHADOW_EVAL_TURN_VALVE,
StartShadowEvalRequest,
StartShadowEvalRubricRequest,
)
VIEWER = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, api_key="sk-view", user_id="viewer")
@ -1598,6 +1607,7 @@ def _shadow_prisma(
key_teams=None,
known_teams=None,
known_users=None,
grade_rows=None,
) -> MagicMock:
"""The job-table fake honours the filters it is handed, so a read that forgets
stopped_at sees rows the partial index would have released, one that forgets
@ -1730,6 +1740,8 @@ def _shadow_prisma(
return [{"judged_count": 10, "error_count": 2, "judge_spend": 0.031}]
if "SELECT job_id AS grp" in sql:
return by_leg_rows if by_leg_rows is not None else []
if "a.real_grade" in sql:
return grade_rows if grade_rows is not None else []
if "COALESCE(a.router_name" in sql:
return by_router_rows if by_router_rows is not None else []
if 'FROM "LiteLLM_ShadowEvalFunnel"' in sql:
@ -4137,3 +4149,228 @@ class TestPerSessionAverages:
row: Final = TestAutoRouterBenchmarks.ROW.model_copy(update={"sessions": sessions, "turns": turns})
totals: Final = _benchmark_totals(row)
assert (totals.avg_turns_per_session, totals.avg_session_seconds, totals.avg_tokens_per_session) == expected
RUBRIC: Final = "Cites the source document and answers in under 100 words."
def _rubric_start_request(
api_key_ids: tuple[str, ...] = ("key-hash",), rubric: str = RUBRIC
) -> StartShadowEvalRubricRequest:
return StartShadowEvalRubricRequest.model_validate(
{**_start_request().model_dump(exclude={"router_names"}), "api_key_ids": api_key_ids, "rubric": rubric}
)
def _rubric_leg(id: str = "leg-1", target_id: str = "key-hash", router_names: tuple[str, ...] = ()) -> MagicMock:
return _leg_record(
id=id,
target_id=target_id,
router_names=router_names,
rubric=RUBRIC,
rubric_version=SHADOW_EVAL_RUBRIC_VERSION,
)
def _grade_row(grp: str, real_grade: int | None, shadow_grade: int | None, turn_count: int) -> dict[str, object]:
return {"grp": grp, "real_grade": real_grade, "shadow_grade": shadow_grade, "turn_count": turn_count}
class TestShadowEvalRubricStart:
@pytest.mark.asyncio
@pytest.mark.parametrize("flag", [None, "false"], ids=["unset", "false"])
async def test_is_refused_until_rubric_grading_is_enabled(
self, monkeypatch: pytest.MonkeyPatch, flag: str | None
) -> None:
if flag is None:
monkeypatch.delenv(SHADOW_EVAL_RUBRIC_ENABLED_ENV_VAR, raising=False)
else:
monkeypatch.setenv(SHADOW_EVAL_RUBRIC_ENABLED_ENV_VAR, flag)
_configure_anthropic_sdk_judge(monkeypatch)
prisma = _shadow_prisma()
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
with pytest.raises(HTTPException) as exc:
await start_shadow_eval_rubric(_rubric_start_request(), ADMIN)
assert exc.value.status_code == 400
assert SHADOW_EVAL_RUBRIC_ENABLED_ENV_VAR in str(exc.value.detail)
prisma.db.litellm_shadowevaljob.create_many.assert_not_called()
@pytest.mark.asyncio
@pytest.mark.parametrize("caller", [VIEWER, NON_ADMIN], ids=["viewer", "internal-user"])
async def test_only_a_proxy_admin_can_start_one(self, monkeypatch: pytest.MonkeyPatch, caller) -> None:
monkeypatch.setenv(SHADOW_EVAL_RUBRIC_ENABLED_ENV_VAR, "true")
_configure_anthropic_sdk_judge(monkeypatch)
prisma = _shadow_prisma()
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
with pytest.raises(HTTPException) as exc:
await start_shadow_eval_rubric(_rubric_start_request(), caller)
assert exc.value.status_code == 403
prisma.db.litellm_shadowevaljob.create_many.assert_not_called()
@pytest.mark.asyncio
async def test_stores_the_rubric_and_its_grading_version_on_every_leg(self, monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv(SHADOW_EVAL_RUBRIC_ENABLED_ENV_VAR, "true")
_configure_anthropic_sdk_judge(monkeypatch)
prisma = _shadow_prisma()
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
response = await start_shadow_eval_rubric(_rubric_start_request(api_key_ids=("key-hash", "key-hash-2")), ADMIN)
rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"]
assert [(row["target_id"], row["rubric"], row["rubric_version"]) for row in rows] == [
("key-hash", RUBRIC, SHADOW_EVAL_RUBRIC_VERSION),
("key-hash-2", RUBRIC, SHADOW_EVAL_RUBRIC_VERSION),
]
assert (response.job_id, response.rubric, response.rubric_version) == (
rows[0]["group_id"],
RUBRIC,
SHADOW_EVAL_RUBRIC_VERSION,
)
assert response.status == "running"
assert [target.target_id for target in response.targets] == ["key-hash", "key-hash-2"]
@pytest.mark.asyncio
async def test_the_preference_start_writes_and_returns_no_rubric_fields(self, monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv(SHADOW_EVAL_RUBRIC_ENABLED_ENV_VAR, "true")
_configure_anthropic_sdk_judge(monkeypatch)
prisma = _shadow_prisma()
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
response = await start_shadow_eval(_start_request(), ADMIN)
(row,) = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"]
assert "rubric" not in row and "rubric_version" not in row
assert "rubric" not in response.model_dump() and "rubric_version" not in response.model_dump()
@pytest.mark.parametrize(
"rubric",
[None, "", " \n\t", "x" * (SHADOW_EVAL_RUBRIC_MAX_CHARS + 1)],
ids=["missing", "empty", "whitespace", "too-long"],
)
def test_a_rubric_request_needs_bounded_criteria(rubric: str | None) -> None:
payload = {**_start_request().model_dump(exclude={"router_names"}), "rubric": rubric}
if rubric is None:
payload.pop("rubric")
with pytest.raises(ValidationError):
StartShadowEvalRubricRequest.model_validate(payload)
def test_a_rubric_at_the_length_bound_is_kept_verbatim() -> None:
rubric = " " + "x" * (SHADOW_EVAL_RUBRIC_MAX_CHARS - 2) + "\n"
assert _rubric_start_request(rubric=rubric).rubric == rubric
class TestShadowEvalRubricResults:
@pytest.mark.asyncio
async def test_reports_joint_grades_per_router_and_counts_ungraded_turns_apart(
self, monkeypatch: pytest.MonkeyPatch
):
prisma = _shadow_prisma(
legs=[
_rubric_leg(router_names=("my-router", "alt-router", "idle-router")),
_rubric_leg(
id="leg-2", target_id="key-hash-2", router_names=("my-router", "alt-router", "idle-router")
),
],
grade_rows=[
_grade_row("my-router", 5, 5, 3),
_grade_row("my-router", 1, 1, 2),
_grade_row("my-router", 5, 1, 1),
_grade_row("my-router", None, None, 4),
_grade_row("alt-router", 4, 2, 2),
_grade_row("alt-router", None, None, 1),
],
)
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
response = await get_shadow_eval_rubric_results("job-1", VIEWER)
assert response.model_dump() == {
"job_id": "job-1",
"rubric": RUBRIC,
"rubric_version": SHADOW_EVAL_RUBRIC_VERSION,
"overall": {
"group": "all",
"graded_count": 8,
"ungraded_count": 5,
"grade_distribution": (
{"real_grade": 1, "shadow_grade": 1, "turn_count": 2},
{"real_grade": 4, "shadow_grade": 2, "turn_count": 2},
{"real_grade": 5, "shadow_grade": 1, "turn_count": 1},
{"real_grade": 5, "shadow_grade": 5, "turn_count": 3},
),
},
"by_router": (
{
"group": "alt-router",
"graded_count": 2,
"ungraded_count": 1,
"grade_distribution": ({"real_grade": 4, "shadow_grade": 2, "turn_count": 2},),
},
{"group": "idle-router", "graded_count": 0, "ungraded_count": 0, "grade_distribution": ()},
{
"group": "my-router",
"graded_count": 6,
"ungraded_count": 4,
"grade_distribution": (
{"real_grade": 1, "shadow_grade": 1, "turn_count": 2},
{"real_grade": 5, "shadow_grade": 1, "turn_count": 1},
{"real_grade": 5, "shadow_grade": 5, "turn_count": 3},
),
},
),
}
grade_query = next(call.args for call in prisma.db.query_raw.await_args_list if "a.real_grade" in call.args[0])
assert grade_query[1] == ["leg-1", "leg-2"]
@pytest.mark.asyncio
async def test_a_job_started_without_a_rubric_has_no_rubric_results(self, monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(proxy_server, "prisma_client", _shadow_prisma(legs=[_leg_record()]))
for job_id in ("job-1", "nope"):
with pytest.raises(HTTPException) as missing:
await get_shadow_eval_rubric_results(job_id, VIEWER)
assert missing.value.status_code == 404
@pytest.mark.asyncio
async def test_only_proxy_admin_roles_can_read_rubric_results(self, monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(proxy_server, "prisma_client", _shadow_prisma(legs=[_rubric_leg()]))
with pytest.raises(HTTPException) as forbidden:
await get_shadow_eval_rubric_results("job-1", NON_ADMIN)
assert forbidden.value.status_code == 403
assert (await get_shadow_eval_rubric_results("job-1", ADMIN)).rubric == RUBRIC
@pytest.mark.asyncio
@pytest.mark.parametrize("caller", [ADMIN, VIEWER], ids=["admin", "viewer"])
async def test_rubric_results_without_a_database_report_it_is_not_connected(
self, monkeypatch: pytest.MonkeyPatch, caller: UserAPIKeyAuth
):
from litellm.proxy._types import CommonProxyErrors
monkeypatch.setattr(proxy_server, "prisma_client", None)
with pytest.raises(HTTPException) as unavailable:
await get_shadow_eval_rubric_results("job-1", caller)
assert (unavailable.value.status_code, unavailable.value.detail) == (
500,
CommonProxyErrors.db_not_connected_error.value,
)
@pytest.mark.asyncio
async def test_the_regular_job_detail_of_a_rubric_job_keeps_its_legacy_shape(self, monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(proxy_server, "prisma_client", _shadow_prisma(legs=[_rubric_leg()]))
detail = (await get_shadow_eval_job("job-1", VIEWER)).model_dump()
assert "rubric" not in detail and "rubric_version" not in detail