From 45115cbfa28ed7c68724094b9d70fd236290d55d Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Tue, 17 Nov 2026 12:04:31 -0800 Subject: [PATCH] feat(shadow-eval): add optional rubric grades (#52321) * feat(shadow-eval): add optional rubric grades * chore(shadow-eval): generate rubric API types * test(shadow-eval): cover unavailable rubric result storage --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- litellm/integrations/shadow_eval_logger.py | 261 +++++++++++- .../migration.sql | 13 + .../litellm_proxy_extras/schema.prisma | 4 + .../auto_router_endpoints.py | 157 +++++++- litellm/proxy/schema.prisma | 4 + litellm/schema.prisma | 4 + .../auto_router_endpoints.py | 88 +++- .../src/lib/http/schema.d.ts | 381 ++++++++++++++++++ .../unbounded_in_baseline.txt | 8 +- .../integrations/test_shadow_eval_logger.py | 335 ++++++++++++++- .../test_auto_router_endpoints.py | 239 ++++++++++- 11 files changed, 1467 insertions(+), 27 deletions(-) create mode 100644 litellm/proxy-extras/litellm_proxy_extras/migrations/20261117000000_shadow_eval_rubric_grades/migration.sql diff --git a/litellm/integrations/shadow_eval_logger.py b/litellm/integrations/shadow_eval_logger.py index f54770c699e..d0d75e927c9 100644 --- a/litellm/integrations/shadow_eval_logger.py +++ b/litellm/integrations/shadow_eval_logger.py @@ -12,6 +12,7 @@ across pods or stop races; the hook reads active jobs through a short-TTL cache. import asyncio import hashlib import json +import os import random import traceback from collections.abc import Awaitable, Callable, Mapping, Sequence @@ -44,9 +45,10 @@ from litellm.llms.base_llm.base_utils import type_to_response_format_param from litellm.redis.stores.spend_counters import ( shadow_eval_counter, ) +from litellm.secret_managers.main import str_to_bool from litellm.types.llms.base import LiteLLMBaseModel -from litellm.types.management_endpoints.auto_router_endpoints import ShadowEvalDirection -from litellm.types.utils import SHADOW_EVAL_JUDGE_CALL_ORIGIN, SHADOW_EVAL_ROUTER_CALL_ORIGIN +from litellm.types.management_endpoints.auto_router_endpoints import SHADOW_EVAL_RUBRIC_VERSION, ShadowEvalDirection +from litellm.types.utils import SHADOW_EVAL_JUDGE_CALL_ORIGIN, SHADOW_EVAL_ROUTER_CALL_ORIGIN, ModelResponse if TYPE_CHECKING: from litellm.proxy.db.shadow_eval_funnel import ShadowEvalFunnelStage @@ -485,6 +487,76 @@ class PairwiseVerdict(LiteLLMBaseModel): PAIRWISE_JUDGE_RESPONSE_FORMAT: Final = type_to_response_format_param(PairwiseVerdict) +_RUBRIC_JUDGE_SYSTEM_PROMPT_TEMPLATE: Final = """You are an impartial quality judge grading two responses to the same conversation +against an operator's rubric. + +The responses are labeled A and B in random order. You do not know which system produced which. + +A response may be prose, or a tool call shown as `[tool call] name(arguments)` if the +assistant chose to act instead of answering directly. A tool call is not a defect: judge +whether calling that tool was the right response to the conversation, the same as you +would judge prose. + +Rubric: + +{rubric} + + +Grade each response on its own against the rubric with an integer from 1 to 5: +1 = fails the rubric's criteria +2 = meets few of the criteria +3 = meets about half of the criteria +4 = meets most of the criteria +5 = fully meets the criteria + +Then say which response is better overall. Equal grades may still have a better response. + +Return ONLY valid JSON in this exact format, no other text: +{{ + "preference": "A" | "B" | "tie", + "confidence": <0.0 to 1.0>, + "grade_a": , + "grade_b": +}}""" + + +def rubric_judge_system_prompt(rubric: str) -> str: + return _RUBRIC_JUDGE_SYSTEM_PROMPT_TEMPLATE.format(rubric=rubric) + + +class RubricVerdict(LiteLLMBaseModel): + """The rubric judge's blind verdict: the pairwise preference plus one grade per label. + A grade is exactly a JSON integer from 1 to 5; booleans, floats, strings, null, and + out-of-range values invalidate the whole verdict rather than being coerced.""" + + model_config = ConfigDict(frozen=True) + + preference: Literal["A", "B", "tie"] + confidence: float + grade_a: Literal[1, 2, 3, 4, 5] + grade_b: Literal[1, 2, 3, 4, 5] + + @field_validator("grade_a", "grade_b", mode="before") + @classmethod + def _exact_integer(cls, value: object) -> object: + if type(value) is not int: + raise ValueError("grade must be a JSON integer") + return value + + +RUBRIC_JUDGE_RESPONSE_FORMAT: Final = _CHAT_REQUEST_ADAPTER.validate_python( + type_to_response_format_param(RubricVerdict) +) + +SHADOW_EVAL_RUBRIC_ENABLED_ENV_VAR: Final = "LITELLM_SHADOW_EVAL_RUBRIC_ENABLED" + + +def is_shadow_eval_rubric_enabled() -> bool: + """Whether this process may start and sample rubric shadow evals. Off by default: a + worker without rubric support would sample a rubric job as a plain preference job, so + operators turn this on only once every worker sharing the database runs rubric support.""" + return str_to_bool(os.environ.get(SHADOW_EVAL_RUBRIC_ENABLED_ENV_VAR)) is True + def _sample_hits(request_id: str, job_id: str, percentage: float) -> bool: """Deterministically decide whether a request falls in the shadowed slice: hash-based @@ -516,6 +588,30 @@ def _judge_reply_shape(response: object) -> str: return f"finish_reason={_chat_finish_reason(response)}, content={body}, model={served}" +def _legacy_judge_call_failure(e: Exception) -> str: + return f"judge call failed: {e}" + + +def _rubric_judge_call_failure(e: Exception) -> str: + """Provider errors can echo the request or response, which carry the rubric and the + sampled turns, so a rubric row keeps only the exception type and HTTP status.""" + status: Final = getattr(e, "status_code", None) + detail: Final = f"type={type(e).__name__}" + (f", status={status}" if isinstance(status, int) else "") + return f"rubric judge call failed; {detail}" + + +def _invalid_verdict_detail(e: Exception) -> str: + """Which verdict fields failed and how, from validation metadata alone, so the row + never carries the judge's reply text.""" + if not isinstance(e, ValidationError): + return f"invalid=verdict:{type(e).__name__}" + fields: Final = ",".join( + f"{'.'.join(str(part) for part in error['loc']) or 'verdict'}:{error['type']}" + for error in e.errors(include_url=False, include_context=False, include_input=False) + ) + return f"invalid={fields}" + + def _call_cost(response: object) -> float: """Price one eval-arm call with the figure the spend pipeline bills: the router client stamps _hidden_params.response_cost from the deployment's own pricing, which the public @@ -537,6 +633,10 @@ def _price_map_cost(response: object) -> float: return 0.0 +def _coin_flip() -> bool: + return random.random() < 0.5 + + def _unmask_preference(raw_preference: str, real_is_a: bool) -> str: """Map the judge's blind A/B/tie verdict back to real/shadow/tie.""" normalized: Final = raw_preference.strip().lower() @@ -737,6 +837,23 @@ class _JudgeVerdict: cost: float +@dataclass(frozen=True, slots=True) +class _RubricJudgeVerdict: + """A parsed rubric verdict, preference and grades unmasked back to real/shadow.""" + + preference: str + confidence: float + cost: float + real_grade: int + shadow_grade: int + + +@dataclass(frozen=True, slots=True) +class _JudgeReply: + response: ModelResponse + real_is_a: bool + + class ActiveShadowEvalJob(LiteLLMBaseModel): """One active job as the sampling path needs it, validated straight off the untyped job row: immutable config plus the attempt count as of the cache fill (the turn @@ -757,9 +874,21 @@ class ActiveShadowEvalJob(LiteLLMBaseModel): max_turns: int max_budget: float | None = None ends_at: datetime + rubric: str | None = None + rubric_version: int | None = None attempts: int = 0 spend: float = 0.0 + @model_validator(mode="after") + def _rubric_runs_under_a_known_contract(self) -> "ActiveShadowEvalJob": + """A rubric row this worker cannot grade under the contract it was created with + fails closed rather than sampling as a preference-only job.""" + if self.rubric is None and self.rubric_version is None: + return self + if self.rubric is None or self.rubric_version != SHADOW_EVAL_RUBRIC_VERSION: + raise ValueError("a rubric job needs its rubric and a grading version this worker implements") + return self + @field_validator("ends_at") @classmethod def _as_utc(cls, value: datetime) -> datetime: @@ -916,6 +1045,8 @@ class ShadowEvalLogger(CustomLogger): job_spend_reader: Callable[[str, float, float], Awaitable[float]] | None = None, job_spend_writer: Callable[[str, float], Awaitable[None]] | None = None, funnel_recorder: Callable[[str, "ShadowEvalFunnelStage"], None] | None = None, + real_is_a_draw: Callable[[], bool] | None = None, + rubric_enabled: Callable[[], bool] | None = None, ) -> None: """Providers are callables so the proxy's lazily-initialized globals are resolved at call time, not at logger construction. The spend reader and writer wrap the @@ -926,6 +1057,8 @@ class ShadowEvalLogger(CustomLogger): self._read_job_spend = job_spend_reader or _job_spend_from_counter self._write_job_spend = job_spend_writer or _add_job_spend_to_counter self._record_funnel = funnel_recorder or _record_funnel_event + self._draw_real_is_a = real_is_a_draw or _coin_flip + self._rubric_enabled = rubric_enabled or is_shadow_eval_rubric_enabled self._inflight_shadow_tasks: int = 0 # Starts per job since the last cache fill, never decremented within a # generation; the refill absorbs written rows and resets. @@ -1006,9 +1139,11 @@ class ShadowEvalLogger(CustomLogger): eligible: list[ActiveShadowEvalJob] = [] # mutable-ok: bucketed per-job admission now: Final = datetime.now(timezone.utc) router: Final = self._router_provider() + rubric_enabled: Final = self._rubric_enabled() for job in active_jobs: if ( - now >= job.ends_at + (job.rubric is not None and not rubric_enabled) + or now >= job.ends_at or job.attempts + self._job_starts.get(job.id, 0) >= job.max_turns or (job.max_budget is not None and job.spend >= job.max_budget) or not _direction_admits(request_metadata, job) @@ -1243,13 +1378,25 @@ class ShadowEvalLogger(CustomLogger): return # From here the shadow call has billed, so every exit records its cost. try: - verdict: Final = await self._call_judge( - judge_model=job.judge_model, - messages=messages, - real_text=real_text, - shadow_text=shadow.text, - tools=shadow_params.get("tools"), - parent_metadata=parent_metadata, + verdict: Final = ( + await self._call_judge( + judge_model=job.judge_model, + messages=messages, + real_text=real_text, + shadow_text=shadow.text, + tools=shadow_params.get("tools"), + parent_metadata=parent_metadata, + ) + if job.rubric is None + else await self._call_rubric_judge( + judge_model=job.judge_model, + rubric=job.rubric, + messages=messages, + real_text=real_text, + shadow_text=shadow.text, + tools=shadow_params.get("tools"), + parent_metadata=parent_metadata, + ) ) if isinstance(verdict, _CallFailure): await self._record_attempt( @@ -1279,6 +1426,9 @@ class ShadowEvalLogger(CustomLogger): shadow=shadow, real_model=real_model, confidence=verdict.confidence, + grades=( + (verdict.real_grade, verdict.shadow_grade) if isinstance(verdict, _RubricJudgeVerdict) else None + ), judge_cost=verdict.cost, shadow_cost=shadow.cost, shadow_classifier_cost=shadow.classifier_cost, @@ -1323,6 +1473,7 @@ class ShadowEvalLogger(CustomLogger): shadow_cost: float = 0.0, shadow_classifier_cost: float = 0.0, error: str | None = None, + grades: tuple[int, int] | None = None, ) -> None: eval_spend: Final = judge_cost + shadow_cost + shadow_classifier_cost if eval_spend > 0: @@ -1347,6 +1498,7 @@ class ShadowEvalLogger(CustomLogger): "real_classifier_cost": real_classifier_cost, "real_cache_hit": real_cache_hit, "error": error[:_MAX_ERROR_CHARS] if error else None, + **({"real_grade": grades[0], "shadow_grade": grades[1]} if grades is not None else {}), } ) except Exception as e: # noqa: BLE001 # a lost row degrades sample size, nothing can disagree with it @@ -1402,19 +1554,22 @@ class ShadowEvalLogger(CustomLogger): classifier_cost=_decision_classifier_cost(shadow_metadata), ) - async def _call_judge( + async def _judge_reply( self, judge_model: str, + system_prompt: str, + response_format: Mapping[str, object] | None, messages: Sequence[Mapping[str, object]], real_text: str, shadow_text: str, tools: object, parent_metadata: Mapping[str, object], - ) -> "_JudgeVerdict | _CallFailure": - """Blind pairwise judge with A/B labels randomized to cancel position bias. Both + call_failure: Callable[[Exception], str], + ) -> "_JudgeReply | _CallFailure": + """One blind judge call with A/B labels randomized to cancel position bias. Both arms were offered the same tools, so the judge is shown their definitions too: a tool call is only assessable against what else was available to call instead.""" - real_is_a: Final = random.random() < 0.5 + real_is_a: Final = self._draw_real_is_a() response_a: Final = real_text if real_is_a else shadow_text response_b: Final = shadow_text if real_is_a else real_text @@ -1425,7 +1580,7 @@ class ShadowEvalLogger(CustomLogger): ) judge_metadata: Final = sanitized_forwardable_call_metadata(parent_metadata, SHADOW_EVAL_JUDGE_CALL_ORIGIN) judge_messages: Final = [ - {"role": "system", "content": PAIRWISE_JUDGE_SYSTEM_PROMPT}, + {"role": "system", "content": system_prompt}, { "role": "user", "content": _judge_user_prompt(conversation, response_a, response_b, _tool_definitions_text(tools)), @@ -1439,12 +1594,37 @@ class ShadowEvalLogger(CustomLogger): team_id=_forwarded_team_id(parent_metadata), temperature=0, max_tokens=JUDGE_MAX_OUTPUT_TOKENS, - response_format=PAIRWISE_JUDGE_RESPONSE_FORMAT, + response_format=response_format, metadata=judge_metadata, ) except Exception as e: # noqa: BLE001 # judge outages become error rows, not crashes verbose_logger.debug("shadow_eval: judge call failed: %s", e) - return _CallFailure(f"judge call failed: {e}") + return _CallFailure(call_failure(e)) + return _JudgeReply(response=response, real_is_a=real_is_a) + + async def _call_judge( + self, + judge_model: str, + messages: Sequence[Mapping[str, object]], + real_text: str, + shadow_text: str, + tools: object, + parent_metadata: Mapping[str, object], + ) -> "_JudgeVerdict | _CallFailure": + reply: Final = await self._judge_reply( + judge_model, + PAIRWISE_JUDGE_SYSTEM_PROMPT, + PAIRWISE_JUDGE_RESPONSE_FORMAT, + messages, + real_text, + shadow_text, + tools, + parent_metadata, + _legacy_judge_call_failure, + ) + if isinstance(reply, _CallFailure): + return reply + response: Final = reply.response try: raw: Final = response["choices"][0]["message"]["content"] or "" verdict: Final = PairwiseVerdict.model_validate(parse_json_verdict(raw)) @@ -1454,11 +1634,56 @@ class ShadowEvalLogger(CustomLogger): f"unparseable judge verdict: {e}; {_judge_reply_shape(response)}", cost=_call_cost(response) ) return _JudgeVerdict( - preference=_unmask_preference(verdict.preference, real_is_a), + preference=_unmask_preference(verdict.preference, reply.real_is_a), confidence=max(0.0, min(1.0, verdict.confidence)), cost=_call_cost(response), ) + async def _call_rubric_judge( + self, + judge_model: str, + rubric: str, + messages: Sequence[Mapping[str, object]], + real_text: str, + shadow_text: str, + tools: object, + parent_metadata: Mapping[str, object], + ) -> "_RubricJudgeVerdict | _CallFailure": + """The pairwise judge plus a grade per response. The whole verdict validates before + anything counts, so a malformed grade is an error row carrying the judge's cost, + never a preference-only or defaulted verdict. Errors name the invalid fields and the + reply's shape, never the reply or the rubric.""" + reply: Final = await self._judge_reply( + judge_model, + rubric_judge_system_prompt(rubric), + RUBRIC_JUDGE_RESPONSE_FORMAT, + messages, + real_text, + shadow_text, + tools, + parent_metadata, + _rubric_judge_call_failure, + ) + if isinstance(reply, _CallFailure): + return reply + response: Final = reply.response + try: + raw: Final = response["choices"][0]["message"]["content"] or "" + verdict: Final = RubricVerdict.model_validate(parse_json_verdict(raw)) + except Exception as e: # noqa: BLE001 # malformed verdicts become error rows + verbose_logger.debug("shadow_eval: unparseable rubric verdict: %s", type(e).__name__) + return _CallFailure( + f"unparseable rubric verdict; {_invalid_verdict_detail(e)}, {_judge_reply_shape(response)}", + cost=_call_cost(response), + ) + return _RubricJudgeVerdict( + preference=_unmask_preference(verdict.preference, reply.real_is_a), + confidence=max(0.0, min(1.0, verdict.confidence)), + cost=_call_cost(response), + real_grade=verdict.grade_a if reply.real_is_a else verdict.grade_b, + shadow_grade=verdict.grade_b if reply.real_is_a else verdict.grade_a, + ) + _EMPTY_JOBS: Final[Mapping[tuple[str, str], tuple[ActiveShadowEvalJob, ...]]] = MappingProxyType({}) diff --git a/litellm/proxy-extras/litellm_proxy_extras/migrations/20261117000000_shadow_eval_rubric_grades/migration.sql b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261117000000_shadow_eval_rubric_grades/migration.sql new file mode 100644 index 00000000000..c649fa3c8c3 --- /dev/null +++ b/litellm/proxy-extras/litellm_proxy_extras/migrations/20261117000000_shadow_eval_rubric_grades/migration.sql @@ -0,0 +1,13 @@ +SET LOCAL lock_timeout = '10s'; + +-- AlterTable +ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN IF NOT EXISTS "rubric" TEXT; + +-- AlterTable +ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN IF NOT EXISTS "rubric_version" INTEGER; + +-- AlterTable +ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN IF NOT EXISTS "real_grade" INTEGER; + +-- AlterTable +ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN IF NOT EXISTS "shadow_grade" INTEGER; diff --git a/litellm/proxy-extras/litellm_proxy_extras/schema.prisma b/litellm/proxy-extras/litellm_proxy_extras/schema.prisma index 9dedb0a4f2c..c02531ee73e 100644 --- a/litellm/proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm/proxy-extras/litellm_proxy_extras/schema.prisma @@ -2091,6 +2091,8 @@ model LiteLLM_ShadowEvalJob { ends_at DateTime stopped_at DateTime? stopped_by String? // operator who stopped it early; null when it ended on its own + rubric String? // criteria both responses are graded against; null on preference-only jobs + rubric_version Int? // grading contract the rubric job runs under; set exactly when rubric is @@index([group_id]) @@index([target_type, target_id]) @@ -2114,6 +2116,8 @@ model LiteLLM_ShadowEvalAttempt { real_classifier_cost Float @default(0) shadow_classifier_cost Float @default(0) real_cache_hit Boolean @default(false) + real_grade Int? // rubric jobs: 1-5 grade of the response the caller received; null = ungraded + shadow_grade Int? // rubric jobs: 1-5 grade of the shadow response; null = ungraded error String? created_at DateTime @default(now()) diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index bebec1aa8e3..c169624d948 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -73,6 +73,7 @@ from litellm.router_strategy.complexity_router.request_models import ( ) from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.management_endpoints.auto_router_endpoints import ( + SHADOW_EVAL_RUBRIC_VERSION, SHADOW_EVAL_TURN_VALVE, AutoRouterAvailabilityRequest, AutoRouterAvailabilityResponse, @@ -86,12 +87,17 @@ from litellm.types.management_endpoints.auto_router_endpoints import ( ComplexityRouterConfigValidationRequest, ComplexityRouterConfigValidationResponse, ShadowEvalDirection, + ShadowEvalGradePair, ShadowEvalJobResponse, ShadowEvalJobTargetResponse, ShadowEvalResult, + ShadowEvalRubricJobResponse, + ShadowEvalRubricResultResponse, + ShadowEvalRubricSlice, ShadowEvalSlice, ShadowEvalTargetType, StartShadowEvalRequest, + StartShadowEvalRubricRequest, ) if TYPE_CHECKING: @@ -1462,6 +1468,51 @@ class _AttemptTotalsRow(LiteLLMBaseModel): _ATTEMPT_TOTALS_ROWS: Final = TypeAdapter(list[_AttemptTotalsRow]) +# Grades are counted per arm on the same rows the preference results read (errors +# excluded); a verdict row without grades is counted as ungraded, never as a grade. +_RUBRIC_GRADE_COUNTS_SQL: Final = """ +SELECT COALESCE(a.router_name, j.router_name) AS grp, a.real_grade, a.shadow_grade, COUNT(*)::int AS turn_count +FROM "LiteLLM_ShadowEvalAttempt" a +JOIN "LiteLLM_ShadowEvalJob" j ON j.id = a.job_id +WHERE a.job_id = ANY($1::text[]) AND a.outcome != 'error' +GROUP BY 1, 2, 3 +""" + + +class _GradeCountRow(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + grp: str + real_grade: int | None + shadow_grade: int | None + turn_count: int + + @property + def grade_pair(self) -> tuple[int, int] | None: + if self.real_grade is None or self.shadow_grade is None: + return None + return self.real_grade, self.shadow_grade + + +_GRADE_COUNT_ROWS: Final = TypeAdapter(list[_GradeCountRow]) + + +def _rubric_slice(group: str, rows: Sequence[_GradeCountRow]) -> ShadowEvalRubricSlice: + pairs: Final = sorted(frozenset(pair for row in rows if (pair := row.grade_pair) is not None)) + return ShadowEvalRubricSlice( + group=group, + graded_count=sum(row.turn_count for row in rows if row.grade_pair is not None), + ungraded_count=sum(row.turn_count for row in rows if row.grade_pair is None), + grade_distribution=tuple( + ShadowEvalGradePair( + real_grade=real_grade, + shadow_grade=shadow_grade, + turn_count=sum(row.turn_count for row in rows if row.grade_pair == (real_grade, shadow_grade)), + ) + for real_grade, shadow_grade in pairs + ), + ) + def _pct_of(numerator: int, denominator: int) -> float: return _pct(numerator, denominator) @@ -1514,6 +1565,8 @@ class _LegRow(LiteLLMBaseModel): ends_at: datetime stopped_at: datetime | None = None stopped_by: str | None = None + rubric: str | None = None + rubric_version: int | None = None @property def arm_router_names(self) -> tuple[str, ...]: @@ -1756,9 +1809,17 @@ async def start_shadow_eval( seconds. Shadow and judge calls bill to the sampled request's own identity but are excluded from request counts and auto-router adoption metrics. """ + _require_admin_writer(user_api_key_dict, "start a shadow eval") + return await _create_shadow_eval_job(data, user_api_key_dict, rubric=None) + + +async def _create_shadow_eval_job( + data: StartShadowEvalRequest, user_api_key_dict: UserAPIKeyAuth, rubric: str | None +) -> ShadowEvalJobResponse: + """Validate and claim a job for an authorized writer. The rubric columns are written + only for a rubric job, so a preference-only job inserts exactly what it always did.""" from litellm.proxy.proxy_server import llm_router, prisma_client - _require_admin_writer(user_api_key_dict, "start a shadow eval") if prisma_client is None: raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) unconfigured: Final = tuple( @@ -1878,6 +1939,7 @@ async def start_shadow_eval( "created_by": user_api_key_dict.user_id, "created_at": now, "ends_at": ends_at, + **({"rubric": rubric, "rubric_version": SHADOW_EVAL_RUBRIC_VERSION} if rubric is not None else {}), } try: # Leg ids are minted here rather than by the DB default so the funnel seed below @@ -1941,6 +2003,99 @@ async def start_shadow_eval( ) +@router.post( + "/auto_router/shadow_eval/rubric/start", + tags=["auto router"], + dependencies=(Depends(user_api_key_auth),), + response_model=ShadowEvalRubricJobResponse, + status_code=status.HTTP_201_CREATED, +) +async def start_shadow_eval_rubric( + data: StartShadowEvalRubricRequest, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> ShadowEvalRubricJobResponse: + """ + Start a shadow eval whose judge, besides picking the better response, grades each + response from 1 (fails the rubric's criteria) to 5 (fully meets them). Everything + else matches /auto_router/shadow_eval/start, and the job's preference results stay + on the regular job endpoints. Read grades from + /auto_router/shadow_eval/{job_id}/rubric_results. + + Disabled unless LITELLM_SHADOW_EVAL_RUBRIC_ENABLED=true. Each worker process reads + its own environment, so a change applies only once every worker is redeployed or + restarted with it. A worker without rubric support ignores the setting and samples a + rubric job as a preference-only job, so enable it only after every API and sampling + worker sharing the database runs a version with this route. Workers with the setting + off neither start nor sample rubric jobs. + + Rolling back to a version without rubric support is not zero-downtime for rubric + jobs. Before any older worker starts: apply the setting off to every upgraded worker, + stop every running rubric job with /auto_router/shadow_eval/{job_id}/stop, and wait + until every upgraded worker's in-flight evaluations have finished, for example by + draining or stopping those workers. The roughly 10 second job cache expiry alone does + not guarantee that drain. + """ + from litellm.integrations.shadow_eval_logger import ( + SHADOW_EVAL_RUBRIC_ENABLED_ENV_VAR, + is_shadow_eval_rubric_enabled, + ) + + _require_admin_writer(user_api_key_dict, "start a shadow eval") + if not is_shadow_eval_rubric_enabled(): + raise HTTPException( + status_code=400, + detail=( + f"Rubric shadow evals are disabled. Set {SHADOW_EVAL_RUBRIC_ENABLED_ENV_VAR}=true once every " + "worker sharing this database supports rubric grading" + ), + ) + job: Final = await _create_shadow_eval_job(data, user_api_key_dict, rubric=data.rubric) + return ShadowEvalRubricJobResponse.model_validate( + { + **job.model_dump(exclude={"router_name", "status"}), + "rubric": data.rubric, + "rubric_version": SHADOW_EVAL_RUBRIC_VERSION, + } + ) + + +@router.get( + "/auto_router/shadow_eval/{job_id}/rubric_results", + tags=["auto router"], + dependencies=(Depends(user_api_key_auth),), + response_model=ShadowEvalRubricResultResponse, +) +async def get_shadow_eval_rubric_results( + job_id: str, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> ShadowEvalRubricResultResponse: + """A rubric job's grades: graded and ungraded turn counts and the joint distribution + of (real_grade, shadow_grade), overall and per router arm. A job started without a + rubric is a 404 here.""" + from litellm.proxy.proxy_server import prisma_client + + _require_admin_viewer(user_api_key_dict, "view shadow evals") + if prisma_client is None: + raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) + legs: Final = _LEG_ROWS.validate_python( + await _shadow_eval_jobs(prisma_client).find_many(where={"group_id": job_id}) or () + ) + first: Final = legs[0] if legs else None + if first is None or first.rubric is None or first.rubric_version is None: + raise HTTPException(status_code=404, detail=f"No rubric shadow eval job {job_id}") + rows: Final = _GRADE_COUNT_ROWS.validate_python( + await _query_raw(prisma_client, _RUBRIC_GRADE_COUNTS_SQL, [leg.id for leg in legs]) or () + ) + routers: Final = sorted(frozenset((*first.arm_router_names, *(row.grp for row in rows)))) + return ShadowEvalRubricResultResponse( + job_id=job_id, + rubric=first.rubric, + rubric_version=first.rubric_version, + overall=_rubric_slice("all", rows), + by_router=tuple(_rubric_slice(name, tuple(row for row in rows if row.grp == name)) for name in routers), + ) + + @router.get( "/auto_router/shadow_eval", tags=("auto router",), diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 9dedb0a4f2c..c02531ee73e 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -2091,6 +2091,8 @@ model LiteLLM_ShadowEvalJob { ends_at DateTime stopped_at DateTime? stopped_by String? // operator who stopped it early; null when it ended on its own + rubric String? // criteria both responses are graded against; null on preference-only jobs + rubric_version Int? // grading contract the rubric job runs under; set exactly when rubric is @@index([group_id]) @@index([target_type, target_id]) @@ -2114,6 +2116,8 @@ model LiteLLM_ShadowEvalAttempt { real_classifier_cost Float @default(0) shadow_classifier_cost Float @default(0) real_cache_hit Boolean @default(false) + real_grade Int? // rubric jobs: 1-5 grade of the response the caller received; null = ungraded + shadow_grade Int? // rubric jobs: 1-5 grade of the shadow response; null = ungraded error String? created_at DateTime @default(now()) diff --git a/litellm/schema.prisma b/litellm/schema.prisma index 9dedb0a4f2c..c02531ee73e 100644 --- a/litellm/schema.prisma +++ b/litellm/schema.prisma @@ -2091,6 +2091,8 @@ model LiteLLM_ShadowEvalJob { ends_at DateTime stopped_at DateTime? stopped_by String? // operator who stopped it early; null when it ended on its own + rubric String? // criteria both responses are graded against; null on preference-only jobs + rubric_version Int? // grading contract the rubric job runs under; set exactly when rubric is @@index([group_id]) @@index([target_type, target_id]) @@ -2114,6 +2116,8 @@ model LiteLLM_ShadowEvalAttempt { real_classifier_cost Float @default(0) shadow_classifier_cost Float @default(0) real_cache_hit Boolean @default(false) + real_grade Int? // rubric jobs: 1-5 grade of the response the caller received; null = ungraded + shadow_grade Int? // rubric jobs: 1-5 grade of the shadow response; null = ungraded error String? created_at DateTime @default(now()) diff --git a/litellm/types/management_endpoints/auto_router_endpoints.py b/litellm/types/management_endpoints/auto_router_endpoints.py index 458899529b8..5892b36107e 100644 --- a/litellm/types/management_endpoints/auto_router_endpoints.py +++ b/litellm/types/management_endpoints/auto_router_endpoints.py @@ -7,7 +7,7 @@ from collections.abc import Mapping from datetime import datetime, timezone from typing import Final, Literal, TypeAlias -from pydantic import Field, computed_field, field_validator, model_validator +from pydantic import ConfigDict, Field, computed_field, field_validator, model_validator from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.utils import StandardLoggingRoutingDecision @@ -246,6 +246,12 @@ SHADOW_EVAL_TURN_VALVE: Final[int] = 10_000 SHADOW_EVAL_MAX_ROUTERS: Final[int] = 4 +SHADOW_EVAL_RUBRIC_MAX_CHARS: Final[int] = 4_000 + +# Identifies the rubric grading contract (prompt, anchors, 1-5 scale) a job was created +# under. A sampler only runs rubric jobs whose version it implements. +SHADOW_EVAL_RUBRIC_VERSION: Final[int] = 1 + class StartShadowEvalRequest(LiteLLMBaseModel): """Start duplicating one or more targets' traffic for blind comparison against an auto-router. @@ -446,6 +452,27 @@ class StartShadowEvalRequest(LiteLLMBaseModel): return self +class StartShadowEvalRubricRequest(StartShadowEvalRequest): # frozen-ok: parent validator assigns router_names + """A shadow eval whose judge also grades each response from 1 to 5 against an operator rubric.""" + + rubric: str = Field( + min_length=1, + max_length=SHADOW_EVAL_RUBRIC_MAX_CHARS, + description=( + "The criteria the judge grades both responses against, in plain language, at most " + f"{SHADOW_EVAL_RUBRIC_MAX_CHARS} characters. Each response gets an integer grade on a fixed scale " + "where 1 fails the criteria and 5 fully meets them. Stored unchanged with the job and never edited" + ), + ) + + @field_validator("rubric") + @classmethod + def _rubric_is_not_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("rubric must contain criteria, not only whitespace") + return value + + class ShadowEvalSlice(LiteLLMBaseModel): """Judge outcomes for one slice of a job's verdicts: a router tier, one of the models that served the real arm, or one scoped target (embedded on that target's @@ -697,6 +724,65 @@ class ShadowEvalJobResponse(LiteLLMBaseModel): return "running" +class ShadowEvalRubricJobResponse(ShadowEvalJobResponse): + """A rubric shadow eval job: the regular job view plus the rubric it grades against.""" + + model_config = ConfigDict(frozen=True) + + rubric: str = Field(description="The criteria both responses are graded against, as stored at start") + rubric_version: int = Field(description="The grading contract (prompt, anchors, 1-5 scale) the job runs under") + + +class ShadowEvalGradePair(LiteLLMBaseModel): + """How many judged turns gave the real and shadow responses this pair of grades.""" + + model_config = ConfigDict(frozen=True) + + real_grade: int = Field(description="Grade of the response the caller actually received, 1 to 5") + shadow_grade: int = Field(description="Grade of the duplicated response nobody was served, 1 to 5") + turn_count: int + + +class ShadowEvalRubricSlice(LiteLLMBaseModel): + """Rubric grades for one router arm, or for every arm together.""" + + model_config = ConfigDict(frozen=True) + + group: str + graded_count: int = Field(description="Judged turns that carry both grades") + ungraded_count: int = Field( + description=( + "Judged turns with a preference but no grades, such as turns a worker without rubric support " + "judged. They are excluded from grade_distribution, never counted as low grades" + ) + ) + grade_distribution: tuple[ShadowEvalGradePair, ...] = Field( + description=( + "Joint counts of (real_grade, shadow_grade) over graded turns, ordered by real then shadow grade, " + "so turns where both responses were good or both were bad stay distinguishable" + ) + ) + + +class ShadowEvalRubricResultResponse(LiteLLMBaseModel): + """A rubric job's grades so far. Error attempts are excluded, like the job's preference results.""" + + model_config = ConfigDict(frozen=True) + + job_id: str + rubric: str + rubric_version: int + overall: ShadowEvalRubricSlice = Field( + description=( + "Every arm together. A judged turn is one (request, router arm) pair, so a multi-router job " + "counts each real response once per arm; compare routers with by_router" + ) + ) + by_router: tuple[ShadowEvalRubricSlice, ...] = Field( + description="One slice per router arm, ordered by router name, each graded against the same real responses" + ) + + def __getattr__(name: str) -> object: if name in _REQUEST_MODELS_EXPORTS: return getattr(importlib.import_module(_REQUEST_MODELS_MODULE), name) diff --git a/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts b/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts index 6a102e0e042..28095006e5c 100644 --- a/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -1576,6 +1576,44 @@ export interface paths { patch?: never; trace?: never; }; + "/auto_router/shadow_eval/rubric/start": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Start Shadow Eval Rubric + * @description Start a shadow eval whose judge, besides picking the better response, grades each + * response from 1 (fails the rubric's criteria) to 5 (fully meets them). Everything + * else matches /auto_router/shadow_eval/start, and the job's preference results stay + * on the regular job endpoints. Read grades from + * /auto_router/shadow_eval/{job_id}/rubric_results. + * + * Disabled unless LITELLM_SHADOW_EVAL_RUBRIC_ENABLED=true. Each worker process reads + * its own environment, so a change applies only once every worker is redeployed or + * restarted with it. A worker without rubric support ignores the setting and samples a + * rubric job as a preference-only job, so enable it only after every API and sampling + * worker sharing the database runs a version with this route. Workers with the setting + * off neither start nor sample rubric jobs. + * + * Rolling back to a version without rubric support is not zero-downtime for rubric + * jobs. Before any older worker starts: apply the setting off to every upgraded worker, + * stop every running rubric job with /auto_router/shadow_eval/{job_id}/stop, and wait + * until every upgraded worker's in-flight evaluations have finished, for example by + * draining or stopping those workers. The roughly 10 second job cache expiry alone does + * not guarantee that drain. + */ + post: operations["start_shadow_eval_rubric_auto_router_shadow_eval_rubric_start_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/auto_router/shadow_eval/start": { parameters: { query?: never; @@ -1641,6 +1679,28 @@ export interface paths { patch?: never; trace?: never; }; + "/auto_router/shadow_eval/{job_id}/rubric_results": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get Shadow Eval Rubric Results + * @description A rubric job's grades: graded and ungraded turn counts and the joint distribution + * of (real_grade, shadow_grade), overall and per router arm. A job started without a + * rubric is a 404 here. + */ + get: operations["get_shadow_eval_rubric_results_auto_router_shadow_eval__job_id__rubric_results_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/auto_router/shadow_eval/{job_id}/stop": { parameters: { query?: never; @@ -48609,6 +48669,24 @@ export interface components { /** Message */ message: string; }; + /** + * ShadowEvalGradePair + * @description How many judged turns gave the real and shadow responses this pair of grades. + */ + ShadowEvalGradePair: { + /** + * Real Grade + * @description Grade of the response the caller actually received, 1 to 5 + */ + real_grade: number; + /** + * Shadow Grade + * @description Grade of the duplicated response nobody was served, 1 to 5 + */ + shadow_grade: number; + /** Turn Count */ + turn_count: number; + }; /** * ShadowEvalJobResponse * @description A shadow-eval job over one or more targets, each with its own budget and stop state; @@ -48821,6 +48899,158 @@ export interface components { */ withheld_count?: number | null; }; + /** + * ShadowEvalRubricJobResponse + * @description A rubric shadow eval job: the regular job view plus the rubric it grades against. + */ + ShadowEvalRubricJobResponse: { + /** Baseline Model */ + baseline_model?: string | null; + /** + * Created At + * Format: date-time + */ + created_at: string; + /** + * Direction + * @default forward + * @enum {string} + */ + direction: "forward" | "reverse"; + /** + * Ends At + * Format: date-time + */ + ends_at: string; + /** + * Error Count + * @description Sampled attempts that errored; detail endpoint only + */ + error_count?: number | null; + /** Job Id */ + job_id: string; + /** Judge Model */ + judge_model: string; + /** + * Judge Spend + * @description Judge cost so far; detail endpoint only + */ + judge_spend?: number | null; + /** + * Judged Count + * @description Verdicts recorded; detail endpoint only + */ + judged_count?: number | null; + /** + * Last Error + * @description Most recent attempt error; detail endpoint only + */ + last_error?: string | null; + /** + * Models + * @description Model groups the sampled traffic is narrowed to; empty means every model the targets use + * @default [] + */ + models: string[]; + /** @description Stratified verdicts; detail endpoint only */ + results?: components["schemas"]["ShadowEvalResult"] | null; + /** + * Router Name + * @description The first router, kept for callers that predate router_names; derived so the + * two fields can never disagree. + */ + readonly router_name: string; + /** + * Router Names + * @description Every auto-router this job runs as a shadow arm. Multi-router jobs sample one slice of traffic and judge every arm against the same real responses + */ + router_names: string[]; + /** + * Rubric + * @description The criteria both responses are graded against, as stored at start + */ + rubric: string; + /** + * Rubric Version + * @description The grading contract (prompt, anchors, 1-5 scale) the job runs under + */ + rubric_version: number; + /** + * Same Model Skipped Count + * @description Sampled requests skipped because baseline_model's own model group served them, summed over targets; detail endpoint only and reported even before any turn is judged. None when coverage counts are unknown for any target + */ + same_model_skipped_count?: number | null; + /** Shadow Percentage */ + shadow_percentage: number; + /** + * Skip Same Model + * @description Whether this reverse job skips samples that baseline_model's own model group served + * @default false + */ + skip_same_model: boolean; + /** + * Status + * @description Three recorded facts, no history-guessing: a stop is stopped_by (the migration + * backfills it for every job that displayed stopped when the column arrived, so the + * pre-column population is closed), completion is the window passing or every target + * spending its budget, and anything else is running. The all-targets-stamped fallback + * covers only stops written by pre-column pods during a rolling deploy. + * @enum {string} + */ + readonly status: "running" | "completed" | "stopped"; + /** + * Stopped By + * @description The operator who stopped the job early, recorded by the stop endpoint; 'unknown' backfilled by migration for jobs that displayed stopped when the column arrived; None when the job ended on its own. Its presence is what makes a job read stopped rather than completed + */ + stopped_by?: string | null; + /** + * Targets + * @description The targets whose traffic this job evaluates, and only theirs, each with its own budget + */ + targets: components["schemas"]["ShadowEvalJobTargetResponse"][]; + }; + /** + * ShadowEvalRubricResultResponse + * @description A rubric job's grades so far. Error attempts are excluded, like the job's preference results. + */ + ShadowEvalRubricResultResponse: { + /** + * By Router + * @description One slice per router arm, ordered by router name, each graded against the same real responses + */ + by_router: components["schemas"]["ShadowEvalRubricSlice"][]; + /** Job Id */ + job_id: string; + /** @description Every arm together. A judged turn is one (request, router arm) pair, so a multi-router job counts each real response once per arm; compare routers with by_router */ + overall: components["schemas"]["ShadowEvalRubricSlice"]; + /** Rubric */ + rubric: string; + /** Rubric Version */ + rubric_version: number; + }; + /** + * ShadowEvalRubricSlice + * @description Rubric grades for one router arm, or for every arm together. + */ + ShadowEvalRubricSlice: { + /** + * Grade Distribution + * @description Joint counts of (real_grade, shadow_grade) over graded turns, ordered by real then shadow grade, so turns where both responses were good or both were bad stay distinguishable + */ + grade_distribution: components["schemas"]["ShadowEvalGradePair"][]; + /** + * Graded Count + * @description Judged turns that carry both grades + */ + graded_count: number; + /** Group */ + group: string; + /** + * Ungraded Count + * @description Judged turns with a preference but no grades, such as turns a worker without rubric support judged. They are excluded from grade_distribution, never counted as low grades + */ + ungraded_count: number; + }; /** * ShadowEvalSlice * @description Judge outcomes for one slice of a job's verdicts: a router tier, one of the @@ -49975,6 +50205,93 @@ export interface components { */ user_ids: string[]; }; + /** + * StartShadowEvalRubricRequest + * @description A shadow eval whose judge also grades each response from 1 to 5 against an operator rubric. + */ + StartShadowEvalRubricRequest: { + /** + * Api Key Ids + * @description Hashed virtual keys whose traffic will be shadowed. Combined with team_ids and user_ids the job needs at least one target and at most 100, which also bounds every read the job's endpoints make. Each target carries its own max_budget spend budget, so one exhausting its budget leaves the others sampling. + * @default [] + */ + api_key_ids: string[]; + /** + * Baseline Model + * @description Required when direction is reverse and rejected otherwise: the fixed model the router's own responses are judged against. Must be a plain model rather than another auto-router + */ + baseline_model?: string | null; + /** + * Direction + * @description forward answers 'should this key adopt router_name': it samples the requests the key did NOT route through the router and duplicates them through it. reverse answers 'is the router still worth it for a key already on it': it samples the requests the router did serve and duplicates them against baseline_model. The response the caller received is always the real arm + * @default forward + * @enum {string} + */ + direction: "forward" | "reverse"; + /** + * Duration Days + * @description How many days the job samples traffic before completing on its own + * @default 7 + */ + duration_days: number; + /** + * Judge Model + * @description Model used to blindly judge real vs. shadow responses. The judge only compares two answers, so a mid-tier model (Claude Sonnet or GPT-4o class) is the sweet spot: small/nano-class models produce unreliable or malformed verdicts, while frontier reasoning models add cost without changing outcomes. + * @default anthropic/claude-sonnet-5 + */ + judge_model: string; + /** + * Max Budget + * @description Per-target USD budget for the eval's own overhead, the shadow-arm and judge calls, priced with the same figures the spend pipeline bills. EACH scoped target samples until its recorded eval spend reaches this, so a job over N targets spends at most about N times max_budget; in-flight samples can overshoot the cap by one sampling cache window. Every router arm draws from the same per-target budget, so a multi-router job reaches it proportionally sooner + * @default 10 + */ + max_budget: number; + /** + * Models + * @description Model groups to narrow the sampled traffic to, matched on the group the caller requested and resolved through model_group_alias, so an alias and its target are one name. Empty samples every model the targets use. This ANDs with the targets: a job over a user and one model samples that user's requests on that model across every key they own, and none of their other traffic. Forward jobs only: a reverse job samples exactly the traffic its own router served, which no other model group can name + * @default [] + */ + models: string[]; + /** + * Router Name + * @description The auto-router under evaluation, in either direction: the single-router spelling of router_names. Provide exactly one of the two fields + */ + router_name?: string | null; + /** + * Router Names + * @description The auto-routers under evaluation, at most 4. Every sampled request runs through every router listed and each arm is judged independently against the same real response, so routers compare head-to-head on identical traffic. More than one router requires direction 'forward'. After validation this field always carries the full deduplicated set, whichever spelling the caller used + * @default [] + */ + router_names: string[]; + /** + * Rubric + * @description The criteria the judge grades both responses against, in plain language, at most 4000 characters. Each response gets an integer grade on a fixed scale where 1 fails the criteria and 5 fully meets them. Stored unchanged with the job and never edited + */ + rubric: string; + /** + * Shadow Percentage + * @description Percentage of each target's requests to duplicate through the router + */ + shadow_percentage: number; + /** + * Skip Same Model + * @description Reverse jobs only. When true, a sampled request whose response came from baseline_model's own model group (aliases resolved) is skipped instead of judged, so no shadow or judge call is made or billed for it and it never counts toward win, tie, or error rates. Skips are reported as same_model_skipped_count. Different model group names stay distinct even when they share a provider model + * @default false + */ + skip_same_model: boolean; + /** + * Team Ids + * @description Teams whose traffic will be shadowed, matched on the team every authenticated request resolves to, so a team's JWT-auth and virtual-key traffic are both sampled + * @default [] + */ + team_ids: string[]; + /** + * User Ids + * @description Users whose traffic will be shadowed, matched on the user every authenticated request resolves to across all their teams: JWT requests carrying their subject claim and virtual keys they own + * @default [] + */ + user_ids: string[]; + }; /** StoredReport */ StoredReport: { /** Id */ @@ -57838,6 +58155,39 @@ export interface operations { }; }; }; + start_shadow_eval_rubric_auto_router_shadow_eval_rubric_start_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["StartShadowEvalRubricRequest"]; + }; + }; + responses: { + /** @description Successful Response */ + 201: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ShadowEvalRubricJobResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; start_shadow_eval_auto_router_shadow_eval_start_post: { parameters: { query?: never; @@ -57902,6 +58252,37 @@ export interface operations { }; }; }; + get_shadow_eval_rubric_results_auto_router_shadow_eval__job_id__rubric_results_get: { + parameters: { + query?: never; + header?: never; + path: { + job_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ShadowEvalRubricResultResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; stop_shadow_eval_job_auto_router_shadow_eval__job_id__stop_post: { parameters: { query?: never; diff --git a/tests/code_coverage_tests/unbounded_in_baseline.txt b/tests/code_coverage_tests/unbounded_in_baseline.txt index c7375c5a7e6..011772eb9b7 100644 --- a/tests/code_coverage_tests/unbounded_in_baseline.txt +++ b/tests/code_coverage_tests/unbounded_in_baseline.txt @@ -30,10 +30,10 @@ litellm/proxy/management_endpoints/auto_router_endpoints.py _with_target_labels litellm/proxy/management_endpoints/auto_router_endpoints.py _with_target_labels prisma token.in `list(tokens)` 0 litellm/proxy/management_endpoints/auto_router_endpoints.py _with_target_labels prisma user_id.in `list(user_ids)` 0 litellm/proxy/management_endpoints/auto_router_endpoints.py get_shadow_eval_job prisma job_id.in `leg_ids` 0 -litellm/proxy/management_endpoints/auto_router_endpoints.py start_shadow_eval prisma target_id.in `list(ids)` 0 -litellm/proxy/management_endpoints/auto_router_endpoints.py start_shadow_eval prisma team_id.in `list(data.team_ids)` 0 -litellm/proxy/management_endpoints/auto_router_endpoints.py start_shadow_eval prisma token.in `list(data.api_key_ids)` 0 -litellm/proxy/management_endpoints/auto_router_endpoints.py start_shadow_eval prisma user_id.in `list(data.user_ids)` 0 +litellm/proxy/management_endpoints/auto_router_endpoints.py _create_shadow_eval_job prisma target_id.in `list(ids)` 0 +litellm/proxy/management_endpoints/auto_router_endpoints.py _create_shadow_eval_job prisma team_id.in `list(data.team_ids)` 0 +litellm/proxy/management_endpoints/auto_router_endpoints.py _create_shadow_eval_job prisma token.in `list(data.api_key_ids)` 0 +litellm/proxy/management_endpoints/auto_router_endpoints.py _create_shadow_eval_job prisma user_id.in `list(data.user_ids)` 0 litellm/proxy/management_endpoints/budget_management_endpoints.py info_budget prisma budget_id.in `data.budgets` 0 litellm/proxy/management_endpoints/common_utils.py _team_admin_can_invite_user prisma team_id.in `admin_user_obj.teams` 0 litellm/proxy/management_endpoints/common_utils.py user_has_admin_privileges prisma team_id.in `user_obj.teams` 0 diff --git a/tests/unit/integrations/test_shadow_eval_logger.py b/tests/unit/integrations/test_shadow_eval_logger.py index d08433c7522..d01a99b0c7f 100644 --- a/tests/unit/integrations/test_shadow_eval_logger.py +++ b/tests/unit/integrations/test_shadow_eval_logger.py @@ -2,6 +2,7 @@ the detached pipeline's single attempt-row write, and the cache-first job lookup.""" import asyncio +import json from collections.abc import AsyncIterator, Coroutine, Mapping from contextlib import asynccontextmanager from contextvars import Context @@ -20,6 +21,9 @@ from litellm.integrations.shadow_eval_logger import ( _MAX_JUDGE_PROMPT_CHARS, JUDGE_MAX_OUTPUT_TOKENS, PAIRWISE_JUDGE_RESPONSE_FORMAT, + PAIRWISE_JUDGE_SYSTEM_PROMPT, + RUBRIC_JUDGE_RESPONSE_FORMAT, + SHADOW_EVAL_RUBRIC_ENABLED_ENV_VAR, ActiveShadowEvalJob, GuardrailRequestSnapshot, ShadowEvalLogger, @@ -27,9 +31,12 @@ from litellm.integrations.shadow_eval_logger import ( _judge_user_prompt, _sample_hits, _unmask_preference, + is_shadow_eval_rubric_enabled, request_guardrail_fingerprint, + rubric_judge_system_prompt, ) from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.management_endpoints.auto_router_endpoints import SHADOW_EVAL_RUBRIC_VERSION from litellm.types.utils import ( SHADOW_EVAL_JUDGE_CALL_ORIGIN, SHADOW_EVAL_ROUTER_CALL_ORIGIN, @@ -79,7 +86,14 @@ def _prisma(jobs=(), attempt_counts=(), attempt_costs=()) -> MagicMock: return prisma -def _job_record(job: ActiveShadowEvalJob, target_type="key", target_id="key-hash") -> MagicMock: +def _job_record( + job: ActiveShadowEvalJob, + target_type="key", + target_id="key-hash", + rubric: str | None = None, + rubric_version: int | None = None, +) -> MagicMock: + """rubric and rubric_version, when given, override the job's, to pose rows the job model rejects.""" record = MagicMock() for field, value in dict( id=job.id, @@ -96,6 +110,8 @@ def _job_record(job: ActiveShadowEvalJob, target_type="key", target_id="key-hash max_turns=job.max_turns, max_budget=job.max_budget, ends_at=job.ends_at, + rubric=rubric or job.rubric, + rubric_version=rubric_version or job.rubric_version, ).items(): setattr(record, field, value) return record @@ -240,7 +256,15 @@ def _spend_counter(store=None): return counter, read, write -def _logger(router=None, prisma=None, jobs=(), counter_store=None, jobs_by_target=None) -> ShadowEvalLogger: +def _logger( + router=None, + prisma=None, + jobs=(), + counter_store=None, + jobs_by_target=None, + real_is_a: bool | None = None, + rubric_enabled: bool = True, +) -> ShadowEvalLogger: cache = InMemoryCache(max_size_in_memory=4, default_ttl=60) counter, read, write = _spend_counter(counter_store) funnel_events = [] @@ -251,6 +275,8 @@ def _logger(router=None, prisma=None, jobs=(), counter_store=None, jobs_by_targe job_spend_reader=read, job_spend_writer=write, funnel_recorder=lambda job_id, stage: funnel_events.append((job_id, stage)), + real_is_a_draw=(lambda: real_is_a) if real_is_a is not None else None, + rubric_enabled=lambda: rubric_enabled, ) logger._test_counter = counter logger._test_funnel = funnel_events @@ -2789,3 +2815,308 @@ class TestSameModelSkip: router.acompletion.assert_not_called() assert logger._test_funnel == [("job-1", "same_model")] + + +RUBRIC: Final = "Cites the source document and answers in under 100 words." + + +def _rubric_job( + id: str = "job-1", + router_names: tuple[str, ...] = (), + direction: Literal["forward", "reverse"] = "forward", + baseline_model: str | None = None, +) -> ActiveShadowEvalJob: + return _job( + id=id, + router_names=router_names, + direction=direction, + baseline_model=baseline_model, + rubric=RUBRIC, + rubric_version=SHADOW_EVAL_RUBRIC_VERSION, + ) + + +def _rubric_reply(grade_a: object, grade_b: object, preference: str = "A") -> str: + return json.dumps({"preference": preference, "confidence": 0.8, "grade_a": grade_a, "grade_b": grade_b}) + + +def _judge_calls(router: MagicMock) -> list[Mapping[str, object]]: + return [ + call.kwargs + for call in router.acompletion.call_args_list + if call.kwargs["metadata"].get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == SHADOW_EVAL_JUDGE_CALL_ORIGIN + ] + + +def _attempt_rows(prisma: MagicMock) -> list[Mapping[str, object]]: + return [call.kwargs["data"] for call in prisma.db.litellm_shadowevalattempt.create.await_args_list] + + +async def _sample_once(logger: ShadowEvalLogger, job: ActiveShadowEvalJob) -> None: + await logger._run_shadow_eval( + job=job, + request_id="req-1", + messages=({"role": "user", "content": "hi"},), + real_text="real answer", + real_model="claude-opus", + real_cost=0.0, + real_classifier_cost=0.0, + real_cache_hit=False, + control_tier=None, + shadow_params={}, + parent_metadata={}, + ) + + +@pytest.mark.asyncio +class TestRubricGrades: + async def test_a_preference_job_sends_the_legacy_judge_request_and_writes_the_legacy_row( + self, monkeypatch: pytest.MonkeyPatch + ): + import litellm as litellm_module + + monkeypatch.setattr(litellm_module, "completion_cost", lambda completion_response: 0.005) + prisma = _prisma() + router = _router(judge_json='{"preference": "A", "confidence": 1.7, "grade_a": 5, "grade_b": 1}') + + await _sample_once(_logger(router=router, prisma=prisma, real_is_a=False), _job()) + + (judge_call,) = _judge_calls(router) + assert judge_call["messages"][0] == {"role": "system", "content": PAIRWISE_JUDGE_SYSTEM_PROMPT} + assert judge_call["response_format"] == PAIRWISE_JUDGE_RESPONSE_FORMAT + assert "Response A:\nshadow answer\n\nResponse B:\nreal answer" in judge_call["messages"][1]["content"] + assert _attempt_rows(prisma) == [ + { + "job_id": "job-1", + "request_id": "req-1", + "router_name": "my-router", + "outcome": "shadow", + "tier": "SIMPLE", + "real_model": "claude-opus", + "shadow_model": "cheap-model", + "confidence": 1.0, + "judge_cost": 0.005, + "shadow_cost": 0.005, + "shadow_classifier_cost": 0.0, + "real_cost": 0.0, + "real_classifier_cost": 0.0, + "real_cache_hit": False, + "error": None, + } + ] + + @pytest.mark.parametrize("direction", ["forward", "reverse"]) + @pytest.mark.parametrize("real_is_a", [True, False], ids=["real-is-a", "real-is-b"]) + async def test_grades_follow_each_response_through_either_label_order(self, direction: str, real_is_a: bool): + """The same draw places the responses and unmasks the grades, and real is always the + response the caller received, including a reverse job's router response.""" + prisma = _prisma() + router = _router(judge_json=_rubric_reply(5, 2)) + job = ( + _rubric_job() + if direction == "forward" + else _rubric_job(direction="reverse", baseline_model="baseline-model") + ) + + await _sample_once(_logger(router=router, prisma=prisma, real_is_a=real_is_a), job) + + (judge_call,) = _judge_calls(router) + first, second = ("real answer", "shadow answer") if real_is_a else ("shadow answer", "real answer") + assert f"Response A:\n{first}\n\nResponse B:\n{second}" in judge_call["messages"][1]["content"] + assert judge_call["messages"][0] == {"role": "system", "content": rubric_judge_system_prompt(RUBRIC)} + assert RUBRIC in judge_call["messages"][0]["content"] + assert judge_call["response_format"] == RUBRIC_JUDGE_RESPONSE_FORMAT + (row,) = _attempt_rows(prisma) + expected = ("real", 5, 2) if real_is_a else ("shadow", 2, 5) + assert (row["outcome"], row["real_grade"], row["shadow_grade"]) == expected + assert (row["confidence"], row["error"]) == (0.8, None) + + @pytest.mark.parametrize( + ("grade_a", "grade_b", "preference", "expected"), + [ + (1, 5, "B", ("shadow", 1, 5)), + (5, 1, "A", ("real", 5, 1)), + (1, 1, "tie", ("tie", 1, 1)), + (5, 5, "tie", ("tie", 5, 5)), + (3, 3, "B", ("shadow", 3, 3)), + ], + ids=["lowest-vs-highest", "highest-vs-lowest", "both-bad", "both-good", "equal-grades-still-a-winner"], + ) + async def test_every_valid_grade_pair_is_recorded_with_an_independent_preference( + self, grade_a: int, grade_b: int, preference: str, expected: tuple[str, int, int] + ): + prisma = _prisma() + router = _router(judge_json=_rubric_reply(grade_a, grade_b, preference)) + + await _sample_once(_logger(router=router, prisma=prisma, real_is_a=True), _rubric_job()) + + (row,) = _attempt_rows(prisma) + assert (row["outcome"], row["real_grade"], row["shadow_grade"]) == expected + + @pytest.mark.parametrize( + ("reply", "invalid"), + [ + (_rubric_reply(True, 2), "grade_a:value_error"), + (_rubric_reply("4", 2), "grade_a:value_error"), + (_rubric_reply(4.0, 2), "grade_a:value_error"), + (_rubric_reply(4.5, 2), "grade_a:value_error"), + (_rubric_reply(None, 2), "grade_a:value_error"), + (_rubric_reply(0, 2), "grade_a:literal_error"), + (_rubric_reply(6, 2), "grade_a:literal_error"), + (_rubric_reply(4, -1), "grade_b:literal_error"), + ('{"preference": "A", "confidence": 0.8, "grade_a": 4}', "grade_b:missing"), + ('{"preference": "A", "confidence": 0.8}', "grade_a:missing,grade_b:missing"), + (_rubric_reply(4, 2, preference="C"), "preference:literal_error"), + ("I would give A a 4 and B a 2", "verdict:JSONDecodeError"), + ], + ids=[ + "bool", + "string", + "integral-float", + "fractional-float", + "null", + "below-range", + "above-range", + "negative", + "one-missing", + "both-missing", + "bad-preference", + "not-json", + ], + ) + async def test_a_malformed_verdict_is_an_error_row_that_keeps_the_billed_costs( + self, reply: str, invalid: str, monkeypatch: pytest.MonkeyPatch + ): + import litellm as litellm_module + + monkeypatch.setattr(litellm_module, "completion_cost", lambda completion_response: 0.007) + prisma = _prisma() + logger = _logger(router=_router(judge_json=reply), prisma=prisma, real_is_a=True) + + await _sample_once(logger, _rubric_job()) + + (row,) = _attempt_rows(prisma) + assert row["outcome"] == "error" + assert row["error"].startswith(f"unparseable rubric verdict; invalid={invalid}, finish_reason=") + assert RUBRIC not in row["error"] and reply not in row["error"] + assert row["confidence"] is None + assert "real_grade" not in row and "shadow_grade" not in row + assert (row["judge_cost"], row["shadow_cost"]) == (0.007, 0.007) + assert sum(logger._test_counter.values()) == pytest.approx(0.014) + + async def test_each_router_arm_records_the_grades_its_own_response_earned(self): + prisma = _prisma() + router = _router(sibling_router_texts={"alt-router": "alt answer"}) + arms = router.acompletion.side_effect + + async def acompletion(**kwargs): + if kwargs["metadata"].get(INTERNAL_CALL_ORIGIN_METADATA_KEY) != SHADOW_EVAL_JUDGE_CALL_ORIGIN: + return await arms(**kwargs) + shadow_grade = 2 if "alt answer" in kwargs["messages"][1]["content"] else 4 + return {"choices": [{"message": {"content": _rubric_reply(3, shadow_grade, preference="tie")}}]} + + router.acompletion = MagicMock(side_effect=acompletion) + + await _sample_once( + _logger(router=router, prisma=prisma, real_is_a=True), + _rubric_job(router_names=("my-router", "alt-router")), + ) + + assert [(row["router_name"], row["real_grade"], row["shadow_grade"]) for row in _attempt_rows(prisma)] == [ + ("my-router", 3, 4), + ("alt-router", 3, 2), + ] + + @pytest.mark.parametrize("rubric", [True, False], ids=["rubric", "preference"]) + async def test_a_failed_judge_call_records_only_what_the_job_may_persist( + self, rubric: bool, monkeypatch: pytest.MonkeyPatch + ): + """A provider error can quote the request it rejected. A rubric row keeps only the + exception type and status; a preference row keeps the message exactly as before.""" + import litellm as litellm_module + from litellm.exceptions import BadRequestError + + monkeypatch.setattr(litellm_module, "completion_cost", lambda completion_response: 0.007) + provider_error: Final = BadRequestError( + message=f"rejected prompt containing {RUBRIC} and real answer", + model="judge-model", + llm_provider="openai", + ) + router: Final = _router() + arms: Final = router.acompletion.side_effect + + async def acompletion(**kwargs: object) -> object: + metadata: Final = kwargs["metadata"] + if isinstance(metadata, Mapping) and metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == ( + SHADOW_EVAL_JUDGE_CALL_ORIGIN + ): + raise provider_error + return await arms(**kwargs) + + router.acompletion = MagicMock(side_effect=acompletion) + prisma: Final = _prisma() + + await _sample_once(_logger(router=router, prisma=prisma, real_is_a=True), _rubric_job() if rubric else _job()) + + (row,) = _attempt_rows(prisma) + expected_error: Final = ( + "rubric judge call failed; type=BadRequestError, status=400" + if rubric + else f"judge call failed: {provider_error}"[:_MAX_ERROR_CHARS] + ) + assert (row["outcome"], row["error"], row["confidence"]) == ("error", expected_error, None) + assert (row["judge_cost"], row["shadow_cost"]) == (0.0, 0.007) + assert "real_grade" not in row and "shadow_grade" not in row + + @pytest.mark.parametrize(("enabled", "sampled"), [(True, ["job-plain", "job-rubric"]), (False, ["job-plain"])]) + async def test_a_rubric_job_samples_only_while_rubric_grading_is_enabled(self, enabled: bool, sampled: list[str]): + prisma = _prisma() + logger = _logger( + router=_router(judge_json=_rubric_reply(4, 4)), + prisma=prisma, + jobs=(_job(id="job-plain"), _rubric_job(id="job-rubric")), + real_is_a=True, + rubric_enabled=enabled, + ) + + await _hook(logger, _success_kwargs()) + + assert sorted(row["job_id"] for row in _attempt_rows(prisma)) == sampled + assert logger._test_funnel == [] + + async def test_a_rubric_row_this_worker_cannot_grade_is_dropped_not_run_as_preference_only(self): + records = [ + _job_record(_job(id="job-plain")), + _job_record(_rubric_job(id="job-rubric")), + _job_record(_job(id="job-newer-contract"), rubric=RUBRIC, rubric_version=SHADOW_EVAL_RUBRIC_VERSION + 1), + _job_record(_job(id="job-no-version"), rubric=RUBRIC), + _job_record(_job(id="job-no-rubric"), rubric_version=SHADOW_EVAL_RUBRIC_VERSION), + ] + logger = ShadowEvalLogger( + router_provider=lambda: None, + prisma_provider=lambda: _prisma(jobs=records), + jobs_cache=InMemoryCache(max_size_in_memory=4, default_ttl=60), + ) + + jobs = (await logger._active_jobs())[("key", "key-hash")] + + assert {job.id: job.rubric for job in jobs} == {"job-plain": None, "job-rubric": RUBRIC} + + +def test_the_rubric_response_format_demands_both_grades_on_the_fixed_scale(): + schema = RUBRIC_JUDGE_RESPONSE_FORMAT["json_schema"]["schema"] + assert schema["required"] == ["preference", "confidence", "grade_a", "grade_b"] + assert schema["properties"]["grade_a"]["enum"] == [1, 2, 3, 4, 5] + assert schema["properties"]["grade_b"]["enum"] == [1, 2, 3, 4, 5] + + +@pytest.mark.parametrize(("value", "enabled"), [(None, False), ("false", False), ("yes", False), ("true", True)]) +def test_rubric_grading_is_off_unless_the_env_var_turns_it_on( + value: str | None, enabled: bool, monkeypatch: pytest.MonkeyPatch +): + if value is None: + monkeypatch.delenv(SHADOW_EVAL_RUBRIC_ENABLED_ENV_VAR, raising=False) + else: + monkeypatch.setenv(SHADOW_EVAL_RUBRIC_ENABLED_ENV_VAR, value) + + assert is_shadow_eval_rubric_enabled() is enabled diff --git a/tests/unit/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/unit/proxy/management_endpoints/test_auto_router_endpoints.py index a42d795429a..64cef0cee2d 100644 --- a/tests/unit/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_auto_router_endpoints.py @@ -1256,13 +1256,22 @@ class TestAutoRouterBenchmarks: from datetime import datetime, timedelta, timezone +from litellm.integrations.shadow_eval_logger import SHADOW_EVAL_RUBRIC_ENABLED_ENV_VAR from litellm.proxy.management_endpoints.auto_router_endpoints import ( get_shadow_eval_job, + get_shadow_eval_rubric_results, list_shadow_eval_jobs, start_shadow_eval, + start_shadow_eval_rubric, stop_shadow_eval_job, ) -from litellm.types.management_endpoints.auto_router_endpoints import SHADOW_EVAL_TURN_VALVE, StartShadowEvalRequest +from litellm.types.management_endpoints.auto_router_endpoints import ( + SHADOW_EVAL_RUBRIC_MAX_CHARS, + SHADOW_EVAL_RUBRIC_VERSION, + SHADOW_EVAL_TURN_VALVE, + StartShadowEvalRequest, + StartShadowEvalRubricRequest, +) VIEWER = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, api_key="sk-view", user_id="viewer") @@ -1598,6 +1607,7 @@ def _shadow_prisma( key_teams=None, known_teams=None, known_users=None, + grade_rows=None, ) -> MagicMock: """The job-table fake honours the filters it is handed, so a read that forgets stopped_at sees rows the partial index would have released, one that forgets @@ -1730,6 +1740,8 @@ def _shadow_prisma( return [{"judged_count": 10, "error_count": 2, "judge_spend": 0.031}] if "SELECT job_id AS grp" in sql: return by_leg_rows if by_leg_rows is not None else [] + if "a.real_grade" in sql: + return grade_rows if grade_rows is not None else [] if "COALESCE(a.router_name" in sql: return by_router_rows if by_router_rows is not None else [] if 'FROM "LiteLLM_ShadowEvalFunnel"' in sql: @@ -4137,3 +4149,228 @@ class TestPerSessionAverages: row: Final = TestAutoRouterBenchmarks.ROW.model_copy(update={"sessions": sessions, "turns": turns}) totals: Final = _benchmark_totals(row) assert (totals.avg_turns_per_session, totals.avg_session_seconds, totals.avg_tokens_per_session) == expected + + +RUBRIC: Final = "Cites the source document and answers in under 100 words." + + +def _rubric_start_request( + api_key_ids: tuple[str, ...] = ("key-hash",), rubric: str = RUBRIC +) -> StartShadowEvalRubricRequest: + return StartShadowEvalRubricRequest.model_validate( + {**_start_request().model_dump(exclude={"router_names"}), "api_key_ids": api_key_ids, "rubric": rubric} + ) + + +def _rubric_leg(id: str = "leg-1", target_id: str = "key-hash", router_names: tuple[str, ...] = ()) -> MagicMock: + return _leg_record( + id=id, + target_id=target_id, + router_names=router_names, + rubric=RUBRIC, + rubric_version=SHADOW_EVAL_RUBRIC_VERSION, + ) + + +def _grade_row(grp: str, real_grade: int | None, shadow_grade: int | None, turn_count: int) -> dict[str, object]: + return {"grp": grp, "real_grade": real_grade, "shadow_grade": shadow_grade, "turn_count": turn_count} + + +class TestShadowEvalRubricStart: + @pytest.mark.asyncio + @pytest.mark.parametrize("flag", [None, "false"], ids=["unset", "false"]) + async def test_is_refused_until_rubric_grading_is_enabled( + self, monkeypatch: pytest.MonkeyPatch, flag: str | None + ) -> None: + if flag is None: + monkeypatch.delenv(SHADOW_EVAL_RUBRIC_ENABLED_ENV_VAR, raising=False) + else: + monkeypatch.setenv(SHADOW_EVAL_RUBRIC_ENABLED_ENV_VAR, flag) + _configure_anthropic_sdk_judge(monkeypatch) + prisma = _shadow_prisma() + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + with pytest.raises(HTTPException) as exc: + await start_shadow_eval_rubric(_rubric_start_request(), ADMIN) + + assert exc.value.status_code == 400 + assert SHADOW_EVAL_RUBRIC_ENABLED_ENV_VAR in str(exc.value.detail) + prisma.db.litellm_shadowevaljob.create_many.assert_not_called() + + @pytest.mark.asyncio + @pytest.mark.parametrize("caller", [VIEWER, NON_ADMIN], ids=["viewer", "internal-user"]) + async def test_only_a_proxy_admin_can_start_one(self, monkeypatch: pytest.MonkeyPatch, caller) -> None: + monkeypatch.setenv(SHADOW_EVAL_RUBRIC_ENABLED_ENV_VAR, "true") + _configure_anthropic_sdk_judge(monkeypatch) + prisma = _shadow_prisma() + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + with pytest.raises(HTTPException) as exc: + await start_shadow_eval_rubric(_rubric_start_request(), caller) + + assert exc.value.status_code == 403 + prisma.db.litellm_shadowevaljob.create_many.assert_not_called() + + @pytest.mark.asyncio + async def test_stores_the_rubric_and_its_grading_version_on_every_leg(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv(SHADOW_EVAL_RUBRIC_ENABLED_ENV_VAR, "true") + _configure_anthropic_sdk_judge(monkeypatch) + prisma = _shadow_prisma() + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + response = await start_shadow_eval_rubric(_rubric_start_request(api_key_ids=("key-hash", "key-hash-2")), ADMIN) + + rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"] + assert [(row["target_id"], row["rubric"], row["rubric_version"]) for row in rows] == [ + ("key-hash", RUBRIC, SHADOW_EVAL_RUBRIC_VERSION), + ("key-hash-2", RUBRIC, SHADOW_EVAL_RUBRIC_VERSION), + ] + assert (response.job_id, response.rubric, response.rubric_version) == ( + rows[0]["group_id"], + RUBRIC, + SHADOW_EVAL_RUBRIC_VERSION, + ) + assert response.status == "running" + assert [target.target_id for target in response.targets] == ["key-hash", "key-hash-2"] + + @pytest.mark.asyncio + async def test_the_preference_start_writes_and_returns_no_rubric_fields(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv(SHADOW_EVAL_RUBRIC_ENABLED_ENV_VAR, "true") + _configure_anthropic_sdk_judge(monkeypatch) + prisma = _shadow_prisma() + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + response = await start_shadow_eval(_start_request(), ADMIN) + + (row,) = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"] + assert "rubric" not in row and "rubric_version" not in row + assert "rubric" not in response.model_dump() and "rubric_version" not in response.model_dump() + + +@pytest.mark.parametrize( + "rubric", + [None, "", " \n\t", "x" * (SHADOW_EVAL_RUBRIC_MAX_CHARS + 1)], + ids=["missing", "empty", "whitespace", "too-long"], +) +def test_a_rubric_request_needs_bounded_criteria(rubric: str | None) -> None: + payload = {**_start_request().model_dump(exclude={"router_names"}), "rubric": rubric} + if rubric is None: + payload.pop("rubric") + with pytest.raises(ValidationError): + StartShadowEvalRubricRequest.model_validate(payload) + + +def test_a_rubric_at_the_length_bound_is_kept_verbatim() -> None: + rubric = " " + "x" * (SHADOW_EVAL_RUBRIC_MAX_CHARS - 2) + "\n" + assert _rubric_start_request(rubric=rubric).rubric == rubric + + +class TestShadowEvalRubricResults: + @pytest.mark.asyncio + async def test_reports_joint_grades_per_router_and_counts_ungraded_turns_apart( + self, monkeypatch: pytest.MonkeyPatch + ): + prisma = _shadow_prisma( + legs=[ + _rubric_leg(router_names=("my-router", "alt-router", "idle-router")), + _rubric_leg( + id="leg-2", target_id="key-hash-2", router_names=("my-router", "alt-router", "idle-router") + ), + ], + grade_rows=[ + _grade_row("my-router", 5, 5, 3), + _grade_row("my-router", 1, 1, 2), + _grade_row("my-router", 5, 1, 1), + _grade_row("my-router", None, None, 4), + _grade_row("alt-router", 4, 2, 2), + _grade_row("alt-router", None, None, 1), + ], + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + response = await get_shadow_eval_rubric_results("job-1", VIEWER) + + assert response.model_dump() == { + "job_id": "job-1", + "rubric": RUBRIC, + "rubric_version": SHADOW_EVAL_RUBRIC_VERSION, + "overall": { + "group": "all", + "graded_count": 8, + "ungraded_count": 5, + "grade_distribution": ( + {"real_grade": 1, "shadow_grade": 1, "turn_count": 2}, + {"real_grade": 4, "shadow_grade": 2, "turn_count": 2}, + {"real_grade": 5, "shadow_grade": 1, "turn_count": 1}, + {"real_grade": 5, "shadow_grade": 5, "turn_count": 3}, + ), + }, + "by_router": ( + { + "group": "alt-router", + "graded_count": 2, + "ungraded_count": 1, + "grade_distribution": ({"real_grade": 4, "shadow_grade": 2, "turn_count": 2},), + }, + {"group": "idle-router", "graded_count": 0, "ungraded_count": 0, "grade_distribution": ()}, + { + "group": "my-router", + "graded_count": 6, + "ungraded_count": 4, + "grade_distribution": ( + {"real_grade": 1, "shadow_grade": 1, "turn_count": 2}, + {"real_grade": 5, "shadow_grade": 1, "turn_count": 1}, + {"real_grade": 5, "shadow_grade": 5, "turn_count": 3}, + ), + }, + ), + } + grade_query = next(call.args for call in prisma.db.query_raw.await_args_list if "a.real_grade" in call.args[0]) + assert grade_query[1] == ["leg-1", "leg-2"] + + @pytest.mark.asyncio + async def test_a_job_started_without_a_rubric_has_no_rubric_results(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(proxy_server, "prisma_client", _shadow_prisma(legs=[_leg_record()])) + + for job_id in ("job-1", "nope"): + with pytest.raises(HTTPException) as missing: + await get_shadow_eval_rubric_results(job_id, VIEWER) + assert missing.value.status_code == 404 + + @pytest.mark.asyncio + async def test_only_proxy_admin_roles_can_read_rubric_results(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(proxy_server, "prisma_client", _shadow_prisma(legs=[_rubric_leg()])) + + with pytest.raises(HTTPException) as forbidden: + await get_shadow_eval_rubric_results("job-1", NON_ADMIN) + assert forbidden.value.status_code == 403 + assert (await get_shadow_eval_rubric_results("job-1", ADMIN)).rubric == RUBRIC + + @pytest.mark.asyncio + @pytest.mark.parametrize("caller", [ADMIN, VIEWER], ids=["admin", "viewer"]) + async def test_rubric_results_without_a_database_report_it_is_not_connected( + self, monkeypatch: pytest.MonkeyPatch, caller: UserAPIKeyAuth + ): + from litellm.proxy._types import CommonProxyErrors + + monkeypatch.setattr(proxy_server, "prisma_client", None) + + with pytest.raises(HTTPException) as unavailable: + await get_shadow_eval_rubric_results("job-1", caller) + + assert (unavailable.value.status_code, unavailable.value.detail) == ( + 500, + CommonProxyErrors.db_not_connected_error.value, + ) + + @pytest.mark.asyncio + async def test_the_regular_job_detail_of_a_rubric_job_keeps_its_legacy_shape(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(proxy_server, "prisma_client", _shadow_prisma(legs=[_rubric_leg()])) + + detail = (await get_shadow_eval_job("job-1", VIEWER)).model_dump() + + assert "rubric" not in detail and "rubric_version" not in detail