mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-13 23:11:40 +00:00
feat: pre-adoption shadow eval foundation (schema, types, logger skeleton)
Core components:
- Added LiteLLM_ShadowEvalJob and LiteLLM_ShadowEvalVerdict tables to schema
- Migration: 20260807000000_add_shadow_eval_job (SQL for both tables + indexes)
- Types: StartShadowEvalRequest, GetShadowEvalJobResponse, etc. (pydantic models)
- ShadowEvalLogger: CustomLogger integration that will fire background shadow tasks
TODOs for next phase:
1. Wire config parsing (read shadow_eval from model_info or key settings)
2. Implement router call (_call_router_for_shadow) to get classifier tier + model
3. Build endpoints (POST /auto_router/shadow_eval/start, GET .../job/{id})
4. Implement verdict accumulation + per-tier result binning
5. Hook ShadowEvalLogger into proxy initialization
Still unresolved (implementation details):
- Cost estimation accuracy (judge model pricing per region)
- Sub-sampling strategy for high-traffic keys
- Job lifecycle: incremental verdict writes vs batch
- Retry logic for failed router/judge calls
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Abhimanyu Kapur
parent
8db2fbaad0
commit
5618bc93df
6 changed files with 605 additions and 1 deletions
|
|
@ -0,0 +1,47 @@
|
|||
CREATE TABLE IF NOT EXISTS "LiteLLM_ShadowEvalJob" (
|
||||
"id" TEXT NOT NULL,
|
||||
"team_id" TEXT,
|
||||
"organization_id" TEXT,
|
||||
"api_key_id" TEXT NOT NULL,
|
||||
"router_name" TEXT NOT NULL,
|
||||
"shadow_percentage" DOUBLE PRECISION NOT NULL,
|
||||
"judge_model" TEXT NOT NULL,
|
||||
"status" TEXT NOT NULL DEFAULT 'pending',
|
||||
"request_count" INTEGER NOT NULL DEFAULT 0,
|
||||
"completed_count" INTEGER NOT NULL DEFAULT 0,
|
||||
"failed_count" INTEGER NOT NULL DEFAULT 0,
|
||||
"result_json" JSONB,
|
||||
"cost_estimate" DOUBLE PRECISION,
|
||||
"cost_actual" DOUBLE PRECISION,
|
||||
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"created_by" TEXT,
|
||||
"completed_at" TIMESTAMP(3),
|
||||
|
||||
CONSTRAINT "LiteLLM_ShadowEvalJob_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalJob_team_id_status_idx" ON "LiteLLM_ShadowEvalJob"("team_id", "status");
|
||||
CREATE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalJob_api_key_id_status_idx" ON "LiteLLM_ShadowEvalJob"("api_key_id", "status");
|
||||
CREATE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalJob_created_at_idx" ON "LiteLLM_ShadowEvalJob"("created_at");
|
||||
|
||||
CREATE TABLE IF NOT EXISTS "LiteLLM_ShadowEvalVerdict" (
|
||||
"id" TEXT NOT NULL,
|
||||
"job_id" TEXT NOT NULL,
|
||||
"request_id" TEXT NOT NULL,
|
||||
"shadow_request_id" TEXT,
|
||||
"tier_classification" TEXT,
|
||||
"real_model" TEXT NOT NULL,
|
||||
"shadow_model" TEXT NOT NULL,
|
||||
"real_response_tokens" INTEGER,
|
||||
"shadow_response_tokens" INTEGER,
|
||||
"judge_preference" TEXT NOT NULL,
|
||||
"judge_confidence" DOUBLE PRECISION,
|
||||
"judge_reasoning" TEXT,
|
||||
"judge_model" TEXT NOT NULL,
|
||||
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "LiteLLM_ShadowEvalVerdict_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalVerdict_job_id_idx" ON "LiteLLM_ShadowEvalVerdict"("job_id");
|
||||
CREATE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalVerdict_request_id_idx" ON "LiteLLM_ShadowEvalVerdict"("request_id");
|
||||
|
|
@ -1444,6 +1444,67 @@ model LiteLLM_AutoRouterSession {
|
|||
@@index([last_turn_at], map: "idx_autorouter_session_last_turn")
|
||||
}
|
||||
|
||||
// Shadow Eval: Pre-adoption evaluation of the auto-router against real traffic.
|
||||
// Shadows a percentage of requests through the router, judges outputs blind,
|
||||
// and stratifies results by the router's own tier classification.
|
||||
model LiteLLM_ShadowEvalJob {
|
||||
id String @id @default(cuid())
|
||||
team_id String?
|
||||
organization_id String?
|
||||
api_key_id String // the deployment/key being shadowed
|
||||
router_name String // the auto-router config to shadow
|
||||
|
||||
shadow_percentage Float // 5.0 to 50.0
|
||||
judge_model String // claude-3-5-sonnet-20241022 or user-provided
|
||||
|
||||
status String @default("pending") // pending | running | completed | failed
|
||||
|
||||
request_count Int @default(0) // total requests on the shadowed key during the job
|
||||
completed_count Int @default(0) // verdicts written
|
||||
failed_count Int @default(0) // shadow calls or judge calls that failed
|
||||
|
||||
// Results (written incrementally or at end)
|
||||
// { groups: [ { tier: "SIMPLE", turn_count: N, real_win_rate_pct: X, shadow_win_rate_pct: Y, tie_rate_pct: Z, avg_confidence: C } ] }
|
||||
result_json Json?
|
||||
|
||||
cost_estimate Float? // estimated cost of running the judge
|
||||
cost_actual Float? // actual cost (judge calls * price)
|
||||
|
||||
created_at DateTime @default(now())
|
||||
created_by String? // user_id of the key that created this job
|
||||
completed_at DateTime?
|
||||
|
||||
@@index([team_id, status])
|
||||
@@index([api_key_id, status])
|
||||
@@index([created_at])
|
||||
}
|
||||
|
||||
// Per-turn judge verdicts (optional, for drill-down and auditing)
|
||||
model LiteLLM_ShadowEvalVerdict {
|
||||
id String @id @default(cuid())
|
||||
job_id String
|
||||
|
||||
request_id String // the original real request ID
|
||||
shadow_request_id String? // the shadow/duplicate request ID
|
||||
|
||||
tier_classification String? // "SIMPLE" | "COMPLEX" | "REASONING" or null if not classified
|
||||
real_model String // the model that actually handled the request (Opus, etc)
|
||||
shadow_model String // the model the router would have picked
|
||||
|
||||
real_response_tokens Int? // for cost calculations
|
||||
shadow_response_tokens Int?
|
||||
|
||||
judge_preference String // "real" | "shadow" | "tie"
|
||||
judge_confidence Float? // 0.0 to 1.0
|
||||
judge_reasoning String? // the judge's explanation
|
||||
judge_model String // which model did the judging
|
||||
|
||||
created_at DateTime @default(now())
|
||||
|
||||
@@index([job_id])
|
||||
@@index([request_id])
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Workflow Run Tracking
|
||||
//
|
||||
|
|
|
|||
291
litellm/integrations/shadow_eval_logger.py
Normal file
291
litellm/integrations/shadow_eval_logger.py
Normal file
|
|
@ -0,0 +1,291 @@
|
|||
"""
|
||||
Shadow Eval Logger: Duplicate requests through an auto-router, judge blind,
|
||||
report per-tier stratified win rates for pre-adoption evaluation.
|
||||
|
||||
Core flow:
|
||||
1. On every successful request, check if the deployment has shadow_eval enabled
|
||||
2. If yes, fire an async background task (non-blocking) to:
|
||||
a. Call the router on the same prompt to get the model it would have picked
|
||||
b. Call the judge to compare real response vs router-picked response (blind)
|
||||
c. Extract the router's tier classification
|
||||
d. Write a verdict row to LiteLLM_ShadowEvalVerdict
|
||||
e. Tally results in LiteLLM_ShadowEvalJob.result_json
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import random
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.types.utils import LLMResponseTypes
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from prisma import Prisma
|
||||
|
||||
from litellm.router import Router
|
||||
from litellm.types.management_endpoints.auto_router_endpoints import (
|
||||
JudgePreference,
|
||||
)
|
||||
|
||||
|
||||
_JSON_FENCE_RE: Final = re.compile(r"```(?:json)?\s*(.*?)\s*```", re.DOTALL | re.IGNORECASE)
|
||||
|
||||
# Pairwise-comparison judge prompt (blind to which is real vs shadow)
|
||||
PAIRWISE_JUDGE_SYSTEM_PROMPT = """You are an impartial quality judge. You will compare two responses to the same question.
|
||||
|
||||
The responses are labeled A and B in random order (you do not know which came from which system).
|
||||
|
||||
Your task: Determine which response is better, or if they are equivalent.
|
||||
|
||||
Criteria:
|
||||
- Correctness: Does it answer accurately?
|
||||
- Completeness: Does it include relevant context?
|
||||
- Clarity: Is it easy to understand?
|
||||
- Conciseness: Is it appropriately brief?
|
||||
|
||||
Return ONLY valid JSON in this exact format, no other text:
|
||||
{
|
||||
"preference": "A" | "B" | "tie",
|
||||
"confidence": <0.0 to 1.0>,
|
||||
"reasoning": "<one sentence explanation>"
|
||||
}"""
|
||||
|
||||
|
||||
def _parse_pairwise_verdict(raw: str) -> dict[str, Any]:
|
||||
"""Parse the judge's JSON pairwise verdict, tolerating markdown fences."""
|
||||
text = raw.strip()
|
||||
fenced: Final = _JSON_FENCE_RE.search(text)
|
||||
if fenced is not None:
|
||||
text = fenced.group(1).strip()
|
||||
parsed: object
|
||||
try:
|
||||
parsed = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
# Fallback: extract JSON object boundaries
|
||||
start: Final = text.find("{")
|
||||
end: Final = text.rfind("}")
|
||||
if start == -1 or end <= start:
|
||||
raise
|
||||
parsed = json.loads(text[start : end + 1])
|
||||
if not isinstance(parsed, dict):
|
||||
raise ValueError("judge response is not a JSON object")
|
||||
return cast(dict[str, Any], parsed)
|
||||
|
||||
|
||||
def _extract_text_from_content(content: Any) -> str:
|
||||
"""Extract plain text from a message content field (str or multimodal list)."""
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
parts: Final = []
|
||||
for part in content:
|
||||
if isinstance(part, dict) and part.get("type") == "text":
|
||||
parts.append(part.get("text", ""))
|
||||
return " ".join(parts)
|
||||
return ""
|
||||
|
||||
|
||||
class ShadowEvalLogger(CustomLogger):
|
||||
"""
|
||||
Integrations hook that fires a background task on every successful response.
|
||||
The task (if shadow_eval is enabled for the key) duplicates the request through
|
||||
an auto-router, judges the two outputs blind, and writes verdict rows.
|
||||
"""
|
||||
|
||||
def __init__(self, router: Optional["Router"] = None, prisma_client: Optional["Prisma"] = None):
|
||||
"""
|
||||
Args:
|
||||
router: LiteLLM Router instance (needed to call the auto-router)
|
||||
prisma_client: Prisma client for writing verdicts to DB
|
||||
"""
|
||||
self.router = router
|
||||
self.prisma_client = prisma_client
|
||||
|
||||
async def async_log_success_event(self, kwargs: dict, response_obj: LLMResponseTypes, start_time: Any, end_time: Any):
|
||||
"""
|
||||
Called after a successful LLM call. Fires a background task to shadow-eval if enabled.
|
||||
|
||||
Args:
|
||||
kwargs: Request data (messages, model, litellm_call_id, litellm_params, etc.)
|
||||
response_obj: The actual LLM response
|
||||
start_time: Request start time
|
||||
end_time: Request end time
|
||||
"""
|
||||
try:
|
||||
# Check if this deployment has shadow_eval enabled
|
||||
# (This would normally come from the model's config, checked here)
|
||||
metadata: Final = kwargs.get("litellm_params", {}).get("metadata", {}) or {}
|
||||
|
||||
# For now, shadow_eval config would come from:
|
||||
# - The model's model_info.shadow_eval config (read from proxy config)
|
||||
# - Or from the key's settings (read from database)
|
||||
# This is a hook point; the actual config fetching happens in the proxy layer.
|
||||
|
||||
# Fire the background task (don't block the logging return)
|
||||
asyncio.create_task(
|
||||
self._run_shadow_eval_async(
|
||||
kwargs=kwargs,
|
||||
response_obj=response_obj,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.debug(f"Failed to schedule shadow eval task: {e}")
|
||||
# Don't raise — logging hook failures should not fail the request
|
||||
|
||||
async def _run_shadow_eval_async(
|
||||
self,
|
||||
kwargs: dict,
|
||||
response_obj: LLMResponseTypes,
|
||||
start_time: Any,
|
||||
end_time: Any,
|
||||
) -> None:
|
||||
"""
|
||||
Background task: call the router, judge the outputs, write verdict.
|
||||
|
||||
This runs detached from the request, so exceptions are logged but not raised.
|
||||
"""
|
||||
try:
|
||||
# 1. Extract the real response text
|
||||
real_response_text = self._extract_response_text(response_obj)
|
||||
if not real_response_text:
|
||||
verbose_logger.debug("Shadow eval: could not extract response text, skipping")
|
||||
return
|
||||
|
||||
# 2. Get messages from the request
|
||||
messages: Final = kwargs.get("messages", [])
|
||||
if not messages:
|
||||
verbose_logger.debug("Shadow eval: no messages in request, skipping")
|
||||
return
|
||||
|
||||
# 3. Call the router to get the model it would have picked
|
||||
# (This is a stub; actual implementation would call self.router with the config)
|
||||
shadow_response_text: Final = "shadow response placeholder" # TODO: call router
|
||||
shadow_model: Final = "claude-haiku-4-5" # TODO: extract from router response
|
||||
tier_classification: Final = "SIMPLE" # TODO: extract from router response
|
||||
|
||||
# 4. Call the judge to compare (blind, randomized A/B order)
|
||||
judge_preference, judge_confidence, judge_reasoning = await self._call_judge(
|
||||
messages=messages,
|
||||
real_response=real_response_text,
|
||||
shadow_response=shadow_response_text,
|
||||
)
|
||||
|
||||
# 5. Write the verdict to the database (if prisma_client is available)
|
||||
if self.prisma_client is not None:
|
||||
# TODO: write to LiteLLM_ShadowEvalVerdict and update job counters
|
||||
verbose_logger.debug(
|
||||
f"Shadow eval verdict: {judge_preference} (confidence {judge_confidence}), tier={tier_classification}"
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.debug(f"Exception in shadow eval task: {e}", exc_info=True)
|
||||
# Don't raise — this is a background task
|
||||
|
||||
def _extract_response_text(self, response_obj: LLMResponseTypes) -> str:
|
||||
"""Extract the assistant's response text from the LLM response object."""
|
||||
if isinstance(response_obj, litellm.ModelResponse):
|
||||
try:
|
||||
return response_obj["choices"][0]["message"]["content"]
|
||||
except (KeyError, IndexError, TypeError):
|
||||
return ""
|
||||
elif isinstance(response_obj, str):
|
||||
return response_obj
|
||||
elif isinstance(response_obj, dict):
|
||||
try:
|
||||
return response_obj["choices"][0]["message"]["content"]
|
||||
except (KeyError, IndexError, TypeError):
|
||||
return ""
|
||||
return ""
|
||||
|
||||
async def _call_judge(
|
||||
self,
|
||||
messages: list[dict[str, Any]],
|
||||
real_response: str,
|
||||
shadow_response: str,
|
||||
) -> tuple["JudgePreference", float, str]:
|
||||
"""
|
||||
Call the judge model to compare two responses blindly.
|
||||
|
||||
Returns: (preference, confidence, reasoning)
|
||||
preference: "real" | "shadow" | "tie"
|
||||
confidence: 0.0 to 1.0
|
||||
reasoning: judge's explanation
|
||||
"""
|
||||
# Randomize A/B labels to cancel position bias
|
||||
is_real_first: Final = random.random() < 0.5
|
||||
response_a = real_response if is_real_first else shadow_response
|
||||
response_b = shadow_response if is_real_first else real_response
|
||||
|
||||
conversation_text: Final = "\n".join(
|
||||
f"{m.get('role', 'user').upper()}: {_extract_text_from_content(m.get('content', ''))}"
|
||||
for m in messages
|
||||
if m.get("content") is not None
|
||||
)
|
||||
|
||||
user_prompt = f"""Conversation:
|
||||
{conversation_text}
|
||||
|
||||
Response A:
|
||||
{response_a}
|
||||
|
||||
Response B:
|
||||
{response_b}
|
||||
|
||||
Which response is better?"""
|
||||
|
||||
try:
|
||||
# Call litellm.acompletion with the judge model
|
||||
response = await litellm.acompletion(
|
||||
model="claude-3-5-sonnet-20241022", # TODO: make configurable
|
||||
messages=[
|
||||
{"role": "system", "content": PAIRWISE_JUDGE_SYSTEM_PROMPT},
|
||||
{"role": "user", "content": user_prompt},
|
||||
],
|
||||
temperature=0,
|
||||
max_tokens=200,
|
||||
)
|
||||
judge_text: Final = response["choices"][0]["message"]["content"]
|
||||
verdict: Final = _parse_pairwise_verdict(judge_text)
|
||||
|
||||
# Unmask the preference (judge said A or B; we need to say "real" or "shadow")
|
||||
raw_preference: Final = verdict.get("preference", "tie").lower()
|
||||
if raw_preference == "a":
|
||||
preference: "JudgePreference" = "real" if is_real_first else "shadow"
|
||||
elif raw_preference == "b":
|
||||
preference = "shadow" if is_real_first else "real"
|
||||
else:
|
||||
preference = "tie"
|
||||
|
||||
confidence: Final = float(verdict.get("confidence", 0.5))
|
||||
reasoning: Final = str(verdict.get("reasoning", ""))
|
||||
|
||||
return preference, confidence, reasoning
|
||||
|
||||
except Exception as e:
|
||||
verbose_logger.debug(f"Judge call failed: {e}")
|
||||
raise
|
||||
|
||||
|
||||
# Placeholder for router call (to be implemented in proxy layer)
|
||||
async def _call_router_for_shadow(
|
||||
router: "Router",
|
||||
router_config_name: str,
|
||||
messages: list[dict[str, Any]],
|
||||
) -> tuple[str, str, str]:
|
||||
"""
|
||||
Call the auto-router on a prompt to determine what model it would pick.
|
||||
|
||||
Returns: (model_name, tier_classification, shadow_response_text)
|
||||
model_name: e.g. "claude-haiku-4-5"
|
||||
tier_classification: e.g. "SIMPLE", "COMPLEX", "REASONING"
|
||||
shadow_response_text: the actual model response
|
||||
"""
|
||||
# TODO: implement router call via router.acompletion with the given config
|
||||
raise NotImplementedError("_call_router_for_shadow not yet implemented")
|
||||
|
|
@ -1444,6 +1444,67 @@ model LiteLLM_AutoRouterSession {
|
|||
@@index([last_turn_at], map: "idx_autorouter_session_last_turn")
|
||||
}
|
||||
|
||||
// Shadow Eval: Pre-adoption evaluation of the auto-router against real traffic.
|
||||
// Shadows a percentage of requests through the router, judges outputs blind,
|
||||
// and stratifies results by the router's own tier classification.
|
||||
model LiteLLM_ShadowEvalJob {
|
||||
id String @id @default(cuid())
|
||||
team_id String?
|
||||
organization_id String?
|
||||
api_key_id String // the deployment/key being shadowed
|
||||
router_name String // the auto-router config to shadow
|
||||
|
||||
shadow_percentage Float // 5.0 to 50.0
|
||||
judge_model String // claude-3-5-sonnet-20241022 or user-provided
|
||||
|
||||
status String @default("pending") // pending | running | completed | failed
|
||||
|
||||
request_count Int @default(0) // total requests on the shadowed key during the job
|
||||
completed_count Int @default(0) // verdicts written
|
||||
failed_count Int @default(0) // shadow calls or judge calls that failed
|
||||
|
||||
// Results (written incrementally or at end)
|
||||
// { groups: [ { tier: "SIMPLE", turn_count: N, real_win_rate_pct: X, shadow_win_rate_pct: Y, tie_rate_pct: Z, avg_confidence: C } ] }
|
||||
result_json Json?
|
||||
|
||||
cost_estimate Float? // estimated cost of running the judge
|
||||
cost_actual Float? // actual cost (judge calls * price)
|
||||
|
||||
created_at DateTime @default(now())
|
||||
created_by String? // user_id of the key that created this job
|
||||
completed_at DateTime?
|
||||
|
||||
@@index([team_id, status])
|
||||
@@index([api_key_id, status])
|
||||
@@index([created_at])
|
||||
}
|
||||
|
||||
// Per-turn judge verdicts (optional, for drill-down and auditing)
|
||||
model LiteLLM_ShadowEvalVerdict {
|
||||
id String @id @default(cuid())
|
||||
job_id String
|
||||
|
||||
request_id String // the original real request ID
|
||||
shadow_request_id String? // the shadow/duplicate request ID
|
||||
|
||||
tier_classification String? // "SIMPLE" | "COMPLEX" | "REASONING" or null if not classified
|
||||
real_model String // the model that actually handled the request (Opus, etc)
|
||||
shadow_model String // the model the router would have picked
|
||||
|
||||
real_response_tokens Int? // for cost calculations
|
||||
shadow_response_tokens Int?
|
||||
|
||||
judge_preference String // "real" | "shadow" | "tie"
|
||||
judge_confidence Float? // 0.0 to 1.0
|
||||
judge_reasoning String? // the judge's explanation
|
||||
judge_model String // which model did the judging
|
||||
|
||||
created_at DateTime @default(now())
|
||||
|
||||
@@index([job_id])
|
||||
@@index([request_id])
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Workflow Run Tracking
|
||||
//
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
Types for auto-router management endpoints
|
||||
"""
|
||||
|
||||
from typing import Final
|
||||
from typing import Final, Literal
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
|
|
@ -130,3 +130,86 @@ class AutoRouterBenchmarksResponse(BaseModel):
|
|||
routers_in_scope: int
|
||||
totals: AutoRouterBenchmarkTotals
|
||||
groups: tuple[AutoRouterBenchmarkGroup, ...]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shadow Eval (pre-adoption): shadow a slice of a deployment's live traffic
|
||||
# through an auto-router, judge the two responses blind, and report how the
|
||||
# router would have fared without ever serving its answer to a real user.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
ShadowEvalStatus = Literal["pending", "running", "completed", "failed"]
|
||||
JudgePreference = Literal["real", "shadow", "tie"]
|
||||
|
||||
DEFAULT_SHADOW_EVAL_JUDGE_MODEL: Final[str] = "claude-3-5-sonnet-20241022"
|
||||
|
||||
|
||||
class StartShadowEvalRequest(BaseModel):
|
||||
"""Start shadowing a deployment's traffic through an auto-router for comparison."""
|
||||
|
||||
api_key_id: str = Field(description="The hashed virtual key whose traffic will be shadowed")
|
||||
router_name: str = Field(description="The auto-router config to shadow requests through")
|
||||
shadow_percentage: float = Field(
|
||||
ge=0.1,
|
||||
le=100.0,
|
||||
description="Percentage of the key's requests to duplicate through the router",
|
||||
)
|
||||
judge_model: str = Field(
|
||||
default=DEFAULT_SHADOW_EVAL_JUDGE_MODEL,
|
||||
description="Model used to blindly judge real vs. shadow responses",
|
||||
)
|
||||
team_id: str | None = Field(default=None, description="Team the shadowed key belongs to, for authorization")
|
||||
|
||||
@field_validator("shadow_percentage")
|
||||
@classmethod
|
||||
def _round_percentage(cls, value: float) -> float:
|
||||
return round(value, 2)
|
||||
|
||||
|
||||
class StartShadowEvalResponse(BaseModel):
|
||||
"""Acknowledgement that a shadow-eval job was created, with an upfront cost estimate."""
|
||||
|
||||
job_id: str
|
||||
status: ShadowEvalStatus
|
||||
estimated_request_count: int = Field(
|
||||
description="Requests expected to be shadowed, based on the key's recent request volume"
|
||||
)
|
||||
estimated_cost: float = Field(description="Estimated dollar cost of the judge calls this job will make")
|
||||
|
||||
|
||||
class ShadowEvalTierResult(BaseModel):
|
||||
"""Judge outcomes for one router-tier classification (e.g. SIMPLE, COMPLEX, REASONING)."""
|
||||
|
||||
tier: str
|
||||
turn_count: int
|
||||
real_win_rate_pct: float = Field(description="Share of judged turns where the real (control) model won")
|
||||
shadow_win_rate_pct: float = Field(description="Share of judged turns where the shadowed router's pick won")
|
||||
tie_rate_pct: float
|
||||
avg_judge_confidence: float
|
||||
|
||||
|
||||
class ShadowEvalResult(BaseModel):
|
||||
"""Stratified results of a completed (or in-progress) shadow-eval job."""
|
||||
|
||||
groups: tuple[ShadowEvalTierResult, ...]
|
||||
overall_shadow_win_rate_pct: float
|
||||
overall_tie_rate_pct: float
|
||||
|
||||
|
||||
class GetShadowEvalJobResponse(BaseModel):
|
||||
"""Status and, once available, results of a shadow-eval job."""
|
||||
|
||||
job_id: str
|
||||
status: ShadowEvalStatus
|
||||
router_name: str
|
||||
shadow_percentage: float
|
||||
request_count: int = Field(description="Total requests observed on the shadowed key since the job started")
|
||||
completed_count: int = Field(description="Verdicts written so far")
|
||||
failed_count: int = Field(description="Shadow or judge calls that errored and were skipped")
|
||||
results: ShadowEvalResult | None = Field(
|
||||
default=None, description="Present once at least one verdict has been recorded"
|
||||
)
|
||||
cost_estimate: float | None = None
|
||||
cost_actual: float | None = Field(default=None, description="Running total of judge-call spend for this job")
|
||||
created_at: str
|
||||
completed_at: str | None = None
|
||||
|
|
|
|||
|
|
@ -1444,6 +1444,67 @@ model LiteLLM_AutoRouterSession {
|
|||
@@index([last_turn_at], map: "idx_autorouter_session_last_turn")
|
||||
}
|
||||
|
||||
// Shadow Eval: Pre-adoption evaluation of the auto-router against real traffic.
|
||||
// Shadows a percentage of requests through the router, judges outputs blind,
|
||||
// and stratifies results by the router's own tier classification.
|
||||
model LiteLLM_ShadowEvalJob {
|
||||
id String @id @default(cuid())
|
||||
team_id String?
|
||||
organization_id String?
|
||||
api_key_id String // the deployment/key being shadowed
|
||||
router_name String // the auto-router config to shadow
|
||||
|
||||
shadow_percentage Float // 5.0 to 50.0
|
||||
judge_model String // claude-3-5-sonnet-20241022 or user-provided
|
||||
|
||||
status String @default("pending") // pending | running | completed | failed
|
||||
|
||||
request_count Int @default(0) // total requests on the shadowed key during the job
|
||||
completed_count Int @default(0) // verdicts written
|
||||
failed_count Int @default(0) // shadow calls or judge calls that failed
|
||||
|
||||
// Results (written incrementally or at end)
|
||||
// { groups: [ { tier: "SIMPLE", turn_count: N, real_win_rate_pct: X, shadow_win_rate_pct: Y, tie_rate_pct: Z, avg_confidence: C } ] }
|
||||
result_json Json?
|
||||
|
||||
cost_estimate Float? // estimated cost of running the judge
|
||||
cost_actual Float? // actual cost (judge calls * price)
|
||||
|
||||
created_at DateTime @default(now())
|
||||
created_by String? // user_id of the key that created this job
|
||||
completed_at DateTime?
|
||||
|
||||
@@index([team_id, status])
|
||||
@@index([api_key_id, status])
|
||||
@@index([created_at])
|
||||
}
|
||||
|
||||
// Per-turn judge verdicts (optional, for drill-down and auditing)
|
||||
model LiteLLM_ShadowEvalVerdict {
|
||||
id String @id @default(cuid())
|
||||
job_id String
|
||||
|
||||
request_id String // the original real request ID
|
||||
shadow_request_id String? // the shadow/duplicate request ID
|
||||
|
||||
tier_classification String? // "SIMPLE" | "COMPLEX" | "REASONING" or null if not classified
|
||||
real_model String // the model that actually handled the request (Opus, etc)
|
||||
shadow_model String // the model the router would have picked
|
||||
|
||||
real_response_tokens Int? // for cost calculations
|
||||
shadow_response_tokens Int?
|
||||
|
||||
judge_preference String // "real" | "shadow" | "tie"
|
||||
judge_confidence Float? // 0.0 to 1.0
|
||||
judge_reasoning String? // the judge's explanation
|
||||
judge_model String // which model did the judging
|
||||
|
||||
created_at DateTime @default(now())
|
||||
|
||||
@@index([job_id])
|
||||
@@index([request_id])
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Workflow Run Tracking
|
||||
//
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue