diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260807000000_add_shadow_eval_job/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260807000000_add_shadow_eval_job/migration.sql new file mode 100644 index 00000000000..e9a08a28eaa --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260807000000_add_shadow_eval_job/migration.sql @@ -0,0 +1,47 @@ +CREATE TABLE IF NOT EXISTS "LiteLLM_ShadowEvalJob" ( + "id" TEXT NOT NULL, + "team_id" TEXT, + "organization_id" TEXT, + "api_key_id" TEXT NOT NULL, + "router_name" TEXT NOT NULL, + "shadow_percentage" DOUBLE PRECISION NOT NULL, + "judge_model" TEXT NOT NULL, + "status" TEXT NOT NULL DEFAULT 'pending', + "request_count" INTEGER NOT NULL DEFAULT 0, + "completed_count" INTEGER NOT NULL DEFAULT 0, + "failed_count" INTEGER NOT NULL DEFAULT 0, + "result_json" JSONB, + "cost_estimate" DOUBLE PRECISION, + "cost_actual" DOUBLE PRECISION, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "created_by" TEXT, + "completed_at" TIMESTAMP(3), + + CONSTRAINT "LiteLLM_ShadowEvalJob_pkey" PRIMARY KEY ("id") +); + +CREATE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalJob_team_id_status_idx" ON "LiteLLM_ShadowEvalJob"("team_id", "status"); +CREATE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalJob_api_key_id_status_idx" ON "LiteLLM_ShadowEvalJob"("api_key_id", "status"); +CREATE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalJob_created_at_idx" ON "LiteLLM_ShadowEvalJob"("created_at"); + +CREATE TABLE IF NOT EXISTS "LiteLLM_ShadowEvalVerdict" ( + "id" TEXT NOT NULL, + "job_id" TEXT NOT NULL, + "request_id" TEXT NOT NULL, + "shadow_request_id" TEXT, + "tier_classification" TEXT, + "real_model" TEXT NOT NULL, + "shadow_model" TEXT NOT NULL, + "real_response_tokens" INTEGER, + "shadow_response_tokens" INTEGER, + "judge_preference" TEXT NOT NULL, + "judge_confidence" DOUBLE PRECISION, + "judge_reasoning" TEXT, + "judge_model" TEXT NOT NULL, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "LiteLLM_ShadowEvalVerdict_pkey" PRIMARY KEY ("id") +); + +CREATE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalVerdict_job_id_idx" ON "LiteLLM_ShadowEvalVerdict"("job_id"); +CREATE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalVerdict_request_id_idx" ON "LiteLLM_ShadowEvalVerdict"("request_id"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index b6557e3006d..2ecff64aa2e 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -1444,6 +1444,67 @@ model LiteLLM_AutoRouterSession { @@index([last_turn_at], map: "idx_autorouter_session_last_turn") } +// Shadow Eval: Pre-adoption evaluation of the auto-router against real traffic. +// Shadows a percentage of requests through the router, judges outputs blind, +// and stratifies results by the router's own tier classification. +model LiteLLM_ShadowEvalJob { + id String @id @default(cuid()) + team_id String? + organization_id String? + api_key_id String // the deployment/key being shadowed + router_name String // the auto-router config to shadow + + shadow_percentage Float // 5.0 to 50.0 + judge_model String // claude-3-5-sonnet-20241022 or user-provided + + status String @default("pending") // pending | running | completed | failed + + request_count Int @default(0) // total requests on the shadowed key during the job + completed_count Int @default(0) // verdicts written + failed_count Int @default(0) // shadow calls or judge calls that failed + + // Results (written incrementally or at end) + // { groups: [ { tier: "SIMPLE", turn_count: N, real_win_rate_pct: X, shadow_win_rate_pct: Y, tie_rate_pct: Z, avg_confidence: C } ] } + result_json Json? + + cost_estimate Float? // estimated cost of running the judge + cost_actual Float? // actual cost (judge calls * price) + + created_at DateTime @default(now()) + created_by String? // user_id of the key that created this job + completed_at DateTime? + + @@index([team_id, status]) + @@index([api_key_id, status]) + @@index([created_at]) +} + +// Per-turn judge verdicts (optional, for drill-down and auditing) +model LiteLLM_ShadowEvalVerdict { + id String @id @default(cuid()) + job_id String + + request_id String // the original real request ID + shadow_request_id String? // the shadow/duplicate request ID + + tier_classification String? // "SIMPLE" | "COMPLEX" | "REASONING" or null if not classified + real_model String // the model that actually handled the request (Opus, etc) + shadow_model String // the model the router would have picked + + real_response_tokens Int? // for cost calculations + shadow_response_tokens Int? + + judge_preference String // "real" | "shadow" | "tie" + judge_confidence Float? // 0.0 to 1.0 + judge_reasoning String? // the judge's explanation + judge_model String // which model did the judging + + created_at DateTime @default(now()) + + @@index([job_id]) + @@index([request_id]) +} + // --------------------------------------------------------------------------- // Workflow Run Tracking // diff --git a/litellm/integrations/shadow_eval_logger.py b/litellm/integrations/shadow_eval_logger.py new file mode 100644 index 00000000000..7896b838cb4 --- /dev/null +++ b/litellm/integrations/shadow_eval_logger.py @@ -0,0 +1,291 @@ +""" +Shadow Eval Logger: Duplicate requests through an auto-router, judge blind, +report per-tier stratified win rates for pre-adoption evaluation. + +Core flow: +1. On every successful request, check if the deployment has shadow_eval enabled +2. If yes, fire an async background task (non-blocking) to: + a. Call the router on the same prompt to get the model it would have picked + b. Call the judge to compare real response vs router-picked response (blind) + c. Extract the router's tier classification + d. Write a verdict row to LiteLLM_ShadowEvalVerdict + e. Tally results in LiteLLM_ShadowEvalJob.result_json +""" + +import asyncio +import json +import random +import re +from collections.abc import Callable +from datetime import datetime +from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast + +import litellm +from litellm._logging import verbose_logger +from litellm.integrations.custom_logger import CustomLogger +from litellm.types.utils import LLMResponseTypes + +if TYPE_CHECKING: + from prisma import Prisma + + from litellm.router import Router + from litellm.types.management_endpoints.auto_router_endpoints import ( + JudgePreference, + ) + + +_JSON_FENCE_RE: Final = re.compile(r"```(?:json)?\s*(.*?)\s*```", re.DOTALL | re.IGNORECASE) + +# Pairwise-comparison judge prompt (blind to which is real vs shadow) +PAIRWISE_JUDGE_SYSTEM_PROMPT = """You are an impartial quality judge. You will compare two responses to the same question. + +The responses are labeled A and B in random order (you do not know which came from which system). + +Your task: Determine which response is better, or if they are equivalent. + +Criteria: +- Correctness: Does it answer accurately? +- Completeness: Does it include relevant context? +- Clarity: Is it easy to understand? +- Conciseness: Is it appropriately brief? + +Return ONLY valid JSON in this exact format, no other text: +{ + "preference": "A" | "B" | "tie", + "confidence": <0.0 to 1.0>, + "reasoning": "" +}""" + + +def _parse_pairwise_verdict(raw: str) -> dict[str, Any]: + """Parse the judge's JSON pairwise verdict, tolerating markdown fences.""" + text = raw.strip() + fenced: Final = _JSON_FENCE_RE.search(text) + if fenced is not None: + text = fenced.group(1).strip() + parsed: object + try: + parsed = json.loads(text) + except json.JSONDecodeError: + # Fallback: extract JSON object boundaries + start: Final = text.find("{") + end: Final = text.rfind("}") + if start == -1 or end <= start: + raise + parsed = json.loads(text[start : end + 1]) + if not isinstance(parsed, dict): + raise ValueError("judge response is not a JSON object") + return cast(dict[str, Any], parsed) + + +def _extract_text_from_content(content: Any) -> str: + """Extract plain text from a message content field (str or multimodal list).""" + if isinstance(content, str): + return content + if isinstance(content, list): + parts: Final = [] + for part in content: + if isinstance(part, dict) and part.get("type") == "text": + parts.append(part.get("text", "")) + return " ".join(parts) + return "" + + +class ShadowEvalLogger(CustomLogger): + """ + Integrations hook that fires a background task on every successful response. + The task (if shadow_eval is enabled for the key) duplicates the request through + an auto-router, judges the two outputs blind, and writes verdict rows. + """ + + def __init__(self, router: Optional["Router"] = None, prisma_client: Optional["Prisma"] = None): + """ + Args: + router: LiteLLM Router instance (needed to call the auto-router) + prisma_client: Prisma client for writing verdicts to DB + """ + self.router = router + self.prisma_client = prisma_client + + async def async_log_success_event(self, kwargs: dict, response_obj: LLMResponseTypes, start_time: Any, end_time: Any): + """ + Called after a successful LLM call. Fires a background task to shadow-eval if enabled. + + Args: + kwargs: Request data (messages, model, litellm_call_id, litellm_params, etc.) + response_obj: The actual LLM response + start_time: Request start time + end_time: Request end time + """ + try: + # Check if this deployment has shadow_eval enabled + # (This would normally come from the model's config, checked here) + metadata: Final = kwargs.get("litellm_params", {}).get("metadata", {}) or {} + + # For now, shadow_eval config would come from: + # - The model's model_info.shadow_eval config (read from proxy config) + # - Or from the key's settings (read from database) + # This is a hook point; the actual config fetching happens in the proxy layer. + + # Fire the background task (don't block the logging return) + asyncio.create_task( + self._run_shadow_eval_async( + kwargs=kwargs, + response_obj=response_obj, + start_time=start_time, + end_time=end_time, + ) + ) + except Exception as e: + verbose_logger.debug(f"Failed to schedule shadow eval task: {e}") + # Don't raise — logging hook failures should not fail the request + + async def _run_shadow_eval_async( + self, + kwargs: dict, + response_obj: LLMResponseTypes, + start_time: Any, + end_time: Any, + ) -> None: + """ + Background task: call the router, judge the outputs, write verdict. + + This runs detached from the request, so exceptions are logged but not raised. + """ + try: + # 1. Extract the real response text + real_response_text = self._extract_response_text(response_obj) + if not real_response_text: + verbose_logger.debug("Shadow eval: could not extract response text, skipping") + return + + # 2. Get messages from the request + messages: Final = kwargs.get("messages", []) + if not messages: + verbose_logger.debug("Shadow eval: no messages in request, skipping") + return + + # 3. Call the router to get the model it would have picked + # (This is a stub; actual implementation would call self.router with the config) + shadow_response_text: Final = "shadow response placeholder" # TODO: call router + shadow_model: Final = "claude-haiku-4-5" # TODO: extract from router response + tier_classification: Final = "SIMPLE" # TODO: extract from router response + + # 4. Call the judge to compare (blind, randomized A/B order) + judge_preference, judge_confidence, judge_reasoning = await self._call_judge( + messages=messages, + real_response=real_response_text, + shadow_response=shadow_response_text, + ) + + # 5. Write the verdict to the database (if prisma_client is available) + if self.prisma_client is not None: + # TODO: write to LiteLLM_ShadowEvalVerdict and update job counters + verbose_logger.debug( + f"Shadow eval verdict: {judge_preference} (confidence {judge_confidence}), tier={tier_classification}" + ) + except Exception as e: + verbose_logger.debug(f"Exception in shadow eval task: {e}", exc_info=True) + # Don't raise — this is a background task + + def _extract_response_text(self, response_obj: LLMResponseTypes) -> str: + """Extract the assistant's response text from the LLM response object.""" + if isinstance(response_obj, litellm.ModelResponse): + try: + return response_obj["choices"][0]["message"]["content"] + except (KeyError, IndexError, TypeError): + return "" + elif isinstance(response_obj, str): + return response_obj + elif isinstance(response_obj, dict): + try: + return response_obj["choices"][0]["message"]["content"] + except (KeyError, IndexError, TypeError): + return "" + return "" + + async def _call_judge( + self, + messages: list[dict[str, Any]], + real_response: str, + shadow_response: str, + ) -> tuple["JudgePreference", float, str]: + """ + Call the judge model to compare two responses blindly. + + Returns: (preference, confidence, reasoning) + preference: "real" | "shadow" | "tie" + confidence: 0.0 to 1.0 + reasoning: judge's explanation + """ + # Randomize A/B labels to cancel position bias + is_real_first: Final = random.random() < 0.5 + response_a = real_response if is_real_first else shadow_response + response_b = shadow_response if is_real_first else real_response + + conversation_text: Final = "\n".join( + f"{m.get('role', 'user').upper()}: {_extract_text_from_content(m.get('content', ''))}" + for m in messages + if m.get("content") is not None + ) + + user_prompt = f"""Conversation: +{conversation_text} + +Response A: +{response_a} + +Response B: +{response_b} + +Which response is better?""" + + try: + # Call litellm.acompletion with the judge model + response = await litellm.acompletion( + model="claude-3-5-sonnet-20241022", # TODO: make configurable + messages=[ + {"role": "system", "content": PAIRWISE_JUDGE_SYSTEM_PROMPT}, + {"role": "user", "content": user_prompt}, + ], + temperature=0, + max_tokens=200, + ) + judge_text: Final = response["choices"][0]["message"]["content"] + verdict: Final = _parse_pairwise_verdict(judge_text) + + # Unmask the preference (judge said A or B; we need to say "real" or "shadow") + raw_preference: Final = verdict.get("preference", "tie").lower() + if raw_preference == "a": + preference: "JudgePreference" = "real" if is_real_first else "shadow" + elif raw_preference == "b": + preference = "shadow" if is_real_first else "real" + else: + preference = "tie" + + confidence: Final = float(verdict.get("confidence", 0.5)) + reasoning: Final = str(verdict.get("reasoning", "")) + + return preference, confidence, reasoning + + except Exception as e: + verbose_logger.debug(f"Judge call failed: {e}") + raise + + +# Placeholder for router call (to be implemented in proxy layer) +async def _call_router_for_shadow( + router: "Router", + router_config_name: str, + messages: list[dict[str, Any]], +) -> tuple[str, str, str]: + """ + Call the auto-router on a prompt to determine what model it would pick. + + Returns: (model_name, tier_classification, shadow_response_text) + model_name: e.g. "claude-haiku-4-5" + tier_classification: e.g. "SIMPLE", "COMPLEX", "REASONING" + shadow_response_text: the actual model response + """ + # TODO: implement router call via router.acompletion with the given config + raise NotImplementedError("_call_router_for_shadow not yet implemented") diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index b6557e3006d..2ecff64aa2e 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -1444,6 +1444,67 @@ model LiteLLM_AutoRouterSession { @@index([last_turn_at], map: "idx_autorouter_session_last_turn") } +// Shadow Eval: Pre-adoption evaluation of the auto-router against real traffic. +// Shadows a percentage of requests through the router, judges outputs blind, +// and stratifies results by the router's own tier classification. +model LiteLLM_ShadowEvalJob { + id String @id @default(cuid()) + team_id String? + organization_id String? + api_key_id String // the deployment/key being shadowed + router_name String // the auto-router config to shadow + + shadow_percentage Float // 5.0 to 50.0 + judge_model String // claude-3-5-sonnet-20241022 or user-provided + + status String @default("pending") // pending | running | completed | failed + + request_count Int @default(0) // total requests on the shadowed key during the job + completed_count Int @default(0) // verdicts written + failed_count Int @default(0) // shadow calls or judge calls that failed + + // Results (written incrementally or at end) + // { groups: [ { tier: "SIMPLE", turn_count: N, real_win_rate_pct: X, shadow_win_rate_pct: Y, tie_rate_pct: Z, avg_confidence: C } ] } + result_json Json? + + cost_estimate Float? // estimated cost of running the judge + cost_actual Float? // actual cost (judge calls * price) + + created_at DateTime @default(now()) + created_by String? // user_id of the key that created this job + completed_at DateTime? + + @@index([team_id, status]) + @@index([api_key_id, status]) + @@index([created_at]) +} + +// Per-turn judge verdicts (optional, for drill-down and auditing) +model LiteLLM_ShadowEvalVerdict { + id String @id @default(cuid()) + job_id String + + request_id String // the original real request ID + shadow_request_id String? // the shadow/duplicate request ID + + tier_classification String? // "SIMPLE" | "COMPLEX" | "REASONING" or null if not classified + real_model String // the model that actually handled the request (Opus, etc) + shadow_model String // the model the router would have picked + + real_response_tokens Int? // for cost calculations + shadow_response_tokens Int? + + judge_preference String // "real" | "shadow" | "tie" + judge_confidence Float? // 0.0 to 1.0 + judge_reasoning String? // the judge's explanation + judge_model String // which model did the judging + + created_at DateTime @default(now()) + + @@index([job_id]) + @@index([request_id]) +} + // --------------------------------------------------------------------------- // Workflow Run Tracking // diff --git a/litellm/types/management_endpoints/auto_router_endpoints.py b/litellm/types/management_endpoints/auto_router_endpoints.py index 6c8fb96a729..87fd5482fdc 100644 --- a/litellm/types/management_endpoints/auto_router_endpoints.py +++ b/litellm/types/management_endpoints/auto_router_endpoints.py @@ -2,7 +2,7 @@ Types for auto-router management endpoints """ -from typing import Final +from typing import Final, Literal from pydantic import BaseModel, Field, field_validator @@ -130,3 +130,86 @@ class AutoRouterBenchmarksResponse(BaseModel): routers_in_scope: int totals: AutoRouterBenchmarkTotals groups: tuple[AutoRouterBenchmarkGroup, ...] + + +# --------------------------------------------------------------------------- +# Shadow Eval (pre-adoption): shadow a slice of a deployment's live traffic +# through an auto-router, judge the two responses blind, and report how the +# router would have fared without ever serving its answer to a real user. +# --------------------------------------------------------------------------- + +ShadowEvalStatus = Literal["pending", "running", "completed", "failed"] +JudgePreference = Literal["real", "shadow", "tie"] + +DEFAULT_SHADOW_EVAL_JUDGE_MODEL: Final[str] = "claude-3-5-sonnet-20241022" + + +class StartShadowEvalRequest(BaseModel): + """Start shadowing a deployment's traffic through an auto-router for comparison.""" + + api_key_id: str = Field(description="The hashed virtual key whose traffic will be shadowed") + router_name: str = Field(description="The auto-router config to shadow requests through") + shadow_percentage: float = Field( + ge=0.1, + le=100.0, + description="Percentage of the key's requests to duplicate through the router", + ) + judge_model: str = Field( + default=DEFAULT_SHADOW_EVAL_JUDGE_MODEL, + description="Model used to blindly judge real vs. shadow responses", + ) + team_id: str | None = Field(default=None, description="Team the shadowed key belongs to, for authorization") + + @field_validator("shadow_percentage") + @classmethod + def _round_percentage(cls, value: float) -> float: + return round(value, 2) + + +class StartShadowEvalResponse(BaseModel): + """Acknowledgement that a shadow-eval job was created, with an upfront cost estimate.""" + + job_id: str + status: ShadowEvalStatus + estimated_request_count: int = Field( + description="Requests expected to be shadowed, based on the key's recent request volume" + ) + estimated_cost: float = Field(description="Estimated dollar cost of the judge calls this job will make") + + +class ShadowEvalTierResult(BaseModel): + """Judge outcomes for one router-tier classification (e.g. SIMPLE, COMPLEX, REASONING).""" + + tier: str + turn_count: int + real_win_rate_pct: float = Field(description="Share of judged turns where the real (control) model won") + shadow_win_rate_pct: float = Field(description="Share of judged turns where the shadowed router's pick won") + tie_rate_pct: float + avg_judge_confidence: float + + +class ShadowEvalResult(BaseModel): + """Stratified results of a completed (or in-progress) shadow-eval job.""" + + groups: tuple[ShadowEvalTierResult, ...] + overall_shadow_win_rate_pct: float + overall_tie_rate_pct: float + + +class GetShadowEvalJobResponse(BaseModel): + """Status and, once available, results of a shadow-eval job.""" + + job_id: str + status: ShadowEvalStatus + router_name: str + shadow_percentage: float + request_count: int = Field(description="Total requests observed on the shadowed key since the job started") + completed_count: int = Field(description="Verdicts written so far") + failed_count: int = Field(description="Shadow or judge calls that errored and were skipped") + results: ShadowEvalResult | None = Field( + default=None, description="Present once at least one verdict has been recorded" + ) + cost_estimate: float | None = None + cost_actual: float | None = Field(default=None, description="Running total of judge-call spend for this job") + created_at: str + completed_at: str | None = None diff --git a/schema.prisma b/schema.prisma index b6557e3006d..2ecff64aa2e 100644 --- a/schema.prisma +++ b/schema.prisma @@ -1444,6 +1444,67 @@ model LiteLLM_AutoRouterSession { @@index([last_turn_at], map: "idx_autorouter_session_last_turn") } +// Shadow Eval: Pre-adoption evaluation of the auto-router against real traffic. +// Shadows a percentage of requests through the router, judges outputs blind, +// and stratifies results by the router's own tier classification. +model LiteLLM_ShadowEvalJob { + id String @id @default(cuid()) + team_id String? + organization_id String? + api_key_id String // the deployment/key being shadowed + router_name String // the auto-router config to shadow + + shadow_percentage Float // 5.0 to 50.0 + judge_model String // claude-3-5-sonnet-20241022 or user-provided + + status String @default("pending") // pending | running | completed | failed + + request_count Int @default(0) // total requests on the shadowed key during the job + completed_count Int @default(0) // verdicts written + failed_count Int @default(0) // shadow calls or judge calls that failed + + // Results (written incrementally or at end) + // { groups: [ { tier: "SIMPLE", turn_count: N, real_win_rate_pct: X, shadow_win_rate_pct: Y, tie_rate_pct: Z, avg_confidence: C } ] } + result_json Json? + + cost_estimate Float? // estimated cost of running the judge + cost_actual Float? // actual cost (judge calls * price) + + created_at DateTime @default(now()) + created_by String? // user_id of the key that created this job + completed_at DateTime? + + @@index([team_id, status]) + @@index([api_key_id, status]) + @@index([created_at]) +} + +// Per-turn judge verdicts (optional, for drill-down and auditing) +model LiteLLM_ShadowEvalVerdict { + id String @id @default(cuid()) + job_id String + + request_id String // the original real request ID + shadow_request_id String? // the shadow/duplicate request ID + + tier_classification String? // "SIMPLE" | "COMPLEX" | "REASONING" or null if not classified + real_model String // the model that actually handled the request (Opus, etc) + shadow_model String // the model the router would have picked + + real_response_tokens Int? // for cost calculations + shadow_response_tokens Int? + + judge_preference String // "real" | "shadow" | "tie" + judge_confidence Float? // 0.0 to 1.0 + judge_reasoning String? // the judge's explanation + judge_model String // which model did the judging + + created_at DateTime @default(now()) + + @@index([job_id]) + @@index([request_id]) +} + // --------------------------------------------------------------------------- // Workflow Run Tracking //