mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-13 23:11:40 +00:00
feat: time-bound shadow eval jobs with a real start form
A shadow eval samples ongoing traffic, so a job without an end date keeps billing judge calls until someone remembers to stop it — and the upfront estimate silently priced exactly one week regardless. Jobs now take a duration_days (1-30, default 7): the start endpoint stamps ends_at, the estimate scales trailing volume to the requested window, and the logger completes a job past its window through the same guarded update + cache eviction path as the spend cap (generalized into _finalize_job). The existing shadow eval migration is amended in place since it has not shipped anywhere yet. The start form no longer asks anyone to paste a key hash: the key is a type-to-search combobox backed by /key/list alias substring search that submits the token, the auto-router is a filter-as-you-type combobox fed by the configured auto-router deployments, and duration is a select. Active job cards show when the job will end. The judge model field is now labelled as such, with guidance: judging two answers blind needs solid comprehension and reliable JSON, not frontier reasoning — a mid-tier model (Claude Sonnet / GPT-4o class) is recommended, nano/mini-class judges give unreliable verdicts, and frontier reasoning models add cost without changing outcomes. Same guidance mirrored into the API field description. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Abhimanyu Kapur
parent
8485a0d859
commit
9f9ae2b718
13 changed files with 503 additions and 72 deletions
|
|
@ -15,6 +15,7 @@ CREATE TABLE IF NOT EXISTS "LiteLLM_ShadowEvalJob" (
|
|||
"cost_actual" DOUBLE PRECISION NOT NULL DEFAULT 0,
|
||||
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"created_by" TEXT,
|
||||
"ends_at" TIMESTAMP(3),
|
||||
"completed_at" TIMESTAMP(3),
|
||||
|
||||
CONSTRAINT "LiteLLM_ShadowEvalJob_pkey" PRIMARY KEY ("id")
|
||||
|
|
|
|||
|
|
@ -1473,6 +1473,7 @@ model LiteLLM_ShadowEvalJob {
|
|||
|
||||
created_at DateTime @default(now())
|
||||
created_by String? // user_id of the key that created this job
|
||||
ends_at DateTime?
|
||||
completed_at DateTime?
|
||||
|
||||
@@index([team_id, status])
|
||||
|
|
|
|||
|
|
@ -19,8 +19,9 @@ Shadow and judge calls carry ``shadow_eval_internal`` metadata so this logger
|
|||
ignores its own traffic and cannot recurse. They also carry the shadowed key's
|
||||
identity metadata, so the provider spend they incur is attributed to that key
|
||||
(and its team/org/user) and counts against every budget that key is subject to,
|
||||
including the global proxy budget. A job additionally stops itself once its judge
|
||||
spend reaches a multiple of the estimate quoted when it started.
|
||||
including the global proxy budget. A job additionally stops itself once its
|
||||
sampling window (``ends_at``) closes or its judge spend reaches a multiple of
|
||||
the estimate quoted when it started.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
|
@ -173,6 +174,24 @@ class ActiveShadowEvalJob:
|
|||
status: str
|
||||
cost_estimate: float | None = None
|
||||
cost_actual: float = 0.0
|
||||
ends_at: datetime | None = None
|
||||
|
||||
|
||||
def _as_utc(value: object) -> datetime | None:
|
||||
if not isinstance(value, datetime):
|
||||
return None
|
||||
return value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value
|
||||
|
||||
|
||||
def _job_is_past_its_end(job: ActiveShadowEvalJob) -> bool:
|
||||
"""Whether the job's sampling window has closed.
|
||||
|
||||
An eval is a measurement, not a service: it prices a fixed window at start
|
||||
time, so sampling past ends_at would bill traffic the estimate never covered.
|
||||
Jobs created before durations existed have no ends_at and only stop by hand
|
||||
or via the spend cap.
|
||||
"""
|
||||
return job.ends_at is not None and datetime.now(timezone.utc) >= job.ends_at
|
||||
|
||||
|
||||
def _job_is_over_spend_cap(job: ActiveShadowEvalJob) -> bool:
|
||||
|
|
@ -238,8 +257,14 @@ class ShadowEvalLogger(CustomLogger):
|
|||
job: Final = await self._get_active_job(api_key_hash)
|
||||
if job is None:
|
||||
return
|
||||
if _job_is_past_its_end(job):
|
||||
await self._finalize_job(job, "reached its scheduled end")
|
||||
return
|
||||
if _job_is_over_spend_cap(job):
|
||||
await self._stop_job_over_spend_cap(job)
|
||||
await self._finalize_job(
|
||||
job,
|
||||
f"spend ${job.cost_actual:.4f} reached the cap for its ${job.cost_estimate or 0.0:.4f} estimate",
|
||||
)
|
||||
return
|
||||
request_id: Final = payload.get("id") or ""
|
||||
if not request_id:
|
||||
|
|
@ -305,6 +330,7 @@ class ShadowEvalLogger(CustomLogger):
|
|||
status=str(record.status),
|
||||
cost_estimate=float(record.cost_estimate) if record.cost_estimate is not None else None,
|
||||
cost_actual=float(record.cost_actual or 0.0),
|
||||
ends_at=_as_utc(getattr(record, "ends_at", None)),
|
||||
)
|
||||
if record is not None
|
||||
else None
|
||||
|
|
@ -315,11 +341,11 @@ class ShadowEvalLogger(CustomLogger):
|
|||
self._job_cache[api_key_hash] = (now, job)
|
||||
return job
|
||||
|
||||
async def _stop_job_over_spend_cap(self, job: ActiveShadowEvalJob) -> None:
|
||||
"""Flip a runaway job to completed, keeping the verdicts it already produced.
|
||||
async def _finalize_job(self, job: ActiveShadowEvalJob, reason: str) -> None:
|
||||
"""Flip a finished job to completed, keeping the verdicts it already produced.
|
||||
|
||||
Guarded on the job still being pending/running so two pods breaching the cap on
|
||||
the same job cannot resurrect one an admin stopped in between.
|
||||
Guarded on the job still being pending/running so two pods finishing the
|
||||
same job cannot resurrect one an admin stopped in between.
|
||||
"""
|
||||
prisma: Final = self._prisma_provider()
|
||||
if prisma is None:
|
||||
|
|
@ -327,12 +353,7 @@ class ShadowEvalLogger(CustomLogger):
|
|||
self._job_cache = { # mutable-ok: TTL cache
|
||||
k: v for k, v in self._job_cache.items() if v[1] is None or v[1].id != job.id
|
||||
}
|
||||
verbose_logger.info(
|
||||
"shadow_eval: stopping job %s, spend $%.4f reached the cap for its $%.4f estimate",
|
||||
job.id,
|
||||
job.cost_actual,
|
||||
job.cost_estimate or 0.0,
|
||||
)
|
||||
verbose_logger.info("shadow_eval: stopping job %s: %s", job.id, reason)
|
||||
try:
|
||||
await prisma.db.litellm_shadowevaljob.update_many(
|
||||
where={ # mutable-ok: Prisma filter
|
||||
|
|
@ -345,7 +366,7 @@ class ShadowEvalLogger(CustomLogger):
|
|||
},
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # logging hooks must never fail the request
|
||||
verbose_logger.debug("shadow_eval: failed to stop over-cap job %s: %s", job.id, e)
|
||||
verbose_logger.debug("shadow_eval: failed to stop job %s: %s", job.id, e)
|
||||
|
||||
async def _flush_seen_counts(self) -> None:
|
||||
prisma: Final = self._prisma_provider()
|
||||
|
|
|
|||
|
|
@ -607,6 +607,7 @@ class _ShadowEvalJobRow(BaseModel):
|
|||
cost_estimate: float | None = None
|
||||
cost_actual: float = 0.0
|
||||
created_at: datetime
|
||||
ends_at: datetime | None = None
|
||||
completed_at: datetime | None = None
|
||||
|
||||
|
||||
|
|
@ -629,6 +630,7 @@ def _job_to_response(record: object, results: ShadowEvalResult | None) -> GetSha
|
|||
cost_estimate=row.cost_estimate,
|
||||
cost_actual=row.cost_actual,
|
||||
created_at=row.created_at.isoformat(),
|
||||
ends_at=row.ends_at.isoformat() if row.ends_at else None,
|
||||
completed_at=row.completed_at.isoformat() if row.completed_at else None,
|
||||
)
|
||||
|
||||
|
|
@ -649,10 +651,11 @@ async def start_shadow_eval(
|
|||
traffic through an auto-router, judge real vs. shadow responses blind, and
|
||||
stratify win rates by the router's tier classification.
|
||||
|
||||
The shadow responses are never served to users. The job stays active until
|
||||
stopped via /auto_router/shadow_eval/{job_id}/stop. Judge calls bill to the
|
||||
proxy; the estimate returned here prices them from the key's trailing
|
||||
request volume.
|
||||
The shadow responses are never served to users. The job samples traffic for
|
||||
duration_days (or until stopped via /auto_router/shadow_eval/{job_id}/stop),
|
||||
then completes itself. Judge calls bill to the shadowed key; the estimate
|
||||
returned here prices them from the key's trailing request volume scaled to
|
||||
the requested duration.
|
||||
"""
|
||||
from litellm.proxy.proxy_server import llm_router, prisma_client
|
||||
|
||||
|
|
@ -684,9 +687,12 @@ async def start_shadow_eval(
|
|||
"startTime": {"gte": lookback_start}, # mutable-ok: Prisma filter
|
||||
},
|
||||
)
|
||||
weekly_sampled: Final = int(recent_requests * data.shadow_percentage / 100.0)
|
||||
sampled: Final = int(
|
||||
recent_requests * (data.duration_days / _ESTIMATE_LOOKBACK_DAYS) * data.shadow_percentage / 100.0
|
||||
)
|
||||
per_call: Final = _estimate_judge_cost_per_call(data.judge_model)
|
||||
estimated_cost: Final = round(weekly_sampled * per_call, 2)
|
||||
estimated_cost: Final = round(sampled * per_call, 2)
|
||||
ends_at: Final = datetime.now(timezone.utc) + timedelta(days=data.duration_days)
|
||||
|
||||
job: Final = await prisma_client.db.litellm_shadowevaljob.create(
|
||||
data={ # mutable-ok: Prisma payload
|
||||
|
|
@ -698,12 +704,13 @@ async def start_shadow_eval(
|
|||
"status": "pending",
|
||||
"cost_estimate": estimated_cost,
|
||||
"created_by": user_api_key_dict.user_id,
|
||||
"ends_at": ends_at,
|
||||
}
|
||||
)
|
||||
return StartShadowEvalResponse(
|
||||
job_id=job.id,
|
||||
status="pending",
|
||||
estimated_request_count=weekly_sampled,
|
||||
estimated_request_count=sampled,
|
||||
estimated_cost=estimated_cost,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1473,6 +1473,7 @@ model LiteLLM_ShadowEvalJob {
|
|||
|
||||
created_at DateTime @default(now())
|
||||
created_by String? // user_id of the key that created this job
|
||||
ends_at DateTime?
|
||||
completed_at DateTime?
|
||||
|
||||
@@index([team_id, status])
|
||||
|
|
|
|||
|
|
@ -172,7 +172,17 @@ class StartShadowEvalRequest(BaseModel):
|
|||
)
|
||||
judge_model: str = Field(
|
||||
default=DEFAULT_SHADOW_EVAL_JUDGE_MODEL,
|
||||
description="Model used to blindly judge real vs. shadow responses",
|
||||
description=(
|
||||
"Model used to blindly judge real vs. shadow responses. The judge only compares two answers, so a "
|
||||
"mid-tier model (Claude Sonnet or GPT-4o class) is the sweet spot: small/nano-class models produce "
|
||||
"unreliable or malformed verdicts, while frontier reasoning models add cost without changing outcomes."
|
||||
),
|
||||
)
|
||||
duration_days: int = Field(
|
||||
default=7,
|
||||
ge=1,
|
||||
le=30,
|
||||
description="How many days the job samples traffic before stopping itself",
|
||||
)
|
||||
team_id: str | None = Field(default=None, description="Team the shadowed key belongs to, for authorization")
|
||||
|
||||
|
|
@ -230,4 +240,7 @@ class GetShadowEvalJobResponse(BaseModel):
|
|||
cost_estimate: float | None = None
|
||||
cost_actual: float = Field(default=0.0, description="Running total of judge-call spend for this job")
|
||||
created_at: str
|
||||
ends_at: str | None = Field(
|
||||
default=None, description="When the job stops sampling on its own; null for jobs started before durations"
|
||||
)
|
||||
completed_at: str | None = None
|
||||
|
|
|
|||
|
|
@ -1473,6 +1473,7 @@ model LiteLLM_ShadowEvalJob {
|
|||
|
||||
created_at DateTime @default(now())
|
||||
created_by String? // user_id of the key that created this job
|
||||
ends_at DateTime?
|
||||
completed_at DateTime?
|
||||
|
||||
@@index([team_id, status])
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
import asyncio
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
|
@ -601,6 +602,92 @@ class TestPerJobSpendCap:
|
|||
assert prisma.db.litellm_shadowevaljob.update_many.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
class TestJobsStopAtTheirScheduledEnd:
|
||||
"""A shadow eval samples ongoing traffic, so a job whose window has closed must
|
||||
stop billing judge calls even if nobody remembers to stop it by hand."""
|
||||
|
||||
@staticmethod
|
||||
def _job(ends_at):
|
||||
return ActiveShadowEvalJob(
|
||||
id="j1",
|
||||
router_name="r",
|
||||
shadow_percentage=100.0,
|
||||
judge_model="m",
|
||||
status="running",
|
||||
cost_estimate=10.0,
|
||||
cost_actual=0.0,
|
||||
ends_at=ends_at,
|
||||
)
|
||||
|
||||
async def test_job_past_its_end_stops_sampling_and_completes(self):
|
||||
logger, prisma, router = _logger_with_mocks(self._job(datetime.now(timezone.utc) - timedelta(seconds=1)))
|
||||
prisma.db.litellm_shadowevaljob.update_many = AsyncMock()
|
||||
logger._run_shadow_eval = AsyncMock()
|
||||
|
||||
await logger.async_log_success_event(TestPerJobSpendCap._success_kwargs(), MagicMock(), None, None)
|
||||
await asyncio.sleep(0)
|
||||
|
||||
logger._run_shadow_eval.assert_not_awaited()
|
||||
router.acompletion.assert_not_called()
|
||||
call_kwargs = prisma.db.litellm_shadowevaljob.update_many.call_args.kwargs
|
||||
assert call_kwargs["where"]["id"] == "j1"
|
||||
assert set(call_kwargs["where"]["status"]["in"]) == {"pending", "running"}
|
||||
assert call_kwargs["data"]["status"] == "completed"
|
||||
|
||||
async def test_job_inside_its_window_keeps_evaluating(self):
|
||||
logger, prisma, _ = _logger_with_mocks(self._job(datetime.now(timezone.utc) + timedelta(hours=1)))
|
||||
prisma.db.litellm_shadowevaljob.update_many = AsyncMock()
|
||||
logger._run_shadow_eval = AsyncMock()
|
||||
|
||||
await logger.async_log_success_event(TestPerJobSpendCap._success_kwargs(), MagicMock(), None, None)
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
logger._run_shadow_eval.assert_awaited_once()
|
||||
prisma.db.litellm_shadowevaljob.update_many.assert_not_awaited()
|
||||
|
||||
async def test_job_without_an_end_is_not_expired(self):
|
||||
logger, prisma, _ = _logger_with_mocks(self._job(None))
|
||||
prisma.db.litellm_shadowevaljob.update_many = AsyncMock()
|
||||
logger._run_shadow_eval = AsyncMock()
|
||||
|
||||
await logger.async_log_success_event(TestPerJobSpendCap._success_kwargs(), MagicMock(), None, None)
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
logger._run_shadow_eval.assert_awaited_once()
|
||||
|
||||
async def test_expiry_evicts_the_cached_job_so_later_requests_do_not_rewrite_it(self):
|
||||
logger, prisma, _ = _logger_with_mocks(self._job(datetime.now(timezone.utc) - timedelta(seconds=1)))
|
||||
prisma.db.litellm_shadowevaljob.update_many = AsyncMock()
|
||||
prisma.db.litellm_shadowevaljob.find_first = AsyncMock(return_value=None)
|
||||
|
||||
await logger.async_log_success_event(TestPerJobSpendCap._success_kwargs(), MagicMock(), None, None)
|
||||
await logger.async_log_success_event(TestPerJobSpendCap._success_kwargs(), MagicMock(), None, None)
|
||||
|
||||
assert prisma.db.litellm_shadowevaljob.update_many.await_count == 1
|
||||
|
||||
async def test_naive_db_datetime_is_treated_as_utc(self):
|
||||
logger, prisma, _ = _logger_with_mocks()
|
||||
record = MagicMock()
|
||||
record.id = "j1"
|
||||
record.router_name = "r"
|
||||
record.shadow_percentage = 100.0
|
||||
record.judge_model = "m"
|
||||
record.status = "running"
|
||||
record.cost_estimate = 10.0
|
||||
record.cost_actual = 0.0
|
||||
record.ends_at = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(seconds=1)
|
||||
prisma.db.litellm_shadowevaljob.find_first = AsyncMock(return_value=record)
|
||||
|
||||
job = await logger._get_active_job("key-hash")
|
||||
|
||||
assert job is not None
|
||||
assert job.ends_at is not None and job.ends_at.tzinfo is not None
|
||||
from litellm.integrations.shadow_eval_logger import _job_is_past_its_end
|
||||
|
||||
assert _job_is_past_its_end(job)
|
||||
|
||||
|
||||
class TestExtractResponseText:
|
||||
def test_dict_response(self):
|
||||
resp = {"choices": [{"message": {"content": "hello"}}]}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@ Unit tests for auto router management endpoints
|
|||
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
|
@ -21,10 +23,11 @@ from litellm.proxy.management_endpoints.auto_router_endpoints import (
|
|||
preview_auto_router_routing,
|
||||
)
|
||||
from litellm.router import Router
|
||||
from litellm.types.utils import Choices, Message, ModelResponse
|
||||
from litellm.types.management_endpoints.auto_router_endpoints import (
|
||||
AutoRouterRoutingTestRequest,
|
||||
StartShadowEvalRequest,
|
||||
)
|
||||
from litellm.types.utils import Choices, Message, ModelResponse
|
||||
|
||||
ADMIN = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-test", user_id="admin")
|
||||
|
||||
|
|
@ -55,7 +58,7 @@ def _request(prompt: str, **config_overrides: object) -> AutoRouterRoutingTestRe
|
|||
|
||||
|
||||
async def _route(prompt: str, monkeypatch: pytest.MonkeyPatch, **config_overrides: object):
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
monkeypatch.setattr(proxy_server, "llm_router", _router())
|
||||
return await preview_auto_router_routing(
|
||||
|
|
@ -119,7 +122,7 @@ async def test_tier_model_missing_from_the_proxy_is_reported(monkeypatch: pytest
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_llm_classifier_call_is_billed_to_the_calling_key(monkeypatch: pytest.MonkeyPatch):
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
router = _router()
|
||||
calls: list[dict] = []
|
||||
|
|
@ -164,7 +167,7 @@ async def test_llm_classifier_call_is_billed_to_the_calling_key(monkeypatch: pyt
|
|||
async def test_a_key_that_cannot_call_the_classifier_model_is_rejected_before_it_is_called(
|
||||
monkeypatch: pytest.MonkeyPatch, config_overrides: dict
|
||||
):
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
router = _router()
|
||||
calls: list[dict] = []
|
||||
|
|
@ -194,7 +197,7 @@ async def test_a_key_that_cannot_call_the_classifier_model_is_rejected_before_it
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_key_over_its_budget_cannot_run_a_classifier_config(monkeypatch: pytest.MonkeyPatch):
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
router = _router()
|
||||
calls: list[dict] = []
|
||||
|
|
@ -228,7 +231,7 @@ async def test_a_key_over_its_budget_cannot_run_a_classifier_config(monkeypatch:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_heuristic_config_does_not_need_a_budget(monkeypatch: pytest.MonkeyPatch):
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
monkeypatch.setattr(proxy_server, "llm_router", _router())
|
||||
|
||||
|
|
@ -249,7 +252,7 @@ async def test_a_heuristic_config_does_not_need_a_budget(monkeypatch: pytest.Mon
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_llm_router_on_the_proxy_is_a_500(monkeypatch: pytest.MonkeyPatch):
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
monkeypatch.setattr(proxy_server, "llm_router", None)
|
||||
|
||||
|
|
@ -261,7 +264,7 @@ async def test_no_llm_router_on_the_proxy_is_a_500(monkeypatch: pytest.MonkeyPat
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_admin_without_a_team_is_rejected(monkeypatch: pytest.MonkeyPatch):
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
monkeypatch.setattr(proxy_server, "llm_router", _router())
|
||||
|
||||
|
|
@ -543,3 +546,95 @@ class TestJudgeCostEstimate:
|
|||
)
|
||||
)
|
||||
assert _estimate_judge_cost_per_call(self.JUDGE_MODEL) > stale
|
||||
|
||||
|
||||
class TestShadowEvalJobsAreTimeBound:
|
||||
"""A shadow eval samples ongoing traffic, so without an end date a forgotten job
|
||||
would keep billing judge calls indefinitely. Every job gets an ends_at, and the
|
||||
upfront estimate must price the requested window, not always one week."""
|
||||
|
||||
@staticmethod
|
||||
def _start_request(**overrides: object) -> StartShadowEvalRequest:
|
||||
return StartShadowEvalRequest.model_validate(
|
||||
{
|
||||
"api_key_id": "hashed-key",
|
||||
"router_name": "claude-auto",
|
||||
"shadow_percentage": 10.0,
|
||||
**overrides,
|
||||
}
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _proxy_mocks(monkeypatch: pytest.MonkeyPatch, recent_requests: int) -> MagicMock:
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
router = MagicMock()
|
||||
router.auto_routers = {}
|
||||
router.complexity_routers = {"claude-auto": [MagicMock()]}
|
||||
router.adaptive_routers = {}
|
||||
router.quality_routers = {}
|
||||
prisma = MagicMock()
|
||||
prisma.db.litellm_spendlogs.count = AsyncMock(return_value=recent_requests)
|
||||
prisma.db.litellm_shadowevaljob.find_first = AsyncMock(return_value=None)
|
||||
created = MagicMock()
|
||||
created.id = "job-1"
|
||||
prisma.db.litellm_shadowevaljob.create = AsyncMock(return_value=created)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
return prisma
|
||||
|
||||
def test_duration_defaults_to_a_week_and_rejects_zero_and_over_a_month(self):
|
||||
assert self._start_request().duration_days == 7
|
||||
with pytest.raises(ValidationError):
|
||||
self._start_request(duration_days=0)
|
||||
with pytest.raises(ValidationError):
|
||||
self._start_request(duration_days=31)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_job_is_created_with_ends_at_duration_days_from_now(self, monkeypatch: pytest.MonkeyPatch):
|
||||
from litellm.proxy.management_endpoints.auto_router_endpoints import start_shadow_eval
|
||||
|
||||
prisma = self._proxy_mocks(monkeypatch, recent_requests=700)
|
||||
before = datetime.now(timezone.utc)
|
||||
|
||||
await start_shadow_eval(self._start_request(duration_days=3), ADMIN)
|
||||
|
||||
after = datetime.now(timezone.utc)
|
||||
ends_at = prisma.db.litellm_shadowevaljob.create.call_args.kwargs["data"]["ends_at"]
|
||||
assert before + timedelta(days=3) <= ends_at <= after + timedelta(days=3)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_estimate_scales_with_duration_not_always_a_week(self, monkeypatch: pytest.MonkeyPatch):
|
||||
from litellm.proxy.management_endpoints.auto_router_endpoints import start_shadow_eval
|
||||
|
||||
self._proxy_mocks(monkeypatch, recent_requests=7000)
|
||||
one_day = await start_shadow_eval(self._start_request(duration_days=1), ADMIN)
|
||||
|
||||
self._proxy_mocks(monkeypatch, recent_requests=7000)
|
||||
two_weeks = await start_shadow_eval(self._start_request(duration_days=14), ADMIN)
|
||||
|
||||
assert one_day.estimated_request_count == 100
|
||||
assert two_weeks.estimated_request_count == 1400
|
||||
assert two_weeks.estimated_cost == pytest.approx(one_day.estimated_cost * 14, rel=0.02)
|
||||
|
||||
def test_response_surfaces_ends_at(self):
|
||||
from litellm.proxy.management_endpoints.auto_router_endpoints import _job_to_response
|
||||
|
||||
fields = {
|
||||
"id": "job-1",
|
||||
"status": "running",
|
||||
"router_name": "claude-auto",
|
||||
"api_key_id": "hashed-key",
|
||||
"team_id": None,
|
||||
"shadow_percentage": 10.0,
|
||||
"request_count": 0,
|
||||
"completed_count": 0,
|
||||
"failed_count": 0,
|
||||
"cost_estimate": 1.0,
|
||||
"cost_actual": 0.0,
|
||||
"created_at": datetime(2026, 8, 1, tzinfo=timezone.utc),
|
||||
"ends_at": datetime(2026, 8, 8, tzinfo=timezone.utc),
|
||||
"completed_at": None,
|
||||
}
|
||||
response = _job_to_response(type("Row", (), fields)(), None)
|
||||
assert response.ends_at == "2026-08-08T00:00:00+00:00"
|
||||
|
|
|
|||
|
|
@ -10,6 +10,30 @@ vi.mock("./useShadowEval", () => ({
|
|||
useStopShadowEval: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/app/(dashboard)/hooks/keys/useKeys", () => ({
|
||||
useKeys: vi.fn(() => ({
|
||||
data: {
|
||||
keys: [
|
||||
{ token: "hash-alpha", token_id: "id-1", key_name: "sk-...alpha", key_alias: "prod-alpha" },
|
||||
{ token: "hash-beta", token_id: "id-2", key_name: "sk-...beta", key_alias: "staging-beta" },
|
||||
],
|
||||
total_count: 2,
|
||||
current_page: 1,
|
||||
total_pages: 1,
|
||||
},
|
||||
isPending: false,
|
||||
})),
|
||||
}));
|
||||
|
||||
vi.mock("@/app/(dashboard)/hooks/models/useModels", () => ({
|
||||
useAutoRouters: vi.fn(() => ({
|
||||
data: [
|
||||
{ model_name: "claude-auto", litellm_params: { model: "auto_router/claude-auto" } },
|
||||
{ model_name: "gpt-auto", litellm_params: { model: "auto_router/gpt-auto" } },
|
||||
],
|
||||
})),
|
||||
}));
|
||||
|
||||
import ShadowEvalSection from "./ShadowEvalSection";
|
||||
import {
|
||||
useShadowEvalJob,
|
||||
|
|
@ -52,7 +76,10 @@ const job = (overrides: Partial<ShadowEvalJob> = {}): ShadowEvalJob => ({
|
|||
cost_estimate: 45.0,
|
||||
cost_actual: 3.21,
|
||||
created_at: "2026-08-07T00:00:00Z",
|
||||
ends_at: null,
|
||||
completed_at: null,
|
||||
api_key_id: "hashed-key-abc",
|
||||
team_id: null,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
|
|
@ -68,12 +95,13 @@ const mockHooks = ({
|
|||
vi.mocked(useShadowEvalJobs).mockReturnValue({ data: jobs, error: null } as unknown as ReturnType<
|
||||
typeof useShadowEvalJobs
|
||||
>);
|
||||
vi.mocked(useShadowEvalJob).mockImplementation(
|
||||
(_token, jobId) =>
|
||||
({
|
||||
data: detailsById ? (jobId ? detailsById[jobId] : undefined) : detail,
|
||||
}) as unknown as ReturnType<typeof useShadowEvalJob>,
|
||||
);
|
||||
vi.mocked(useShadowEvalJob).mockImplementation((_token, jobId) => {
|
||||
let data = detail;
|
||||
if (detailsById) {
|
||||
data = jobId ? detailsById[jobId] : undefined;
|
||||
}
|
||||
return { data } as unknown as ReturnType<typeof useShadowEvalJob>;
|
||||
});
|
||||
vi.mocked(useStartShadowEval).mockReturnValue({
|
||||
mutate: vi.fn(),
|
||||
isPending: false,
|
||||
|
|
@ -131,16 +159,70 @@ describe("ShadowEvalSection", () => {
|
|||
expect(screen.getByText("Start a shadow eval")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("starts a job from picked key + router with duration, submitting the key token not its alias", async () => {
|
||||
const user = userEvent.setup();
|
||||
const mutate = vi.fn();
|
||||
mockHooks({ jobs: [] });
|
||||
vi.mocked(useStartShadowEval).mockReturnValue({ mutate, isPending: false, error: null } as unknown as ReturnType<
|
||||
typeof useStartShadowEval
|
||||
>);
|
||||
render(<ShadowEvalSection accessToken="token" />);
|
||||
|
||||
await user.click(screen.getByPlaceholderText("Search keys by alias"));
|
||||
await user.click(await screen.findByText("prod-alpha"));
|
||||
await user.click(screen.getByPlaceholderText("Select an auto-router"));
|
||||
await user.click(await screen.findByText("gpt-auto"));
|
||||
await user.click(screen.getByText("Start shadow eval"));
|
||||
|
||||
const expectedBody = {
|
||||
api_key_id: "hash-alpha",
|
||||
router_name: "gpt-auto",
|
||||
shadow_percentage: 10,
|
||||
duration_days: 7,
|
||||
judge_model: "anthropic/claude-sonnet-5",
|
||||
};
|
||||
expect(mutate).toHaveBeenCalledWith({ body: expectedBody });
|
||||
});
|
||||
|
||||
it("filters the key list as you type via the server-side alias search", async () => {
|
||||
const user = userEvent.setup();
|
||||
const { useKeys } = await import("@/app/(dashboard)/hooks/keys/useKeys");
|
||||
mockHooks({ jobs: [] });
|
||||
render(<ShadowEvalSection accessToken="token" />);
|
||||
|
||||
await user.type(screen.getByPlaceholderText("Search keys by alias"), "prod");
|
||||
|
||||
await vi.waitFor(() => {
|
||||
const calls = vi.mocked(useKeys).mock.calls;
|
||||
expect(calls[calls.length - 1][2]).toMatchObject({ selectedKeyAlias: "prod" });
|
||||
});
|
||||
});
|
||||
|
||||
it("explains what the judge model is for and recommends a tier", () => {
|
||||
mockHooks({ jobs: [] });
|
||||
render(<ShadowEvalSection accessToken="token" />);
|
||||
expect(screen.getByText("Judge model")).toBeInTheDocument();
|
||||
expect(screen.getByText(/mid-tier model \(Claude Sonnet or GPT-4o class\)/)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("shows when an active job will end", () => {
|
||||
const j = job({ ends_at: new Date(Date.now() + 3 * 86_400_000).toISOString() });
|
||||
mockHooks({ jobs: [j], detail: j });
|
||||
render(<ShadowEvalSection accessToken="token" />);
|
||||
expect(screen.getByText(/ends in 3 days/)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("keeps an older populated job reachable when the newest job has no verdicts", async () => {
|
||||
const user = userEvent.setup();
|
||||
const empty = job({
|
||||
const emptyOverrides: Partial<ShadowEvalJob> = {
|
||||
job_id: "job-new",
|
||||
status: "pending",
|
||||
completed_count: 0,
|
||||
failed_count: 0,
|
||||
results: null,
|
||||
cost_actual: 0,
|
||||
});
|
||||
};
|
||||
const empty = job(emptyOverrides);
|
||||
const older = job({ job_id: "job-old", status: "completed" });
|
||||
mockHooks({
|
||||
jobs: [empty, older],
|
||||
|
|
|
|||
|
|
@ -2,10 +2,16 @@
|
|||
|
||||
import React, { useMemo, useState } from "react";
|
||||
|
||||
import { useKeys } from "@/app/(dashboard)/hooks/keys/useKeys";
|
||||
import { useAutoRouters } from "@/app/(dashboard)/hooks/models/useModels";
|
||||
import { PaginatedSearchSelect } from "@/components/shared/PaginatedSearchSelect";
|
||||
import { SearchSelect, type SearchSelectOption } from "@/components/shared/SearchSelect";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
|
||||
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
|
||||
import { ApiError } from "@/lib/http/client";
|
||||
|
||||
|
|
@ -21,6 +27,17 @@ import {
|
|||
|
||||
const pct = (value: number): string => `${value.toFixed(1)}%`;
|
||||
|
||||
const endsIn = (endsAt: string | null | undefined): string | null => {
|
||||
if (!endsAt) return null;
|
||||
const remainingMs = new Date(endsAt).getTime() - Date.now();
|
||||
if (Number.isNaN(remainingMs)) return null;
|
||||
if (remainingMs <= 0) return "ending now";
|
||||
const days = Math.round(remainingMs / 86_400_000);
|
||||
if (days >= 2) return `ends in ${days} days`;
|
||||
const hours = Math.round(remainingMs / 3_600_000);
|
||||
return hours >= 2 ? `ends in ${hours} hours` : "ends within the hour";
|
||||
};
|
||||
|
||||
const STATUS_STYLES: Record<string, string> = {
|
||||
pending: "bg-secondary text-muted-foreground",
|
||||
running: "bg-blue-50 text-blue-700",
|
||||
|
|
@ -91,7 +108,8 @@ const JobResults: React.FC<{
|
|||
<p className="text-xs text-muted-foreground">
|
||||
{job.completed_count.toLocaleString()} judged · {job.failed_count.toLocaleString()} failed ·{" "}
|
||||
{job.cost_actual != null ? `${usd(job.cost_actual)} judge spend` : "no judge spend yet"}
|
||||
{job.cost_estimate != null ? ` (est. ${usd(job.cost_estimate)}/wk)` : ""}
|
||||
{job.cost_estimate != null ? ` (est. ${usd(job.cost_estimate)})` : ""}
|
||||
{active && endsIn(job.ends_at) ? ` · ${endsIn(job.ends_at)}` : ""}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -132,13 +150,61 @@ const JobResults: React.FC<{
|
|||
/** Kept in sync with DEFAULT_SHADOW_EVAL_JUDGE_MODEL on the backend. */
|
||||
const DEFAULT_JUDGE_MODEL = "anthropic/claude-sonnet-5";
|
||||
|
||||
const DURATION_OPTIONS = [
|
||||
{ value: "1", label: "1 day" },
|
||||
{ value: "3", label: "3 days" },
|
||||
{ value: "7", label: "7 days" },
|
||||
{ value: "14", label: "14 days" },
|
||||
{ value: "30", label: "30 days" },
|
||||
] as const;
|
||||
|
||||
const DEFAULT_DURATION_DAYS = "7";
|
||||
const KEY_PAGE_SIZE = 50;
|
||||
|
||||
const KeySelect: React.FC<{ value: string; onChange: (token: string) => void }> = ({ value, onChange }) => {
|
||||
const [search, setSearch] = useState("");
|
||||
const { data, isPending } = useKeys(1, KEY_PAGE_SIZE, { selectedKeyAlias: search || null });
|
||||
const options = useMemo<SearchSelectOption[]>(
|
||||
() =>
|
||||
(data?.keys ?? []).map((key) => ({
|
||||
label: key.key_alias || key.key_name || key.token,
|
||||
value: key.token,
|
||||
sublabel: key.token,
|
||||
})),
|
||||
[data],
|
||||
);
|
||||
return (
|
||||
<PaginatedSearchSelect
|
||||
inputId="shadow-eval-key"
|
||||
options={options}
|
||||
value={value}
|
||||
onValueChange={onChange}
|
||||
onSearchChange={setSearch}
|
||||
onLoadMore={() => {}}
|
||||
isLoading={isPending}
|
||||
placeholder="Search keys by alias"
|
||||
emptyText="No matching keys"
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
const StartForm: React.FC<{ accessToken: string | null }> = ({ accessToken }) => {
|
||||
const [apiKeyId, setApiKeyId] = useState("");
|
||||
const [routerName, setRouterName] = useState("");
|
||||
const [percentage, setPercentage] = useState("10");
|
||||
const [durationDays, setDurationDays] = useState(DEFAULT_DURATION_DAYS);
|
||||
const [judgeModel, setJudgeModel] = useState(DEFAULT_JUDGE_MODEL);
|
||||
const { data: autoRouters } = useAutoRouters();
|
||||
const start = useStartShadowEval();
|
||||
|
||||
const routerOptions = useMemo<SearchSelectOption[]>(() => {
|
||||
const names = new Set<string>();
|
||||
for (const deployment of autoRouters ?? []) {
|
||||
if (deployment.model_name) names.add(deployment.model_name);
|
||||
}
|
||||
return [...names].sort().map((name) => ({ label: name, value: name }));
|
||||
}, [autoRouters]);
|
||||
|
||||
const parsedPct = Number.parseFloat(percentage);
|
||||
const valid =
|
||||
Boolean(accessToken) && apiKeyId.trim() !== "" && routerName.trim() !== "" && parsedPct > 0 && parsedPct <= 100;
|
||||
|
|
@ -149,41 +215,84 @@ const StartForm: React.FC<{ accessToken: string | null }> = ({ accessToken }) =>
|
|||
<CardTitle className="text-sm font-medium text-foreground">Start a shadow eval</CardTitle>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Duplicates a sampled slice of the key's traffic through the auto-router and has an LLM judge compare both
|
||||
answers blind. The router's answers are never served to users. Judge calls bill to the proxy — an
|
||||
estimate is shown before anything runs.
|
||||
answers blind. The router's answers are never served to users. Judge calls bill to the shadowed key — an
|
||||
estimate is shown before anything runs, and the job stops itself when the duration ends.
|
||||
</p>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-3">
|
||||
<div className="grid gap-3 sm:grid-cols-3">
|
||||
<Input
|
||||
placeholder="Key hash (token) to shadow"
|
||||
value={apiKeyId}
|
||||
onChange={(e) => setApiKeyId(e.target.value)}
|
||||
/>
|
||||
<Input
|
||||
placeholder="Auto-router name (e.g. claude-auto)"
|
||||
value={routerName}
|
||||
onChange={(e) => setRouterName(e.target.value)}
|
||||
/>
|
||||
<div className="flex items-center gap-2">
|
||||
<Input
|
||||
type="number"
|
||||
min={0.1}
|
||||
max={100}
|
||||
step={0.1}
|
||||
className="w-24"
|
||||
value={percentage}
|
||||
onChange={(e) => setPercentage(e.target.value)}
|
||||
<div className="space-y-1.5">
|
||||
<Label htmlFor="shadow-eval-key" className="text-xs">
|
||||
Key to shadow
|
||||
</Label>
|
||||
<KeySelect value={apiKeyId} onChange={setApiKeyId} />
|
||||
</div>
|
||||
<div className="space-y-1.5">
|
||||
<Label htmlFor="shadow-eval-router" className="text-xs">
|
||||
Auto-router
|
||||
</Label>
|
||||
<SearchSelect
|
||||
options={routerOptions}
|
||||
value={routerName}
|
||||
onValueChange={setRouterName}
|
||||
placeholder="Select an auto-router"
|
||||
emptyText="No auto-routers configured"
|
||||
/>
|
||||
<span className="text-sm text-muted-foreground">% of traffic</span>
|
||||
</div>
|
||||
<div className="space-y-1.5">
|
||||
<Label htmlFor="shadow-eval-pct" className="text-xs">
|
||||
Traffic sampled
|
||||
</Label>
|
||||
<div className="flex items-center gap-2">
|
||||
<Input
|
||||
id="shadow-eval-pct"
|
||||
type="number"
|
||||
min={0.1}
|
||||
max={100}
|
||||
step={0.1}
|
||||
className="w-24"
|
||||
value={percentage}
|
||||
onChange={(e) => setPercentage(e.target.value)}
|
||||
/>
|
||||
<span className="text-sm text-muted-foreground">% of traffic</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className="grid gap-3 sm:grid-cols-3">
|
||||
<Input
|
||||
placeholder={`Judge model (default: ${DEFAULT_JUDGE_MODEL})`}
|
||||
value={judgeModel}
|
||||
onChange={(e) => setJudgeModel(e.target.value)}
|
||||
/>
|
||||
<div className="space-y-1.5">
|
||||
<Label className="text-xs">Duration</Label>
|
||||
<Select
|
||||
value={durationDays}
|
||||
onValueChange={(v: string | null) => setDurationDays(v ?? DEFAULT_DURATION_DAYS)}
|
||||
>
|
||||
<SelectTrigger className="w-full">
|
||||
<SelectValue>{DURATION_OPTIONS.find((o) => o.value === durationDays)?.label}</SelectValue>
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{DURATION_OPTIONS.map((option) => (
|
||||
<SelectItem key={option.value} value={option.value}>
|
||||
{option.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
<div className="space-y-1.5 sm:col-span-2">
|
||||
<Label htmlFor="shadow-eval-judge" className="text-xs">
|
||||
Judge model
|
||||
</Label>
|
||||
<Input
|
||||
id="shadow-eval-judge"
|
||||
placeholder={`Default: ${DEFAULT_JUDGE_MODEL}`}
|
||||
value={judgeModel}
|
||||
onChange={(e) => setJudgeModel(e.target.value)}
|
||||
/>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
The judge only compares two answers blind — a mid-tier model (Claude Sonnet or GPT-4o class) is the sweet
|
||||
spot. Small "nano/mini" models give unreliable verdicts; frontier reasoning models cost more
|
||||
without changing outcomes.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
{start.error ? (
|
||||
<p className="text-sm text-destructive">
|
||||
|
|
@ -198,6 +307,7 @@ const StartForm: React.FC<{ accessToken: string | null }> = ({ accessToken }) =>
|
|||
api_key_id: apiKeyId.trim(),
|
||||
router_name: routerName.trim(),
|
||||
shadow_percentage: parsedPct,
|
||||
duration_days: Number.parseInt(durationDays, 10),
|
||||
judge_model: judgeModel.trim() || DEFAULT_JUDGE_MODEL,
|
||||
},
|
||||
})
|
||||
|
|
|
|||
22
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
22
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -842,10 +842,11 @@ export interface paths {
|
|||
* traffic through an auto-router, judge real vs. shadow responses blind, and
|
||||
* stratify win rates by the router's tier classification.
|
||||
*
|
||||
* The shadow responses are never served to users. The job stays active until
|
||||
* stopped via /auto_router/shadow_eval/{job_id}/stop. Judge calls bill to the
|
||||
* proxy; the estimate returned here prices them from the key's trailing
|
||||
* request volume.
|
||||
* The shadow responses are never served to users. The job samples traffic for
|
||||
* duration_days (or until stopped via /auto_router/shadow_eval/{job_id}/stop),
|
||||
* then completes itself. Judge calls bill to the shadowed key; the estimate
|
||||
* returned here prices them from the key's trailing request volume scaled to
|
||||
* the requested duration.
|
||||
*/
|
||||
post: operations["start_shadow_eval_auto_router_shadow_eval_start_post"];
|
||||
delete?: never;
|
||||
|
|
@ -25351,6 +25352,11 @@ export interface components {
|
|||
cost_estimate?: number | null;
|
||||
/** Created At */
|
||||
created_at: string;
|
||||
/**
|
||||
* Ends At
|
||||
* @description When the job stops sampling on its own; null for jobs started before durations
|
||||
*/
|
||||
ends_at?: string | null;
|
||||
/**
|
||||
* Failed Count
|
||||
* @description Shadow or judge calls that errored and were skipped
|
||||
|
|
@ -32781,9 +32787,15 @@ export interface components {
|
|||
* @description The hashed virtual key whose traffic will be shadowed. Shadow evaluation runs ONLY on this key's traffic; requests made with any other key are not sampled.
|
||||
*/
|
||||
api_key_id: string;
|
||||
/**
|
||||
* Duration Days
|
||||
* @description How many days the job samples traffic before stopping itself
|
||||
* @default 7
|
||||
*/
|
||||
duration_days: number;
|
||||
/**
|
||||
* Judge Model
|
||||
* @description Model used to blindly judge real vs. shadow responses
|
||||
* @description Model used to blindly judge real vs. shadow responses. The judge only compares two answers, so a mid-tier model (Claude Sonnet or GPT-4o class) is the sweet spot: small/nano-class models produce unreliable or malformed verdicts, while frontier reasoning models add cost without changing outcomes.
|
||||
* @default anthropic/claude-sonnet-5
|
||||
*/
|
||||
judge_model: string;
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
Loading…
Add table
Reference in a new issue