diff --git a/litellm/integrations/shadow_eval_logger.py b/litellm/integrations/shadow_eval_logger.py index 10ee32af282..b34788f48fe 100644 --- a/litellm/integrations/shadow_eval_logger.py +++ b/litellm/integrations/shadow_eval_logger.py @@ -532,6 +532,8 @@ class ShadowEvalLogger(CustomLogger): """ prisma: Final = self._prisma_provider() try: + if prisma is None: + return # nowhere to record a verdict; spending on shadow/judge calls would be pure waste real_text: Final = self._extract_response_text(response_obj) if not real_text or not messages: return @@ -553,9 +555,6 @@ class ShadowEvalLogger(CustomLogger): if isinstance(verdict, _CallFailure): await self._bump_failed(job.id, verdict.error) return - - if prisma is None: - return await prisma.db.litellm_shadowevalverdict.create( data={ # mutable-ok: Prisma payload "job_id": job.id, diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index 62217e230a9..b91f0006348 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -820,6 +820,7 @@ async def start_shadow_eval( async def list_shadow_eval_jobs( user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], api_key_id: Annotated[str | None, Query(description="Filter to jobs shadowing this key")] = None, + limit: Annotated[int, Query(ge=1, le=200, description="Newest jobs to return")] = 50, ) -> list[GetShadowEvalJobResponse]: """List shadow eval jobs, newest first. Results are omitted; fetch a single job for them.""" from litellm.proxy.proxy_server import prisma_client @@ -832,7 +833,7 @@ async def list_shadow_eval_jobs( records: Final = await prisma_client.db.litellm_shadowevaljob.find_many( where=where, order={"created_at": "desc"}, # mutable-ok: Prisma order - take=50, # mutable-ok: Prisma order + take=limit, ) return [_job_to_response(record, results=None) for record in records or ()] # mutable-ok: FastAPI response_model diff --git a/tests/test_litellm/integrations/test_shadow_eval_logger.py b/tests/test_litellm/integrations/test_shadow_eval_logger.py index bf8b5b86cf1..b8498c164a5 100644 --- a/tests/test_litellm/integrations/test_shadow_eval_logger.py +++ b/tests/test_litellm/integrations/test_shadow_eval_logger.py @@ -388,6 +388,33 @@ class TestStoppedJobCannotBeReactivated: assert call_kwargs["data"]["status"] == "running" +@pytest.mark.asyncio +class TestNoPaidCallsWithoutAPlaceToRecordThem: + async def test_pipeline_spends_nothing_when_prisma_is_unavailable(self): + """Regression: the prisma check sat below the shadow and judge dispatch, so a + DB outage mid-flight paid for both provider calls and then dropped the + verdict. No verdict storage means no paid calls at all.""" + router = _router_mock() + router.acompletion = AsyncMock() + logger = ShadowEvalLogger(router_provider=lambda: router, prisma_provider=lambda: None) + logger._call_judge = AsyncMock() + + job = ActiveShadowEvalJob(id="j1", router_name="r", judge_model="m", shadow_percentage=100.0, status="running") + await logger._run_shadow_eval( + job=job, + request_id="req-1", + messages=[{"role": "user", "content": "hi"}], + response_obj={"choices": [{"message": {"content": "real text"}}]}, + real_model="gpt-4o", + real_response_tokens=42, + model_parameters={}, + parent_metadata={}, + ) + + router.acompletion.assert_not_awaited() + logger._call_judge.assert_not_awaited() + + @pytest.mark.asyncio class TestVerdictRowRecordsBothSides: async def test_verdict_row_carries_ids_and_token_counts_for_both_arms(self): diff --git a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py index 6facee6b63e..9197b0c55a1 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py @@ -874,6 +874,17 @@ class TestShadowEvalJobLifecycleEndpoints: assert all(j.results is None for j in jobs) assert prisma.db.litellm_shadowevaljob.find_many.call_args.kwargs["order"] == {"created_at": "desc"} + @pytest.mark.asyncio + async def test_list_limit_reaches_the_query(self, monkeypatch: pytest.MonkeyPatch): + from litellm.proxy.management_endpoints.auto_router_endpoints import list_shadow_eval_jobs + + prisma = self._prisma(monkeypatch) + prisma.db.litellm_shadowevaljob.find_many = AsyncMock(return_value=[]) + + await list_shadow_eval_jobs(ADMIN, limit=120) + + assert prisma.db.litellm_shadowevaljob.find_many.call_args.kwargs["take"] == 120 + @pytest.mark.asyncio async def test_get_returns_the_job_with_aggregated_results(self, monkeypatch: pytest.MonkeyPatch): from litellm.proxy.management_endpoints.auto_router_endpoints import get_shadow_eval_job diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 965cc5ee55c..69203f755f1 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -37194,6 +37194,8 @@ export interface operations { query?: { /** @description Filter to jobs shadowing this key */ api_key_id?: string | null; + /** @description Newest jobs to return */ + limit?: number; }; header?: never; path?: never;