fix: gate paid shadow/judge calls on verdict storage; pageable job list

litellm/integrations/shadow_eval_logger.py

@@ -532,6 +532,8 @@ class ShadowEvalLogger(CustomLogger):
         """
         prisma: Final = self._prisma_provider()
         try:
+            if prisma is None:
+                return  # nowhere to record a verdict; spending on shadow/judge calls would be pure waste
             real_text: Final = self._extract_response_text(response_obj)
             if not real_text or not messages:
                 return
@@ -553,9 +555,6 @@ class ShadowEvalLogger(CustomLogger):
             if isinstance(verdict, _CallFailure):
                 await self._bump_failed(job.id, verdict.error)
                 return
-
-            if prisma is None:
-                return
             await prisma.db.litellm_shadowevalverdict.create(
                 data={  # mutable-ok: Prisma payload
                     "job_id": job.id,

litellm/proxy/management_endpoints/auto_router_endpoints.py

@@ -820,6 +820,7 @@ async def start_shadow_eval(
 async def list_shadow_eval_jobs(
     user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
     api_key_id: Annotated[str | None, Query(description="Filter to jobs shadowing this key")] = None,
+    limit: Annotated[int, Query(ge=1, le=200, description="Newest jobs to return")] = 50,
 ) -> list[GetShadowEvalJobResponse]:
     """List shadow eval jobs, newest first. Results are omitted; fetch a single job for them."""
     from litellm.proxy.proxy_server import prisma_client
@@ -832,7 +833,7 @@ async def list_shadow_eval_jobs(
     records: Final = await prisma_client.db.litellm_shadowevaljob.find_many(
         where=where,
         order={"created_at": "desc"},  # mutable-ok: Prisma order
-        take=50,  # mutable-ok: Prisma order
+        take=limit,
     )
     return [_job_to_response(record, results=None) for record in records or ()]  # mutable-ok: FastAPI response_model
 

tests/test_litellm/integrations/test_shadow_eval_logger.py

@@ -388,6 +388,33 @@ class TestStoppedJobCannotBeReactivated:
         assert call_kwargs["data"]["status"] == "running"
 
 
+@pytest.mark.asyncio
+class TestNoPaidCallsWithoutAPlaceToRecordThem:
+    async def test_pipeline_spends_nothing_when_prisma_is_unavailable(self):
+        """Regression: the prisma check sat below the shadow and judge dispatch, so a
+        DB outage mid-flight paid for both provider calls and then dropped the
+        verdict. No verdict storage means no paid calls at all."""
+        router = _router_mock()
+        router.acompletion = AsyncMock()
+        logger = ShadowEvalLogger(router_provider=lambda: router, prisma_provider=lambda: None)
+        logger._call_judge = AsyncMock()
+
+        job = ActiveShadowEvalJob(id="j1", router_name="r", judge_model="m", shadow_percentage=100.0, status="running")
+        await logger._run_shadow_eval(
+            job=job,
+            request_id="req-1",
+            messages=[{"role": "user", "content": "hi"}],
+            response_obj={"choices": [{"message": {"content": "real text"}}]},
+            real_model="gpt-4o",
+            real_response_tokens=42,
+            model_parameters={},
+            parent_metadata={},
+        )
+
+        router.acompletion.assert_not_awaited()
+        logger._call_judge.assert_not_awaited()
+
+
 @pytest.mark.asyncio
 class TestVerdictRowRecordsBothSides:
     async def test_verdict_row_carries_ids_and_token_counts_for_both_arms(self):

tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py

@@ -874,6 +874,17 @@ class TestShadowEvalJobLifecycleEndpoints:
         assert all(j.results is None for j in jobs)
         assert prisma.db.litellm_shadowevaljob.find_many.call_args.kwargs["order"] == {"created_at": "desc"}
 
+    @pytest.mark.asyncio
+    async def test_list_limit_reaches_the_query(self, monkeypatch: pytest.MonkeyPatch):
+        from litellm.proxy.management_endpoints.auto_router_endpoints import list_shadow_eval_jobs
+
+        prisma = self._prisma(monkeypatch)
+        prisma.db.litellm_shadowevaljob.find_many = AsyncMock(return_value=[])
+
+        await list_shadow_eval_jobs(ADMIN, limit=120)
+
+        assert prisma.db.litellm_shadowevaljob.find_many.call_args.kwargs["take"] == 120
+
     @pytest.mark.asyncio
     async def test_get_returns_the_job_with_aggregated_results(self, monkeypatch: pytest.MonkeyPatch):
         from litellm.proxy.management_endpoints.auto_router_endpoints import get_shadow_eval_job

ui/litellm-dashboard/src/lib/http/schema.d.ts

@@ -37194,6 +37194,8 @@ export interface operations {
             query?: {
                 /** @description Filter to jobs shadowing this key */
                 api_key_id?: string | null;
+                /** @description Newest jobs to return */
+                limit?: number;
             };
             header?: never;
             path?: never;