feat: jump-to-shadow-eval button; fix: block shadow/judge calls past budget

Adds a "Shadow eval" button next to the Auto-router usage heading that
smooth-scrolls to the shadow eval section, so it's reachable without
scrolling past the benchmarks body first.

Also fixes a PR review comment (veria-ai): shadow and judge calls ran
outside the normal auth path, so they never went through
reserve_budget_for_request and could push an already-exhausted key or
team further over budget before their own spend was even recorded.
_key_or_team_is_over_budget reads the same cross-pod spend counters
that path reserves against (via the existing get_current_spend) and
skips the shadow/judge pair outright when the shadowed key or its team
is already at or over budget. This is a read-time check, not a
reservation — appropriate for a best-effort background measurement
task, not a billed user request — so it narrows the window rather than
closing it against concurrent bursts, which the response comment
explains.

Co-Authored-By: Claude <noreply@anthropic.com>

litellm/integrations/shadow_eval_logger.py

@@ -19,9 +19,11 @@ Shadow and judge calls carry ``shadow_eval_internal`` metadata so this logger
 ignores its own traffic and cannot recurse. They also carry the shadowed key's
 identity metadata, so the provider spend they incur is attributed to that key
 (and its team/org/user) and counts against every budget that key is subject to,
-including the global proxy budget. A job additionally stops itself once its
-sampling window (``ends_at``) closes or its judge spend reaches a multiple of
-the estimate quoted when it started.
+including the global proxy budget. A shadow/judge pair is skipped outright if
+the shadowed key or its team is already at or over budget, read from the same
+cross-pod spend counters the normal auth path reserves against. A job
+additionally stops itself once its sampling window (``ends_at``) closes or its
+judge spend reaches a multiple of the estimate quoted when it started.
 """
 
 import asyncio
@@ -194,6 +196,48 @@ def _job_is_past_its_end(job: ActiveShadowEvalJob) -> bool:
     return job.ends_at is not None and datetime.now(timezone.utc) >= job.ends_at
 
 
+async def _key_or_team_is_over_budget(metadata: Mapping[str, object]) -> bool:
+    """Whether the shadowed key or its team has no budget left for the shadow/judge calls.
+
+    These calls run outside the normal auth path (no /key/generate-issued request
+    reaches user_api_key_auth for them), so they never go through
+    reserve_budget_for_request. Reading the same cross-pod counters that path
+    reserves against, at read time, stops a key or team that is already out of
+    budget from having a shadow eval push it further over: not a hard reservation
+    (a burst of concurrent requests could still all pass this read), but it closes
+    the gap for the common case a background quality-measurement task should
+    never make worse.
+    """
+    try:
+        from litellm.proxy.proxy_server import get_current_spend
+    except ImportError:
+        return False
+
+    api_key_hash: Final = metadata.get("user_api_key_hash")
+    max_budget: Final = metadata.get("user_api_key_max_budget")
+    if isinstance(api_key_hash, str) and isinstance(max_budget, (int, float)) and max_budget > 0:
+        spend: Final = await get_current_spend(
+            counter_key=f"spend:key:{api_key_hash}",
+            fallback_spend=float(metadata.get("user_api_key_spend") or 0.0),
+            max_budget=float(max_budget),
+        )
+        if spend >= max_budget:
+            return True
+
+    team_id: Final = metadata.get("user_api_key_team_id")
+    team_max_budget: Final = metadata.get("user_api_key_team_max_budget")
+    if isinstance(team_id, str) and isinstance(team_max_budget, (int, float)) and team_max_budget > 0:
+        team_spend: Final = await get_current_spend(
+            counter_key=f"spend:team:{team_id}",
+            fallback_spend=float(metadata.get("user_api_key_team_spend") or 0.0),
+            max_budget=float(team_max_budget),
+        )
+        if team_spend >= team_max_budget:
+            return True
+
+    return False
+
+
 def _job_is_over_spend_cap(job: ActiveShadowEvalJob) -> bool:
     """Whether the job has spent past what its start-time estimate justifies.
 
@@ -282,6 +326,8 @@ class ShadowEvalLogger(CustomLogger):
                 return  # only chat-shaped traffic is comparable
             if self._inflight_shadow_tasks >= _MAX_CONCURRENT_SHADOW_TASKS:
                 return
+            if await _key_or_team_is_over_budget(metadata):
+                return  # the shadowed key/team has no budget left for the extra calls
             raw_messages: Final = kwargs.get("messages")
             self._inflight_shadow_tasks += 1
             task: Final = asyncio.create_task(

tests/test_litellm/integrations/test_shadow_eval_logger.py

@@ -3,7 +3,7 @@
 import asyncio
 import json
 from datetime import datetime, timedelta, timezone
-from unittest.mock import AsyncMock, MagicMock
+from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
 
@@ -13,6 +13,7 @@ from litellm.integrations.shadow_eval_logger import (
     SHADOW_EVAL_INTERNAL_MARKER,
     ActiveShadowEvalJob,
     ShadowEvalLogger,
+    _key_or_team_is_over_budget,
     _parse_pairwise_verdict,
     _sample_hits,
     _unmask_preference,
@@ -475,6 +476,112 @@ class TestSubCallsAreAttributedToTheShadowedKey:
         assert logger._call_judge.await_args.kwargs["parent_metadata"]["user_api_key"] == "hashed-key"
 
 
+@pytest.mark.asyncio
+class TestKeyOrTeamIsOverBudget:
+    """Shadow/judge calls run outside the normal auth path and never reserve budget
+    for themselves, so an already-exhausted key or team must not be pushed further
+    over by a background eval it never asked to run."""
+
+    @staticmethod
+    def _metadata(**overrides):
+        return {
+            "user_api_key_hash": "key-hash",
+            "user_api_key_max_budget": 10.0,
+            "user_api_key_spend": 4.0,
+            **overrides,
+        }
+
+    async def test_under_key_budget_is_not_over_budget(self):
+        with patch("litellm.proxy.proxy_server.get_current_spend", AsyncMock(return_value=4.0)):
+            assert not await _key_or_team_is_over_budget(self._metadata())
+
+    async def test_at_key_budget_is_over_budget(self):
+        with patch("litellm.proxy.proxy_server.get_current_spend", AsyncMock(return_value=10.0)):
+            assert await _key_or_team_is_over_budget(self._metadata())
+
+    async def test_reads_the_cross_pod_counter_not_the_stale_metadata_spend(self):
+        get_current_spend = AsyncMock(return_value=11.0)
+        with patch("litellm.proxy.proxy_server.get_current_spend", get_current_spend):
+            assert await _key_or_team_is_over_budget(self._metadata(user_api_key_spend=0.0))
+        get_current_spend.assert_awaited_once_with(
+            counter_key="spend:key:key-hash", fallback_spend=0.0, max_budget=10.0
+        )
+
+    async def test_team_over_budget_is_caught_even_when_key_has_room(self):
+        metadata = self._metadata(
+            user_api_key_team_id="team-1",
+            user_api_key_team_max_budget=5.0,
+            user_api_key_team_spend=5.0,
+        )
+
+        async def fake_spend(counter_key, fallback_spend, max_budget):
+            return max_budget if counter_key == "spend:team:team-1" else 1.0
+
+        with patch("litellm.proxy.proxy_server.get_current_spend", AsyncMock(side_effect=fake_spend)):
+            assert await _key_or_team_is_over_budget(metadata)
+
+    async def test_no_budget_configured_is_never_over_budget(self):
+        assert not await _key_or_team_is_over_budget({"user_api_key_hash": "key-hash"})
+
+    async def test_missing_proxy_server_fails_open_rather_than_blocking_logging(self):
+        with patch.dict("sys.modules", {"litellm.proxy.proxy_server": None}):
+            assert not await _key_or_team_is_over_budget(self._metadata())
+
+
+@pytest.mark.asyncio
+class TestSuccessHookSkipsWhenOverBudget:
+    async def test_over_budget_key_is_skipped_before_scheduling_the_shadow_task(self):
+        job = ActiveShadowEvalJob(id="j1", router_name="r", shadow_percentage=100.0, judge_model="m", status="running")
+        logger, _, router = _logger_with_mocks(job)
+        logger._run_shadow_eval = AsyncMock()
+        kwargs = {
+            "standard_logging_object": {
+                "id": "req-1",
+                "model": "gpt-4o",
+                "call_type": "acompletion",
+                "metadata": {
+                    "user_api_key_hash": "key-hash",
+                    "user_api_key_max_budget": 10.0,
+                    "user_api_key_spend": 10.0,
+                },
+            },
+            "litellm_params": {"metadata": {}},
+            "messages": [{"role": "user", "content": "hi"}],
+        }
+
+        with patch("litellm.proxy.proxy_server.get_current_spend", AsyncMock(return_value=10.0)):
+            await logger.async_log_success_event(kwargs, MagicMock(), None, None)
+        await asyncio.sleep(0)
+
+        logger._run_shadow_eval.assert_not_awaited()
+        router.acompletion.assert_not_called()
+
+    async def test_under_budget_key_still_schedules_the_shadow_task(self):
+        job = ActiveShadowEvalJob(id="j1", router_name="r", shadow_percentage=100.0, judge_model="m", status="running")
+        logger, _, router = _logger_with_mocks(job)
+        logger._run_shadow_eval = AsyncMock()
+        kwargs = {
+            "standard_logging_object": {
+                "id": "req-1",
+                "model": "gpt-4o",
+                "call_type": "acompletion",
+                "metadata": {
+                    "user_api_key_hash": "key-hash",
+                    "user_api_key_max_budget": 10.0,
+                    "user_api_key_spend": 4.0,
+                },
+            },
+            "litellm_params": {"metadata": {}},
+            "messages": [{"role": "user", "content": "hi"}],
+        }
+
+        with patch("litellm.proxy.proxy_server.get_current_spend", AsyncMock(return_value=4.0)):
+            await logger.async_log_success_event(kwargs, MagicMock(), None, None)
+        await asyncio.sleep(0)
+
+        logger._run_shadow_eval.assert_awaited_once()
+
+
 @pytest.mark.asyncio
 class TestPerJobSpendCap:
     """Budgets bound the key; the cap bounds a single eval, so a bad estimate or a

ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx

@@ -266,4 +266,18 @@ describe("AutoRouterBenchmarksTab", () => {
     expect(screen.getByRole("tab", { name: "30d" })).toBeInTheDocument();
     expect(screen.getByText("All auto-routers")).toBeInTheDocument();
   });
+
+  it("scrolls to the shadow eval section when its jump button is clicked", () => {
+    mockHook({ data: response([group()]) });
+    renderTab();
+    const target = document.createElement("div");
+    target.id = "shadow-eval-section";
+    target.scrollIntoView = vi.fn();
+    document.body.appendChild(target);
+
+    fireEvent.click(screen.getByRole("button", { name: /shadow eval/i }));
+
+    expect(target.scrollIntoView).toHaveBeenCalledWith({ behavior: "smooth" });
+    target.remove();
+  });
 });

ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx

@@ -1,8 +1,10 @@
 "use client";
 
 import React, { useState } from "react";
+import { ArrowDown } from "lucide-react";
 
 import { Badge } from "@/components/ui/badge";
+import { Button } from "@/components/ui/button";
 import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
 import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
 import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
@@ -291,7 +293,17 @@ const AutoRouterBenchmarksTab: React.FC<AutoRouterBenchmarksTabProps> = ({ acces
     <div className="w-full space-y-6">
       <div className="flex flex-col gap-3 sm:flex-row sm:items-start sm:justify-between">
         <div>
-          <h2 className="text-xl font-semibold text-foreground">Auto-router usage</h2>
+          <div className="flex flex-wrap items-center gap-2">
+            <h2 className="text-xl font-semibold text-foreground">Auto-router usage</h2>
+            <Button
+              variant="outline"
+              size="sm"
+              onClick={() => document.getElementById("shadow-eval-section")?.scrollIntoView({ behavior: "smooth" })}
+            >
+              <ArrowDown className="size-3.5" />
+              Shadow eval
+            </Button>
+          </div>
           <p className="mt-1 text-sm text-muted-foreground">{WINDOW_LABELS[range]}</p>
         </div>
         <div className="flex w-full flex-col gap-3 sm:w-auto sm:flex-row sm:items-center">

ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx

@@ -412,7 +412,7 @@ const ShadowEvalSection: React.FC<ShadowEvalSectionProps> = ({ accessToken }) =>
   if (error instanceof ApiError && error.status === 403) return null; // admin-only section
 
   return (
-    <div className="space-y-4">
+    <div id="shadow-eval-section" className="space-y-4 scroll-mt-6">
       <div className="flex flex-wrap items-baseline gap-2">
         <h3 className="text-lg font-semibold text-foreground">Shadow eval</h3>
         <p className="text-xs text-muted-foreground">