fix(shadow_eval): size the judge output cap for a judge that reasons

The cap covers reasoning tokens as well as the verdict, and the models people
pick as judges reason before answering whether the call asks them to or not:
Anthropic's 5 family thinks adaptively and cannot be told not to, so the
reasoning bills against max_tokens with nothing in the request to opt out.

At 1500 the reasoning consumed the budget and the reply arrived empty or cut
off mid-object, which the attempt recorded as an unparseable judge verdict
rather than a result. Headroom costs nothing: max_tokens is a ceiling and only
generated tokens bill, so the only movement is that judge calls which used to
bill their full budget and return nothing now return a verdict.

Deliberately not passing reasoning_effort to bound the reasoning instead:
is_thinking_enabled treats any reasoning_effort as thinking-enabled, which
drops the forced tool_choice that json_mode relies on and turns thinking on
with a 1024-token floor for judges that were not reasoning at all.

litellm/integrations/shadow_eval_logger.py

@@ -60,9 +60,13 @@ _MAX_CONCURRENT_SHADOW_TASKS: Final = 16
 _MAX_JUDGE_RESPONSE_CHARS: Final = 8_000
 _MAX_JUDGE_PROMPT_CHARS: Final = 24_000
 
-# The judge answers with a small JSON object; a tighter budget truncates the JSON
-# mid-object and the attempt is lost to an error row.
-JUDGE_MAX_OUTPUT_TOKENS: Final = 1500
+# The judge answers with a small JSON object, but the cap covers reasoning tokens too,
+# and the models people pick as judges reason before answering whether or not the call
+# asks them to (Anthropic's 5 family thinks adaptively and cannot be told not to). A
+# budget sized for the JSON alone is spent on invisible reasoning instead, and the reply
+# arrives empty or truncated mid-object, which the attempt records as an unparseable
+# verdict. Headroom is free: max_tokens is a ceiling, and only generated tokens bill.
+JUDGE_MAX_OUTPUT_TOKENS: Final = 4096
 
 _MAX_ERROR_CHARS: Final = 500
 

tests/test_litellm/integrations/test_shadow_eval_logger.py

@@ -120,6 +120,27 @@ def _router(
     return router
 
 
+def _reasoning_judge_router(reasoning_tokens, verdict='{"preference": "A", "confidence": 0.9}'):
+    """A router whose judge arm reasons before it answers, the way Anthropic's 5 family
+    does whether or not the call asks it to. Reasoning is billed against the caller's own
+    max_tokens and the reply is cut off at that cap, so a cap that does not clear the
+    reasoning budget yields a truncated verdict or no verdict at all. One character stands
+    in for one token, which is what makes the cap the thing under test."""
+    router = MagicMock()
+    router.model_group_alias = {}
+    router.get_model_list = MagicMock(return_value=[{"litellm_params": {"model": "openai/gpt-4o-mini"}}])
+
+    async def acompletion(**kwargs):
+        if kwargs["metadata"].get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == SHADOW_EVAL_ROUTER_CALL_ORIGIN:
+            kwargs["metadata"]["routing_decision"] = {"tier_label": "SIMPLE", "routed_model": "cheap-model"}
+            return {"choices": [{"message": {"content": "shadow answer"}}]}
+        budget_for_the_answer = kwargs["max_tokens"] - reasoning_tokens
+        return {"choices": [{"message": {"content": verdict[: max(0, budget_for_the_answer)]}}]}
+
+    router.acompletion = MagicMock(side_effect=acompletion)
+    return router
+
+
 def _spend_counter(store=None):
     """In-memory stand-in for the proxy's cross-pod spend counter: reads take the max of
     the counter and the caller's fallback, exactly like get_current_spend does for a key
@@ -1134,6 +1155,34 @@ class TestShadowPipeline:
         assert row["shadow_cost"] == 0.007
         assert logger._test_counter["spend:shadow_eval:job-1"] == 0.007
 
+    async def test_the_judge_output_cap_leaves_room_for_a_reasoning_judge(self):
+        """The output cap covers reasoning tokens as well as the answer, and the models
+        people pick as judges reason before answering whether or not the call asks them to.
+        A cap sized for the verdict JSON alone is spent on reasoning instead and the reply
+        arrives empty, which the attempt records as an unparseable verdict rather than a
+        result. The judge here burns a reasoning budget typical of a thinking model on a
+        comparison task, so the cap has to clear it for the verdict to survive."""
+        reasoning_tokens = 2000
+        logger = _logger(router=_reasoning_judge_router(reasoning_tokens), prisma=(prisma := _prisma()))
+
+        await logger._run_shadow_eval(
+            job=_job(),
+            request_id="req-1",
+            messages=({"role": "user", "content": "hi"},),
+            real_text="real answer",
+            real_model="claude-opus",
+            real_cost=0.0,
+            real_classifier_cost=0.0,
+            real_cache_hit=False,
+            control_tier=None,
+            shadow_params={},
+            parent_metadata={},
+        )
+
+        row = prisma.db.litellm_shadowevalattempt.create.call_args.kwargs["data"]
+        assert row["outcome"] in ("real", "shadow", "tie"), row["error"]
+        assert row["error"] is None
+
     async def test_a_pipeline_error_after_the_shadow_call_keeps_its_billed_cost(self, monkeypatch: pytest.MonkeyPatch):
         """An unexpected error between the billed shadow call and the attempt write must
         still record the shadow cost, or the per-key dollar gate undercounts forever."""