Merge pull request #39817 from BerriAI/litellm_shadow_eval_judge_output_cap

fix(shadow_eval): size the judge output cap for a judge that reasons

litellm/integrations/shadow_eval_logger.py

@@ -61,9 +61,9 @@ _MAX_CONCURRENT_SHADOW_TASKS: Final = 16
 _MAX_JUDGE_RESPONSE_CHARS: Final = 8_000
 _MAX_JUDGE_PROMPT_CHARS: Final = 24_000
 
-# The judge answers with a small JSON object; a tighter budget truncates the JSON
-# mid-object and the attempt is lost to an error row.
-JUDGE_MAX_OUTPUT_TOKENS: Final = 1500
+# Covers the judge's reasoning tokens as well as its small JSON answer: a judge deployment
+# carrying an elevated reasoning_effort spends a tight cap before it ever answers.
+JUDGE_MAX_OUTPUT_TOKENS: Final = 4096
 
 _MAX_ERROR_CHARS: Final = 500
 
@@ -419,6 +419,20 @@ def _failure_detail(e: BaseException) -> str:
     return f"{type(e).__name__}{location}: {e}"
 
 
+def _judge_reply_shape(response: object) -> str:
+    """How an unparseable judge reply was shaped. The parser's own message cannot separate a
+    judge that answered with nothing from one truncated mid-object, and those want opposite
+    fixes. Shape only, never the reply text: the judge quotes the sampled turns it compares,
+    and no attempt row carries sampled content today."""
+    read: Final = _chat_message_reader(response)
+    if read is None:
+        return "unreadable judge reply"
+    content: Final = read("content")
+    served: Final = str(_field_reader(response)("model") or "unknown")
+    body: Final = f"{len(str(content))} chars" if content else "no content"
+    return f"finish_reason={_chat_finish_reason(response)}, content={body}, model={served}"
+
+
 def _call_cost(response: object) -> float:
     """Price one eval-arm call with the figure the spend pipeline bills: the router client
     stamps _hidden_params.response_cost from the deployment's own pricing, which the public
@@ -1266,7 +1280,9 @@ class ShadowEvalLogger(CustomLogger):
             verdict: Final = PairwiseVerdict.model_validate(parse_json_verdict(raw))
         except Exception as e:  # noqa: BLE001  # malformed verdicts become error rows
             verbose_logger.debug("shadow_eval: unparseable judge verdict: %s", e)
-            return _CallFailure(f"unparseable judge verdict: {e}", cost=_call_cost(response))
+            return _CallFailure(
+                f"unparseable judge verdict: {e}; {_judge_reply_shape(response)}", cost=_call_cost(response)
+            )
         return _JudgeVerdict(
             preference=_unmask_preference(verdict.preference, real_is_a),
             confidence=max(0.0, min(1.0, verdict.confidence)),

tests/test_litellm/integrations/test_shadow_eval_logger.py

@@ -3,6 +3,7 @@ the detached pipeline's single attempt-row write, and the cache-first job lookup
 
 import asyncio
 from datetime import datetime, timedelta, timezone
+from typing import Final
 from unittest.mock import AsyncMock, MagicMock
 
 import pytest
@@ -145,6 +146,48 @@ def _shadow_reply_router(message, finish_reason="stop", routed_model="cheap-mode
     return router
 
 
+def _reasoning_judge_router(
+    reasoning_tokens: int, verdict: str = '{"preference": "A", "confidence": 0.9}'
+) -> MagicMock:
+    """A router whose judge arm reasons before it answers, the way a deployment carrying an
+    elevated reasoning_effort does: reasoning bills against the caller's own max_tokens and
+    the reply is cut off at that cap. One character stands in for one token."""
+    router = MagicMock()
+    router.model_group_alias = {}
+    router.get_model_list = MagicMock(return_value=[{"litellm_params": {"model": "openai/gpt-4o-mini"}}])
+
+    async def acompletion(**kwargs):
+        if kwargs["metadata"].get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == SHADOW_EVAL_ROUTER_CALL_ORIGIN:
+            kwargs["metadata"]["routing_decision"] = {"tier_label": "SIMPLE", "routed_model": "cheap-model"}
+            return {"choices": [{"message": {"content": "shadow answer"}}]}
+        budget_for_the_answer: Final = kwargs["max_tokens"] - reasoning_tokens
+        return {"choices": [{"message": {"content": verdict[: max(0, budget_for_the_answer)]}}]}
+
+    router.acompletion = MagicMock(side_effect=acompletion)
+    return router
+
+
+def _judge_reply_router(content: str | None, finish_reason: str = "stop", served_model: str = "judge-pick") -> MagicMock:
+    """A router whose judge arm returns a caller-shaped reply, so the shapes that all land
+    on the same parser error can be posed apart: no content at all, versus JSON cut off
+    mid-object."""
+    router = MagicMock()
+    router.model_group_alias = {}
+    router.get_model_list = MagicMock(return_value=[{"litellm_params": {"model": "openai/gpt-4o-mini"}}])
+
+    async def acompletion(**kwargs):
+        if kwargs["metadata"].get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == SHADOW_EVAL_ROUTER_CALL_ORIGIN:
+            kwargs["metadata"]["routing_decision"] = {"tier_label": "SIMPLE", "routed_model": "cheap-model"}
+            return {"choices": [{"message": {"content": "shadow answer"}}]}
+        return ModelResponse(
+            model=served_model,
+            choices=[{"index": 0, "finish_reason": finish_reason, "message": {"role": "assistant", "content": content}}],
+        )
+
+    router.acompletion = MagicMock(side_effect=acompletion)
+    return router
+
+
 TOOL_CALL_MESSAGE = {
     "content": None,
     "tool_calls": [{"id": "c1", "type": "function", "function": {"name": "Read", "arguments": "{}"}}],
@@ -1258,6 +1301,79 @@ class TestShadowPipeline:
         assert row["judge_cost"] == expected_cost
         assert row["shadow_cost"] == expected_shadow_cost
 
+    async def _judge_error(self, router: MagicMock, monkeypatch: pytest.MonkeyPatch) -> str:
+        import litellm as litellm_module
+
+        monkeypatch.setattr(litellm_module, "completion_cost", lambda completion_response: 0.007)
+        prisma = _prisma()
+        await _logger(router=router, prisma=prisma)._run_shadow_eval(
+            job=_job(),
+            request_id="req-1",
+            messages=({"role": "user", "content": "hi"},),
+            real_text="real answer",
+            real_model="claude-opus",
+            real_cost=0.0,
+            real_classifier_cost=0.0,
+            real_cache_hit=False,
+            control_tier=None,
+            shadow_params={},
+            parent_metadata={},
+        )
+        return prisma.db.litellm_shadowevalattempt.create.call_args.kwargs["data"]["error"]
+
+    async def test_a_judge_that_answered_nothing_is_told_apart_from_one_cut_off(
+        self, monkeypatch: pytest.MonkeyPatch
+    ):
+        """Both land on the same parser message, and they want opposite fixes: a judge
+        returning no content points at the reply never being text, while one cut off
+        mid-object points at the output cap. The row has to say which."""
+        truncated = '{"preference": "A", "confidence": 0.9, "reasoning": "'
+        answered_nothing = await self._judge_error(_judge_reply_router(None), monkeypatch)
+        cut_off = await self._judge_error(
+            _judge_reply_router(truncated, finish_reason="length"), monkeypatch
+        )
+
+        assert "content=no content" in answered_nothing
+        assert "finish_reason=stop" in answered_nothing
+        assert f"content={len(truncated)} chars" in cut_off
+        assert "finish_reason=length" in cut_off
+
+    async def test_an_unparseable_verdict_names_the_model_that_served_it(self, monkeypatch: pytest.MonkeyPatch):
+        """A judge_model that fans out over deployments hides which one truncates: without
+        the served model the operator cannot tell a bad deployment from a bad cap."""
+        error = await self._judge_error(_judge_reply_router(None, served_model="claude-sonnet-5"), monkeypatch)
+
+        assert "model=claude-sonnet-5" in error
+
+    async def test_a_diagnosed_verdict_error_stays_groupable(self, monkeypatch: pytest.MonkeyPatch):
+        """The customer groups attempt rows by error text. Every varying part has to sit
+        after the first semicolon or each row becomes its own group."""
+        first = await self._judge_error(_judge_reply_router(None, served_model="model-a"), monkeypatch)
+        second = await self._judge_error(_judge_reply_router(None, served_model="model-b"), monkeypatch)
+
+        assert first != second
+        assert first.split(";")[0] == second.split(";")[0]
+
+    async def test_a_judge_reply_that_cannot_be_read_still_records_an_error(self, monkeypatch: pytest.MonkeyPatch):
+        """The shape reader runs inside the failure path: it must never raise a second time
+        and cost the row entirely."""
+        router = MagicMock()
+        router.model_group_alias = {}
+        router.get_model_list = MagicMock(return_value=[{"litellm_params": {"model": "openai/gpt-4o-mini"}}])
+
+        async def acompletion(**kwargs):
+            if kwargs["metadata"].get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == SHADOW_EVAL_ROUTER_CALL_ORIGIN:
+                kwargs["metadata"]["routing_decision"] = {"tier_label": "SIMPLE", "routed_model": "cheap-model"}
+                return {"choices": [{"message": {"content": "shadow answer"}}]}
+            return {"choices": []}
+
+        router.acompletion = MagicMock(side_effect=acompletion)
+
+        error = await self._judge_error(router, monkeypatch)
+
+        assert "unparseable judge verdict" in error
+        assert "unreadable judge reply" in error
+
     async def test_an_empty_shadow_reply_still_bills_its_cost(self, monkeypatch: pytest.MonkeyPatch):
         """A shadow call that returns no extractable text has still billed; pricing it at
         zero would keep the dollar gate open while shadow calls keep charging the key."""
@@ -1287,6 +1403,34 @@ class TestShadowPipeline:
         assert row["shadow_cost"] == 0.007
         assert logger._test_counter["spend:shadow_eval:job-1"] == 0.007
 
+    async def test_the_judge_output_cap_leaves_room_for_a_reasoning_judge(self):
+        """The output cap covers reasoning tokens as well as the answer, and a judge_model
+        deployment carrying an elevated reasoning_effort spends that budget before it writes
+        anything. A cap sized for the verdict JSON alone goes entirely to reasoning and the
+        reply arrives empty, which the attempt records as an unparseable verdict rather than
+        a result. The judge here burns a reasoning budget a live claude-sonnet-5 call was
+        measured at, so the cap has to clear it for the verdict to survive."""
+        reasoning_tokens = 2000
+        logger = _logger(router=_reasoning_judge_router(reasoning_tokens), prisma=(prisma := _prisma()))
+
+        await logger._run_shadow_eval(
+            job=_job(),
+            request_id="req-1",
+            messages=({"role": "user", "content": "hi"},),
+            real_text="real answer",
+            real_model="claude-opus",
+            real_cost=0.0,
+            real_classifier_cost=0.0,
+            real_cache_hit=False,
+            control_tier=None,
+            shadow_params={},
+            parent_metadata={},
+        )
+
+        row = prisma.db.litellm_shadowevalattempt.create.call_args.kwargs["data"]
+        assert row["outcome"] in ("real", "shadow", "tie"), row["error"]
+        assert row["error"] is None
+
     async def _no_text_error(self, router) -> str:
         prisma = _prisma()
         await _logger(router=router, prisma=prisma)._run_shadow_eval(