diff --git a/litellm/integrations/shadow_eval_logger.py b/litellm/integrations/shadow_eval_logger.py index 59403874eb0..54b116639e7 100644 --- a/litellm/integrations/shadow_eval_logger.py +++ b/litellm/integrations/shadow_eval_logger.py @@ -61,9 +61,9 @@ _MAX_CONCURRENT_SHADOW_TASKS: Final = 16 _MAX_JUDGE_RESPONSE_CHARS: Final = 8_000 _MAX_JUDGE_PROMPT_CHARS: Final = 24_000 -# The judge answers with a small JSON object; a tighter budget truncates the JSON -# mid-object and the attempt is lost to an error row. -JUDGE_MAX_OUTPUT_TOKENS: Final = 1500 +# Covers the judge's reasoning tokens as well as its small JSON answer: a judge deployment +# carrying an elevated reasoning_effort spends a tight cap before it ever answers. +JUDGE_MAX_OUTPUT_TOKENS: Final = 4096 _MAX_ERROR_CHARS: Final = 500 @@ -419,6 +419,20 @@ def _failure_detail(e: BaseException) -> str: return f"{type(e).__name__}{location}: {e}" +def _judge_reply_shape(response: object) -> str: + """How an unparseable judge reply was shaped. The parser's own message cannot separate a + judge that answered with nothing from one truncated mid-object, and those want opposite + fixes. Shape only, never the reply text: the judge quotes the sampled turns it compares, + and no attempt row carries sampled content today.""" + read: Final = _chat_message_reader(response) + if read is None: + return "unreadable judge reply" + content: Final = read("content") + served: Final = str(_field_reader(response)("model") or "unknown") + body: Final = f"{len(str(content))} chars" if content else "no content" + return f"finish_reason={_chat_finish_reason(response)}, content={body}, model={served}" + + def _call_cost(response: object) -> float: """Price one eval-arm call with the figure the spend pipeline bills: the router client stamps _hidden_params.response_cost from the deployment's own pricing, which the public @@ -1266,7 +1280,9 @@ class ShadowEvalLogger(CustomLogger): verdict: Final = PairwiseVerdict.model_validate(parse_json_verdict(raw)) except Exception as e: # noqa: BLE001 # malformed verdicts become error rows verbose_logger.debug("shadow_eval: unparseable judge verdict: %s", e) - return _CallFailure(f"unparseable judge verdict: {e}", cost=_call_cost(response)) + return _CallFailure( + f"unparseable judge verdict: {e}; {_judge_reply_shape(response)}", cost=_call_cost(response) + ) return _JudgeVerdict( preference=_unmask_preference(verdict.preference, real_is_a), confidence=max(0.0, min(1.0, verdict.confidence)), diff --git a/tests/test_litellm/integrations/test_shadow_eval_logger.py b/tests/test_litellm/integrations/test_shadow_eval_logger.py index ebfa1d0eb2f..eecd876219e 100644 --- a/tests/test_litellm/integrations/test_shadow_eval_logger.py +++ b/tests/test_litellm/integrations/test_shadow_eval_logger.py @@ -3,6 +3,7 @@ the detached pipeline's single attempt-row write, and the cache-first job lookup import asyncio from datetime import datetime, timedelta, timezone +from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest @@ -145,6 +146,48 @@ def _shadow_reply_router(message, finish_reason="stop", routed_model="cheap-mode return router +def _reasoning_judge_router( + reasoning_tokens: int, verdict: str = '{"preference": "A", "confidence": 0.9}' +) -> MagicMock: + """A router whose judge arm reasons before it answers, the way a deployment carrying an + elevated reasoning_effort does: reasoning bills against the caller's own max_tokens and + the reply is cut off at that cap. One character stands in for one token.""" + router = MagicMock() + router.model_group_alias = {} + router.get_model_list = MagicMock(return_value=[{"litellm_params": {"model": "openai/gpt-4o-mini"}}]) + + async def acompletion(**kwargs): + if kwargs["metadata"].get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == SHADOW_EVAL_ROUTER_CALL_ORIGIN: + kwargs["metadata"]["routing_decision"] = {"tier_label": "SIMPLE", "routed_model": "cheap-model"} + return {"choices": [{"message": {"content": "shadow answer"}}]} + budget_for_the_answer: Final = kwargs["max_tokens"] - reasoning_tokens + return {"choices": [{"message": {"content": verdict[: max(0, budget_for_the_answer)]}}]} + + router.acompletion = MagicMock(side_effect=acompletion) + return router + + +def _judge_reply_router(content: str | None, finish_reason: str = "stop", served_model: str = "judge-pick") -> MagicMock: + """A router whose judge arm returns a caller-shaped reply, so the shapes that all land + on the same parser error can be posed apart: no content at all, versus JSON cut off + mid-object.""" + router = MagicMock() + router.model_group_alias = {} + router.get_model_list = MagicMock(return_value=[{"litellm_params": {"model": "openai/gpt-4o-mini"}}]) + + async def acompletion(**kwargs): + if kwargs["metadata"].get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == SHADOW_EVAL_ROUTER_CALL_ORIGIN: + kwargs["metadata"]["routing_decision"] = {"tier_label": "SIMPLE", "routed_model": "cheap-model"} + return {"choices": [{"message": {"content": "shadow answer"}}]} + return ModelResponse( + model=served_model, + choices=[{"index": 0, "finish_reason": finish_reason, "message": {"role": "assistant", "content": content}}], + ) + + router.acompletion = MagicMock(side_effect=acompletion) + return router + + TOOL_CALL_MESSAGE = { "content": None, "tool_calls": [{"id": "c1", "type": "function", "function": {"name": "Read", "arguments": "{}"}}], @@ -1258,6 +1301,79 @@ class TestShadowPipeline: assert row["judge_cost"] == expected_cost assert row["shadow_cost"] == expected_shadow_cost + async def _judge_error(self, router: MagicMock, monkeypatch: pytest.MonkeyPatch) -> str: + import litellm as litellm_module + + monkeypatch.setattr(litellm_module, "completion_cost", lambda completion_response: 0.007) + prisma = _prisma() + await _logger(router=router, prisma=prisma)._run_shadow_eval( + job=_job(), + request_id="req-1", + messages=({"role": "user", "content": "hi"},), + real_text="real answer", + real_model="claude-opus", + real_cost=0.0, + real_classifier_cost=0.0, + real_cache_hit=False, + control_tier=None, + shadow_params={}, + parent_metadata={}, + ) + return prisma.db.litellm_shadowevalattempt.create.call_args.kwargs["data"]["error"] + + async def test_a_judge_that_answered_nothing_is_told_apart_from_one_cut_off( + self, monkeypatch: pytest.MonkeyPatch + ): + """Both land on the same parser message, and they want opposite fixes: a judge + returning no content points at the reply never being text, while one cut off + mid-object points at the output cap. The row has to say which.""" + truncated = '{"preference": "A", "confidence": 0.9, "reasoning": "' + answered_nothing = await self._judge_error(_judge_reply_router(None), monkeypatch) + cut_off = await self._judge_error( + _judge_reply_router(truncated, finish_reason="length"), monkeypatch + ) + + assert "content=no content" in answered_nothing + assert "finish_reason=stop" in answered_nothing + assert f"content={len(truncated)} chars" in cut_off + assert "finish_reason=length" in cut_off + + async def test_an_unparseable_verdict_names_the_model_that_served_it(self, monkeypatch: pytest.MonkeyPatch): + """A judge_model that fans out over deployments hides which one truncates: without + the served model the operator cannot tell a bad deployment from a bad cap.""" + error = await self._judge_error(_judge_reply_router(None, served_model="claude-sonnet-5"), monkeypatch) + + assert "model=claude-sonnet-5" in error + + async def test_a_diagnosed_verdict_error_stays_groupable(self, monkeypatch: pytest.MonkeyPatch): + """The customer groups attempt rows by error text. Every varying part has to sit + after the first semicolon or each row becomes its own group.""" + first = await self._judge_error(_judge_reply_router(None, served_model="model-a"), monkeypatch) + second = await self._judge_error(_judge_reply_router(None, served_model="model-b"), monkeypatch) + + assert first != second + assert first.split(";")[0] == second.split(";")[0] + + async def test_a_judge_reply_that_cannot_be_read_still_records_an_error(self, monkeypatch: pytest.MonkeyPatch): + """The shape reader runs inside the failure path: it must never raise a second time + and cost the row entirely.""" + router = MagicMock() + router.model_group_alias = {} + router.get_model_list = MagicMock(return_value=[{"litellm_params": {"model": "openai/gpt-4o-mini"}}]) + + async def acompletion(**kwargs): + if kwargs["metadata"].get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == SHADOW_EVAL_ROUTER_CALL_ORIGIN: + kwargs["metadata"]["routing_decision"] = {"tier_label": "SIMPLE", "routed_model": "cheap-model"} + return {"choices": [{"message": {"content": "shadow answer"}}]} + return {"choices": []} + + router.acompletion = MagicMock(side_effect=acompletion) + + error = await self._judge_error(router, monkeypatch) + + assert "unparseable judge verdict" in error + assert "unreadable judge reply" in error + async def test_an_empty_shadow_reply_still_bills_its_cost(self, monkeypatch: pytest.MonkeyPatch): """A shadow call that returns no extractable text has still billed; pricing it at zero would keep the dollar gate open while shadow calls keep charging the key.""" @@ -1287,6 +1403,34 @@ class TestShadowPipeline: assert row["shadow_cost"] == 0.007 assert logger._test_counter["spend:shadow_eval:job-1"] == 0.007 + async def test_the_judge_output_cap_leaves_room_for_a_reasoning_judge(self): + """The output cap covers reasoning tokens as well as the answer, and a judge_model + deployment carrying an elevated reasoning_effort spends that budget before it writes + anything. A cap sized for the verdict JSON alone goes entirely to reasoning and the + reply arrives empty, which the attempt records as an unparseable verdict rather than + a result. The judge here burns a reasoning budget a live claude-sonnet-5 call was + measured at, so the cap has to clear it for the verdict to survive.""" + reasoning_tokens = 2000 + logger = _logger(router=_reasoning_judge_router(reasoning_tokens), prisma=(prisma := _prisma())) + + await logger._run_shadow_eval( + job=_job(), + request_id="req-1", + messages=({"role": "user", "content": "hi"},), + real_text="real answer", + real_model="claude-opus", + real_cost=0.0, + real_classifier_cost=0.0, + real_cache_hit=False, + control_tier=None, + shadow_params={}, + parent_metadata={}, + ) + + row = prisma.db.litellm_shadowevalattempt.create.call_args.kwargs["data"] + assert row["outcome"] in ("real", "shadow", "tie"), row["error"] + assert row["error"] is None + async def _no_text_error(self, router) -> str: prisma = _prisma() await _logger(router=router, prisma=prisma)._run_shadow_eval(