diff --git a/litellm/integrations/shadow_eval_logger.py b/litellm/integrations/shadow_eval_logger.py index 99d5ab47f1a..1797ac0da14 100644 --- a/litellm/integrations/shadow_eval_logger.py +++ b/litellm/integrations/shadow_eval_logger.py @@ -9,6 +9,7 @@ across pods or stop races; the hook reads active jobs through a short-TTL cache. import asyncio import hashlib import random +import traceback from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timezone @@ -55,7 +56,7 @@ _MAX_JUDGE_PROMPT_CHARS: Final = 24_000 # The judge answers with a small JSON object; a tighter budget truncates the JSON # mid-object and the attempt is lost to an error row. -JUDGE_MAX_OUTPUT_TOKENS: Final = 500 +JUDGE_MAX_OUTPUT_TOKENS: Final = 1500 _MAX_ERROR_CHARS: Final = 500 @@ -325,6 +326,14 @@ def _sample_hits(request_id: str, job_id: str, percentage: float) -> bool: return bucket * 100.0 < percentage +def _failure_detail(e: BaseException) -> str: + """Exception class, message, and the raising frame, so an attempt's error row names + the faulty code path without needing debug logs on the pod.""" + frames: Final = traceback.extract_tb(e.__traceback__) + location: Final = f" at {frames[-1].filename.rsplit('/', 1)[-1]}:{frames[-1].lineno}" if frames else "" + return f"{type(e).__name__}{location}: {e}" + + def _judge_call_cost(response: object) -> float: """Price a judge call, treating an unmapped judge model as free rather than fatal.""" import litellm @@ -764,7 +773,9 @@ class ShadowEvalLogger(CustomLogger): try: response: Final = await router.acompletion( model=target_model, - messages=messages, # pyright: ignore[reportArgumentType] # snapshot of the SDK's own message dicts + messages=[ # mutable-ok: provider transforms rewrite messages in place, so the router gets its own copy + dict(m) for m in messages + ], # pyright: ignore[reportArgumentType] # snapshot of the SDK's own message dicts metadata=shadow_metadata, num_retries=0, fallbacks=[], # mutable-ok: SDK kwarg; a failed shadow is a recorded error, never a spend multiplier @@ -772,7 +783,7 @@ class ShadowEvalLogger(CustomLogger): ) except Exception as e: # noqa: BLE001 # provider errors become error rows, not crashes verbose_logger.debug("shadow_eval: router call failed: %s", e) - return _CallFailure(f"shadow router call failed: {e}") + return _CallFailure(f"shadow router call failed: {_failure_detail(e)}") text: Final = _chat_final_text(response) if not text: return _CallFailure("shadow router returned an empty response") diff --git a/tests/test_litellm/integrations/test_shadow_eval_logger.py b/tests/test_litellm/integrations/test_shadow_eval_logger.py index 8d2f9482fa7..ce284a7b201 100644 --- a/tests/test_litellm/integrations/test_shadow_eval_logger.py +++ b/tests/test_litellm/integrations/test_shadow_eval_logger.py @@ -12,10 +12,12 @@ from litellm.caching.in_memory_cache import InMemoryCache from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY from litellm.integrations.shadow_eval_logger import ( _MAX_CONCURRENT_SHADOW_TASKS, + _MAX_ERROR_CHARS, _MAX_JUDGE_PROMPT_CHARS, JUDGE_MAX_OUTPUT_TOKENS, ActiveShadowEvalJob, ShadowEvalLogger, + _failure_detail, _judge_user_prompt, _sample_hits, _unmask_preference, @@ -438,6 +440,21 @@ def test_unmask_preference(raw, real_is_a, expected): assert _unmask_preference(raw, real_is_a) == expected +def test_failure_detail_names_the_raising_frame(): + try: + raise TypeError("'tuple' object does not support item assignment") + except TypeError as e: + detail = _failure_detail(e) + lineno = e.__traceback__.tb_lineno + assert detail == f"TypeError at test_shadow_eval_logger.py:{lineno}: 'tuple' object does not support item assignment" + + try: + raise ValueError("p" * 5 * _MAX_ERROR_CHARS) + except ValueError as long_e: + truncated_row_error = _failure_detail(long_e)[:_MAX_ERROR_CHARS] + assert "ValueError at test_shadow_eval_logger.py:" in truncated_row_error + + def test_judge_prompt_is_bounded_however_large_the_inputs(): prompt = _judge_user_prompt("c" * 200_000, "a" * 200_000, "b" * 200_000) assert len(prompt) < _MAX_JUDGE_PROMPT_CHARS + 100 @@ -476,6 +493,61 @@ class TestSuccessHookSkipChain: assert row["error"] is None assert prisma.db.litellm_shadowevaljob.find_many.await_count == 0 + async def test_shadow_call_messages_survive_in_place_provider_rewrites(self, monkeypatch: pytest.MonkeyPatch): + """Provider transforms (anthropic factory, cache-control hook) rewrite messages with + `messages[i] = ...`; the logger's immutable snapshot must never reach them directly.""" + import litellm as litellm_module + + monkeypatch.setattr(litellm_module, "completion_cost", lambda completion_response: 0.005) + prisma = _prisma() + router = _router() + inner = router.acompletion.side_effect + + async def mutating_acompletion(**kwargs): + kwargs["messages"][0] = dict(kwargs["messages"][0]) + return await inner(**kwargs) + + router.acompletion = MagicMock(side_effect=mutating_acompletion) + logger = _logger(router=router, prisma=prisma, jobs=(_job(),)) + + await logger.async_log_success_event(_success_kwargs(), RESPONSE, None, None) + await _drain(logger) + + row = prisma.db.litellm_shadowevalattempt.create.call_args.kwargs["data"] + assert row["error"] is None + assert row["outcome"] in ("real", "shadow", "tie") + + async def test_pipeline_continues_judging_after_a_failed_attempt(self, monkeypatch: pytest.MonkeyPatch): + import litellm as litellm_module + + monkeypatch.setattr(litellm_module, "completion_cost", lambda completion_response: 0.005) + prisma = _prisma() + router = _router() + inner = router.acompletion.side_effect + shadow_calls = {"count": 0} + + async def flaky_acompletion(**kwargs): + if kwargs["metadata"].get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == SHADOW_EVAL_ROUTER_CALL_ORIGIN: + shadow_calls["count"] += 1 + if shadow_calls["count"] == 1: + raise RuntimeError("provider exploded") + return await inner(**kwargs) + + router.acompletion = MagicMock(side_effect=flaky_acompletion) + logger = _logger(router=router, prisma=prisma, jobs=(_job(),)) + + await logger.async_log_success_event(_success_kwargs(), RESPONSE, None, None) + await _drain(logger) + await logger.async_log_success_event(_success_kwargs(request_id="req-2"), RESPONSE, None, None) + await _drain(logger) + + rows = [c.kwargs["data"] for c in prisma.db.litellm_shadowevalattempt.create.call_args_list] + assert [rows[0]["outcome"], rows[1]["outcome"] in ("real", "shadow")] == ["error", True] + assert "provider exploded" in rows[0]["error"] + assert rows[1]["request_id"] == "req-2" + assert rows[1]["error"] is None + assert logger._inflight_shadow_tasks == 0 + @pytest.mark.parametrize( "kwargs_mutation,job_mutation", [