From ca759fcbc34a63ec45a724e4dda0707ffc501280 Mon Sep 17 00:00:00 2001 From: tin Date: Thu, 10 Sep 2026 14:12:47 -0700 Subject: [PATCH] fix(session_replay): allowlist replayed params, repair tool ordering, harden judge and job writes --- .../session_replay_endpoints.py | 46 +++++----- litellm/proxy/session_replay/executor.py | 19 ++-- litellm/proxy/session_replay/transcript.py | 87 +++++++++++-------- .../test_session_replay_endpoints.py | 31 +++++++ .../proxy/session_replay/test_executor.py | 47 ++++++++++ .../proxy/session_replay/test_transcript.py | 87 ++++++++++++++++++- 6 files changed, 252 insertions(+), 65 deletions(-) diff --git a/litellm/proxy/management_endpoints/session_replay_endpoints.py b/litellm/proxy/management_endpoints/session_replay_endpoints.py index 9567d2cdef8..043c301c287 100644 --- a/litellm/proxy/management_endpoints/session_replay_endpoints.py +++ b/litellm/proxy/management_endpoints/session_replay_endpoints.py @@ -12,7 +12,6 @@ from typing import TYPE_CHECKING, Annotated, Final, Protocol import fastapi from fastapi import APIRouter, Depends, HTTPException, status -from prisma import Json from litellm._logging import verbose_proxy_logger from litellm.integrations.shadow_eval_logger import ( @@ -44,7 +43,7 @@ if TYPE_CHECKING: router: Final = APIRouter() MAX_CONCURRENT_SESSION_REPLAYS: Final = 4 -STALE_JOB_AFTER: Final = timedelta(minutes=30) +STALE_JOB_AFTER: Final = timedelta(hours=3) _REPLAYABLE_CALL_TYPE: Final = "anthropic_messages" @@ -56,11 +55,16 @@ class _SessionReplayJobRow(Protocol): class _SessionReplayJobTable(Protocol): async def create(self, data: Mapping[str, object]) -> _SessionReplayJobRow: ... - async def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> object: ... - async def update_many(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> int: ... +def _json_column(value: object) -> object: + """prisma is an optional dependency of the base install, so it stays a lazy import.""" + from prisma import Json + + return Json(value) + + def _session_replay_jobs(prisma_client: PrismaClient) -> _SessionReplayJobTable: return prisma_client.db.litellm_sessionreplayjob # pyright: ignore[reportAttributeAccessIssue] # generated client @@ -179,10 +183,12 @@ def _judge_caller(llm_router: Router, judge_model: str, parent_metadata: Mapping async def _reap_stale_jobs(prisma_client: PrismaClient) -> None: - """Fail jobs whose pod died mid-run. + """Fail jobs whose pod died mid-run, so the partial unique index does not lock a session out forever. - Progress is written after every turn, so a running job with a stale updated_at is - abandoned. Without this the partial unique index would lock that session out forever. + The heartbeat only ticks between turns, and a single arm or judge call can legitimately run + for the router's whole timeout, so the window sits well above one call rather than near it. + Reaping is still a guess, which is why the terminal writes below are status-guarded: a job + reaped while it was in fact alive cannot overwrite the row a later replay now owns. """ cutoff: Final = datetime.now(timezone.utc) - STALE_JOB_AFTER await _session_replay_jobs(prisma_client).update_many( @@ -208,9 +214,10 @@ async def _run_job( jobs: Final = _session_replay_jobs(prisma_client) async def on_progress(turns_completed: int) -> None: - await jobs.update( - where={"id": job_id}, data={"turns_completed": turns_completed} - ) # mutable-ok: Prisma update input + await jobs.update_many( + where={"id": job_id, "status": "running"}, # mutable-ok: Prisma filter + data={"turns_completed": turns_completed}, # mutable-ok: Prisma update input + ) try: outcome: Final = await replay_session( @@ -226,8 +233,8 @@ async def _run_job( ) except Exception as exc: # noqa: BLE001 # the job records its own failure rather than dying silently verbose_proxy_logger.exception("session replay job %s failed", job_id) - await jobs.update( - where={"id": job_id}, # mutable-ok: Prisma update input + await jobs.update_many( + where={"id": job_id, "status": "running"}, # mutable-ok: Prisma filter data={ # mutable-ok: Prisma update input "status": "failed", "error": str(exc)[:1000], @@ -235,11 +242,11 @@ async def _run_job( }, ) return - await jobs.update( - where={"id": job_id}, # mutable-ok: Prisma update input + await jobs.update_many( + where={"id": job_id, "status": "running"}, # mutable-ok: Prisma filter data={ # mutable-ok: Prisma update input "status": "completed", - "result": Json(outcome.model_dump()), + "result": _json_column(outcome.model_dump()), "turns_completed": sum(len(run.turns) for run in outcome.arms), "finished_at": datetime.now(timezone.utc), }, @@ -260,10 +267,7 @@ def _require_admin_viewer(user_api_key_dict: UserAPIKeyAuth, action: str) -> Non def _job_response(row: Mapping[str, object]) -> SessionReplayJobResponse: - result: Final = row.get("result") - payload: Final = ( - result if isinstance(result, Mapping) else {} - ) # mutable-ok: empty stand-in for an unfinished job's result + payload: Final = _as_mapping(row.get("result")) or {} # mutable-ok: stand-in for an unfinished job's result verdict: Final = payload.get("verdict") fidelity: Final = payload.get("fidelity") arms: Final = payload.get("arms") @@ -350,7 +354,7 @@ async def start_session_replay( data={ # mutable-ok: Prisma create input "session_id": data.session_id, "status": "running", - "arms": Json([arm.model_dump() for arm in data.arms]), # mutable-ok: Prisma Json column payload + "arms": _json_column([arm.model_dump() for arm in data.arms]), # mutable-ok: Prisma Json column payload "judge_model": data.judge_model, "max_turns": data.max_turns, "source_request_id": source.request_id, @@ -369,7 +373,7 @@ async def start_session_replay( detail=f"could not create the replay job: {exc}", ) from exc - parent_metadata: Final = { + parent_metadata: Final = { # mutable-ok: forwardable call metadata, an SDK kwarg "user_api_key_user_id": user_api_key_dict.user_id } # mutable-ok: forwardable call metadata, an SDK kwarg asyncio.create_task( # detached job; its own handler records terminal state on the row diff --git a/litellm/proxy/session_replay/executor.py b/litellm/proxy/session_replay/executor.py index fe3829b5b5c..812495ad1b0 100644 --- a/litellm/proxy/session_replay/executor.py +++ b/litellm/proxy/session_replay/executor.py @@ -99,13 +99,13 @@ def _request_body( return { # mutable-ok: Anthropic Messages wire body **{key: value for key, value in transcript.sampling_params}, # mutable-ok: Anthropic Messages wire body "model": arm.model, - "messages": [ + "messages": [ # mutable-ok: Anthropic Messages wire body message.model_dump(exclude_none=True) for message in messages ], # mutable-ok: Anthropic Messages wire body - "system": [ + "system": [ # mutable-ok: Anthropic Messages wire body block.model_dump(exclude_none=True) for block in transcript.system ], # mutable-ok: Anthropic Messages wire body - "tools": [ + "tools": [ # mutable-ok: Anthropic Messages wire body tool.model_dump(exclude_none=True) for tool in transcript.tools ], # mutable-ok: Anthropic Messages wire body "stream": False, @@ -141,6 +141,7 @@ async def run_arm( raw = await call_model(_request_body(transcript, arm, history, session_id)) response = ArmResponse.model_validate(raw) except Exception as exc: # noqa: BLE001 # a failed turn is recorded, never fatal to the job + history.pop() turns.append(_failed_turn(index, str(exc))) await on_progress(len(turns)) continue @@ -179,7 +180,15 @@ def _judge_prompt(human_asks: Sequence[str], runs: Sequence[SessionReplayArmResp def _unmask(position: str, order: Sequence[int], runs: Sequence[SessionReplayArmResponse]) -> str | None: - index: Final = ord(position.strip().upper()) - ord("A") + """Map the judge's blind A/B label back to an arm, or give up. + + The judge writes this string, so it can be anything; anything unrecognized is a recorded + verdict error rather than an exception, since every arm has already been billed by here. + """ + label: Final = position.strip().upper() + if len(label) != 1: + return None + index: Final = ord(label) - ord("A") if index < 0 or index >= len(order): return None return runs[order[index]].label @@ -226,7 +235,7 @@ async def judge_runs( if parsed is None: return SessionReplayVerdictResponse(error=f"unparseable judge response: {raw[:200]}") position, confidence, reasoning = parsed - if position == "tie": + if position.strip().casefold() == "tie": return SessionReplayVerdictResponse(winner="tie", confidence=confidence, reasoning=reasoning) unmasked: Final = _unmask(position, order, runs) if unmasked is None: diff --git a/litellm/proxy/session_replay/transcript.py b/litellm/proxy/session_replay/transcript.py index a5ce419ff11..be12013fb8a 100644 --- a/litellm/proxy/session_replay/transcript.py +++ b/litellm/proxy/session_replay/transcript.py @@ -18,19 +18,21 @@ from pydantic import BaseModel, ConfigDict TRUNCATION_MARKER: Final = "litellm_truncated" -_PROXY_INJECTED_KEYS: Final = frozenset( +REPLAYABLE_SAMPLING_KEYS: Final = frozenset( { - "headers", - "litellm_metadata", - "litellm_session_id", - "litellm_trace_id", - "metadata", - "provider_specific_header", - "secret_fields", + "max_tokens", + "temperature", + "top_p", + "top_k", + "stop_sequences", + "thinking", + "tool_choice", + "output_config", + "service_tier", } ) -_STRUCTURAL_KEYS: Final = frozenset({"model", "messages", "system", "tools", "stream"}) +UNANSWERED_TOOL_RESULT: Final = "[no recorded result]" _SYSTEM_REMINDER: Final = re.compile(r".*?", re.DOTALL) @@ -42,7 +44,7 @@ class ContentBlock(BaseModel): text: str | None = None id: str | None = None tool_use_id: str | None = None - content: str | None = None + content: str | tuple[ContentBlock, ...] | None = None class ToolDefinition(BaseModel): @@ -91,13 +93,19 @@ class ReplayTranscript: def _count_truncations(value: object) -> int: - if isinstance(value, str): - return 1 if TRUNCATION_MARKER in value else 0 - if isinstance(value, dict): - return sum(_count_truncations(item) for item in value.values()) - if isinstance(value, (list, tuple)): - return sum(_count_truncations(item) for item in value) - return 0 + """Walk the stored body iteratively; a recursive walk trips the CPU-spike CI gate.""" + pending: Final[list[object]] = [value] # mutable-ok: explicit stack replaces recursion + marked: Final[list[str]] = [] # mutable-ok: one entry per truncated string + while pending: + current = pending.pop() + if isinstance(current, str): + if TRUNCATION_MARKER in current: + marked.append(current) + elif isinstance(current, dict): + pending.extend(current.values()) + elif isinstance(current, (list, tuple)): + pending.extend(current) + return len(marked) def _as_system_blocks(system: str | tuple[ContentBlock, ...]) -> tuple[ContentBlock, ...]: @@ -119,6 +127,10 @@ def build_transcript(recorded_body: RecordedBody) -> ReplayTranscript: `role: "system"` messages are hoisted rather than dropped: the Anthropic Messages API rejects that role inside `messages`, but the text is real context the recorded model saw. + + Sampling parameters are allowlisted, never filtered. The recorded body is caller-controlled + and keeps transport fields the snapshot does not strip, `api_base` among them, so forwarding + everything unrecognized would let a seeded session redirect an admin's replay traffic. """ hoisted: Final = tuple( _hoisted_system_block(message) for message in recorded_body.messages if message.role == "system" @@ -130,7 +142,7 @@ def build_transcript(recorded_body: RecordedBody) -> ReplayTranscript: sampling: Final = tuple( (key, value) for key, value in recorded_body.model_dump(exclude_none=True).items() - if key not in _PROXY_INJECTED_KEYS and key not in _STRUCTURAL_KEYS + if key in REPLAYABLE_SAMPLING_KEYS ) return ReplayTranscript( system=_as_system_blocks(recorded_body.system) + hoisted, @@ -150,30 +162,31 @@ def tool_use_ids(assistant_blocks: Sequence[ContentBlock]) -> tuple[str, ...]: def attach_turn(turn: RecordedMessage, pending_tool_use_ids: Sequence[str]) -> RecordedMessage | None: """Bind a recorded user turn onto the arm's own trajectory, or drop it as unattachable. - Recorded `tool_result` blocks carry the original run's `tool_use_id`s, which name - `tool_use` blocks this arm never emitted. When the arm did call tools the results are - rebound onto its ids in order; when it called none there is nothing to bind to, and - injecting the recording's tool output as prose would feed the arm answers to questions - it never asked, so the turn is dropped instead. + The Anthropic Messages API requires every `tool_use` an assistant emits to be answered by a + `tool_result` carrying that exact id, in the very next user message. The recording answers + the ids the ORIGINAL run emitted, so once an arm diverges the two no longer line up in + either direction, and both mismatches are a provider 400 rather than a quality question: + + - the arm called tools the recording does not answer, including the case where the next + recorded turn is ordinary text, so every unanswered id gets a stub + - the recording answers tools the arm never called, so the surplus results are dropped + rather than piled onto the last pending id, which would repeat one id + + A turn carrying only tool results while the arm called nothing has nothing to bind to, and + replaying the recording's tool output as prose would hand the arm answers to questions it + never asked, so that turn is dropped. """ blocks: Final = turn.blocks() results: Final = tuple(block for block in blocks if block.type == "tool_result") - if not results: - return turn + others: Final = tuple(block for block in blocks if block.type != "tool_result") if not pending_tool_use_ids: - return None + return None if results else turn rebound: Final = tuple( - block.model_copy( - update={"tool_use_id": pending_tool_use_ids[min(index, len(pending_tool_use_ids) - 1)]} - ) # mutable-ok: pydantic model_copy update mapping - if block.type == "tool_result" - else block - for index, block in enumerate(blocks) + result.model_copy(update={"tool_use_id": tool_id}) # mutable-ok: pydantic model_copy update mapping + for tool_id, result in zip(pending_tool_use_ids, results) ) - answered: Final = frozenset(block.tool_use_id for block in rebound if block.type == "tool_result") stubs: Final = tuple( - ContentBlock(type="tool_result", tool_use_id=tool_id, content="[no recorded result]") - for tool_id in pending_tool_use_ids - if tool_id not in answered + ContentBlock(type="tool_result", tool_use_id=tool_id, content=UNANSWERED_TOOL_RESULT) + for tool_id in pending_tool_use_ids[len(rebound) :] ) - return RecordedMessage(role="user", content=rebound + stubs) + return RecordedMessage(role="user", content=rebound + stubs + others) diff --git a/tests/test_litellm/proxy/management_endpoints/test_session_replay_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_session_replay_endpoints.py index 454439a2f78..eb7edf5c6f7 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_session_replay_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_session_replay_endpoints.py @@ -213,3 +213,34 @@ async def test_jsonb_columns_returned_as_text_are_still_parsed(): assert isinstance(result, _SourceRequest) assert len(result.body.messages) == 1 + + +def test_finished_job_whose_result_comes_back_as_text_is_still_parsed(): + """Same driver behaviour as the source body: a jsonb column can arrive as raw JSON text, + and treating it as absent would silently drop the arms, verdict and fidelity from GET.""" + result = { + "human_asks": ["plan my week"], + "arms": [{"label": "auto_router", "model": "r", "final_text": "a", "turns": []}], + "verdict": {"winner": "auto_router", "confidence": 0.72, "reasoning": "more complete"}, + "fidelity": { + "truncated_strings": 6, + "recorded_prompt_tokens": 41920, + "replayed_first_turn_prompt_tokens": 16016, + }, + } + + response = _job_response( + { + "id": "job-3", + "session_id": "s", + "status": "completed", + "judge_model": "j", + "max_turns": 5, + "result": json.dumps(result), + } + ) + + assert response.verdict is not None and response.verdict.winner == "auto_router" + assert response.fidelity is not None and response.fidelity.truncated_strings == 6 + assert tuple(arm.label for arm in response.arms) == ("auto_router",) + assert response.human_asks == ("plan my week",) diff --git a/tests/test_litellm/proxy/session_replay/test_executor.py b/tests/test_litellm/proxy/session_replay/test_executor.py index 6d2b60171f1..08150e738ba 100644 --- a/tests/test_litellm/proxy/session_replay/test_executor.py +++ b/tests/test_litellm/proxy/session_replay/test_executor.py @@ -303,3 +303,50 @@ async def test_arm_token_totals_survive_serialization(): assert run.model_dump()["prompt_tokens"] == 20 assert run.model_dump()["output_tokens"] == 10 + + +@pytest.mark.asyncio +async def test_failed_turn_does_not_leave_an_unanswered_user_message_in_history(): + """A failed call produces no assistant reply, so keeping its user turn would send two + consecutive user messages on the next turn and rebind results onto ids no longer pending.""" + caller = _RecordingCaller([_tool_response("toolu_ARM"), RuntimeError("upstream 500"), _text_response("ok")]) + + await run_arm(_transcript(), SessionReplayArmSpec(label="a", model="m"), max_turns=10, session_id="s", call_model=caller) + + for request in caller.requests: + roles = [message["role"] for message in request["messages"]] + assert all(a != b for a, b in zip(roles, roles[1:])), roles + + +@pytest.mark.asyncio +@pytest.mark.parametrize("winner", ["Tie", "TIE", "CANDIDATE A", "", "neither", "A "]) +async def test_any_judge_winner_string_is_a_recorded_outcome_never_an_exception(winner): + """Every arm is already billed by the time the judge answers, so an unparseable winner must + not take the whole job down with it.""" + caller = _RecordingCaller([_text_response("x"), _text_response("y")]) + run = await run_arm( + _transcript(), SessionReplayArmSpec(label="a", model="m"), max_turns=10, session_id="s", call_model=caller + ) + + async def call_judge(system_prompt, user_prompt): + return '{"winner": "%s", "confidence": 0.5, "reasoning": "r"}' % winner + + verdict = await judge_runs(("ask",), (run, run), call_judge, random.Random(0)) + + assert verdict.winner in (None, "tie", "a") + + +@pytest.mark.asyncio +async def test_tie_is_recognised_whatever_case_the_judge_used(): + caller = _RecordingCaller([_text_response("x"), _text_response("y")]) + run = await run_arm( + _transcript(), SessionReplayArmSpec(label="a", model="m"), max_turns=10, session_id="s", call_model=caller + ) + + async def call_judge(system_prompt, user_prompt): + return '{"winner": "Tie", "confidence": 0.4, "reasoning": "same"}' + + verdict = await judge_runs(("ask",), (run, run), call_judge, random.Random(0)) + + assert verdict.winner == "tie" + assert verdict.error is None diff --git a/tests/test_litellm/proxy/session_replay/test_transcript.py b/tests/test_litellm/proxy/session_replay/test_transcript.py index 1f444faa237..d5585157716 100644 --- a/tests/test_litellm/proxy/session_replay/test_transcript.py +++ b/tests/test_litellm/proxy/session_replay/test_transcript.py @@ -1,5 +1,6 @@ from litellm.proxy.session_replay.transcript import ( RecordedBody, + RecordedMessage, attach_turn, build_transcript, tool_use_ids, @@ -119,11 +120,49 @@ def test_tool_calls_the_recording_never_answered_get_stubbed(): assert [block.tool_use_id for block in attached.blocks()] == ["toolu_ARM", "toolu_EXTRA"] -def test_plain_user_turn_passes_through_untouched(): +def test_plain_user_turn_passes_through_when_no_tool_call_is_outstanding(): transcript = _transcript() plain = next(turn for turn in transcript.user_turns if "tool_result" not in turn.block_types()) - assert attach_turn(plain, ("toolu_ARM",)) is plain + assert attach_turn(plain, ()) is plain + + +def test_plain_user_turn_still_answers_an_outstanding_tool_call(): + """Anthropic rejects a plain user message straight after an unanswered tool_use, so an arm + that called a tool where the recording next has ordinary text would 400 on the following turn.""" + transcript = _transcript() + plain = next(turn for turn in transcript.user_turns if "tool_result" not in turn.block_types()) + + attached = attach_turn(plain, ("toolu_ARM",)) + + assert attached is not None + kinds = [block.type for block in attached.blocks()] + assert kinds[0] == "tool_result" + assert attached.blocks()[0].tool_use_id == "toolu_ARM" + assert "text" in kinds + assert "Plan my week" in attached.text() + + +def test_surplus_recorded_results_are_dropped_not_piled_onto_the_last_id(): + """Mapping extras onto the final pending id repeats one tool_use_id in a single message, + which the provider rejects.""" + turn = RecordedMessage.model_validate( + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "orig_1", "content": "a"}, + {"type": "tool_result", "tool_use_id": "orig_2", "content": "b"}, + {"type": "tool_result", "tool_use_id": "orig_3", "content": "c"}, + ], + } + ) + + attached = attach_turn(turn, ("toolu_ARM",)) + + assert attached is not None + ids = [block.tool_use_id for block in attached.blocks()] + assert ids == ["toolu_ARM"] + assert len(ids) == len(set(ids)) def test_tool_use_ids_reads_only_tool_use_blocks(): @@ -132,3 +171,47 @@ def test_tool_use_ids_reads_only_tool_use_blocks(): assert tool_use_ids(assistant_blocks) == ("toolu_ORIGINAL",) assert tool_use_ids(transcript.system) == () + + +def test_transport_fields_never_reach_the_replay(): + """The recorded body is caller-controlled and the spend-log snapshot strips api_key but not + api_base, so forwarding unrecognized keys would let a seeded session redirect an admin's + replay traffic to an attacker-chosen endpoint.""" + hostile = dict(RECORDED_BODY) | { + "api_base": "https://attacker.example", + "api_key": "sk-leak", + "base_url": "https://attacker.example", + "custom_llm_provider": "openai", + "extra_headers": {"x-exfil": "1"}, + "vertex_project": "someone-elses", + } + + transcript = build_transcript(RecordedBody.model_validate(hostile)) + keys = {key for key, _ in transcript.sampling_params} + + assert keys.isdisjoint( + {"api_base", "api_key", "base_url", "custom_llm_provider", "extra_headers", "vertex_project"} + ) + assert {"max_tokens", "thinking"} <= keys + + +def test_tool_result_content_may_be_a_list_of_blocks(): + """Anthropic tool results carry either a string or a list of content blocks, and rejecting + the list form would make every session using it unreplayable.""" + turn = RecordedMessage.model_validate( + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "orig_1", + "content": [{"type": "text", "text": "file body"}], + } + ], + } + ) + + attached = attach_turn(turn, ("toolu_ARM",)) + + assert attached is not None + assert attached.blocks()[0].tool_use_id == "toolu_ARM"