fix(session_replay): allowlist replayed params, repair tool ordering, harden judge and job writes

This commit is contained in:
tin 2026-09-10 14:12:47 -07:00
parent 2225c6d433
commit ca759fcbc3
6 changed files with 252 additions and 65 deletions

View file

@ -12,7 +12,6 @@ from typing import TYPE_CHECKING, Annotated, Final, Protocol
import fastapi
from fastapi import APIRouter, Depends, HTTPException, status
from prisma import Json
from litellm._logging import verbose_proxy_logger
from litellm.integrations.shadow_eval_logger import (
@ -44,7 +43,7 @@ if TYPE_CHECKING:
router: Final = APIRouter()
MAX_CONCURRENT_SESSION_REPLAYS: Final = 4
STALE_JOB_AFTER: Final = timedelta(minutes=30)
STALE_JOB_AFTER: Final = timedelta(hours=3)
_REPLAYABLE_CALL_TYPE: Final = "anthropic_messages"
@ -56,11 +55,16 @@ class _SessionReplayJobRow(Protocol):
class _SessionReplayJobTable(Protocol):
async def create(self, data: Mapping[str, object]) -> _SessionReplayJobRow: ...
async def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> object: ...
async def update_many(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> int: ...
def _json_column(value: object) -> object:
"""prisma is an optional dependency of the base install, so it stays a lazy import."""
from prisma import Json
return Json(value)
def _session_replay_jobs(prisma_client: PrismaClient) -> _SessionReplayJobTable:
return prisma_client.db.litellm_sessionreplayjob # pyright: ignore[reportAttributeAccessIssue] # generated client
@ -179,10 +183,12 @@ def _judge_caller(llm_router: Router, judge_model: str, parent_metadata: Mapping
async def _reap_stale_jobs(prisma_client: PrismaClient) -> None:
"""Fail jobs whose pod died mid-run.
"""Fail jobs whose pod died mid-run, so the partial unique index does not lock a session out forever.
Progress is written after every turn, so a running job with a stale updated_at is
abandoned. Without this the partial unique index would lock that session out forever.
The heartbeat only ticks between turns, and a single arm or judge call can legitimately run
for the router's whole timeout, so the window sits well above one call rather than near it.
Reaping is still a guess, which is why the terminal writes below are status-guarded: a job
reaped while it was in fact alive cannot overwrite the row a later replay now owns.
"""
cutoff: Final = datetime.now(timezone.utc) - STALE_JOB_AFTER
await _session_replay_jobs(prisma_client).update_many(
@ -208,9 +214,10 @@ async def _run_job(
jobs: Final = _session_replay_jobs(prisma_client)
async def on_progress(turns_completed: int) -> None:
await jobs.update(
where={"id": job_id}, data={"turns_completed": turns_completed}
) # mutable-ok: Prisma update input
await jobs.update_many(
where={"id": job_id, "status": "running"}, # mutable-ok: Prisma filter
data={"turns_completed": turns_completed}, # mutable-ok: Prisma update input
)
try:
outcome: Final = await replay_session(
@ -226,8 +233,8 @@ async def _run_job(
)
except Exception as exc: # noqa: BLE001 # the job records its own failure rather than dying silently
verbose_proxy_logger.exception("session replay job %s failed", job_id)
await jobs.update(
where={"id": job_id}, # mutable-ok: Prisma update input
await jobs.update_many(
where={"id": job_id, "status": "running"}, # mutable-ok: Prisma filter
data={ # mutable-ok: Prisma update input
"status": "failed",
"error": str(exc)[:1000],
@ -235,11 +242,11 @@ async def _run_job(
},
)
return
await jobs.update(
where={"id": job_id}, # mutable-ok: Prisma update input
await jobs.update_many(
where={"id": job_id, "status": "running"}, # mutable-ok: Prisma filter
data={ # mutable-ok: Prisma update input
"status": "completed",
"result": Json(outcome.model_dump()),
"result": _json_column(outcome.model_dump()),
"turns_completed": sum(len(run.turns) for run in outcome.arms),
"finished_at": datetime.now(timezone.utc),
},
@ -260,10 +267,7 @@ def _require_admin_viewer(user_api_key_dict: UserAPIKeyAuth, action: str) -> Non
def _job_response(row: Mapping[str, object]) -> SessionReplayJobResponse:
result: Final = row.get("result")
payload: Final = (
result if isinstance(result, Mapping) else {}
) # mutable-ok: empty stand-in for an unfinished job's result
payload: Final = _as_mapping(row.get("result")) or {} # mutable-ok: stand-in for an unfinished job's result
verdict: Final = payload.get("verdict")
fidelity: Final = payload.get("fidelity")
arms: Final = payload.get("arms")
@ -350,7 +354,7 @@ async def start_session_replay(
data={ # mutable-ok: Prisma create input
"session_id": data.session_id,
"status": "running",
"arms": Json([arm.model_dump() for arm in data.arms]), # mutable-ok: Prisma Json column payload
"arms": _json_column([arm.model_dump() for arm in data.arms]), # mutable-ok: Prisma Json column payload
"judge_model": data.judge_model,
"max_turns": data.max_turns,
"source_request_id": source.request_id,
@ -369,7 +373,7 @@ async def start_session_replay(
detail=f"could not create the replay job: {exc}",
) from exc
parent_metadata: Final = {
parent_metadata: Final = { # mutable-ok: forwardable call metadata, an SDK kwarg
"user_api_key_user_id": user_api_key_dict.user_id
} # mutable-ok: forwardable call metadata, an SDK kwarg
asyncio.create_task( # detached job; its own handler records terminal state on the row

View file

@ -99,13 +99,13 @@ def _request_body(
return { # mutable-ok: Anthropic Messages wire body
**{key: value for key, value in transcript.sampling_params}, # mutable-ok: Anthropic Messages wire body
"model": arm.model,
"messages": [
"messages": [ # mutable-ok: Anthropic Messages wire body
message.model_dump(exclude_none=True) for message in messages
], # mutable-ok: Anthropic Messages wire body
"system": [
"system": [ # mutable-ok: Anthropic Messages wire body
block.model_dump(exclude_none=True) for block in transcript.system
], # mutable-ok: Anthropic Messages wire body
"tools": [
"tools": [ # mutable-ok: Anthropic Messages wire body
tool.model_dump(exclude_none=True) for tool in transcript.tools
], # mutable-ok: Anthropic Messages wire body
"stream": False,
@ -141,6 +141,7 @@ async def run_arm(
raw = await call_model(_request_body(transcript, arm, history, session_id))
response = ArmResponse.model_validate(raw)
except Exception as exc: # noqa: BLE001 # a failed turn is recorded, never fatal to the job
history.pop()
turns.append(_failed_turn(index, str(exc)))
await on_progress(len(turns))
continue
@ -179,7 +180,15 @@ def _judge_prompt(human_asks: Sequence[str], runs: Sequence[SessionReplayArmResp
def _unmask(position: str, order: Sequence[int], runs: Sequence[SessionReplayArmResponse]) -> str | None:
index: Final = ord(position.strip().upper()) - ord("A")
"""Map the judge's blind A/B label back to an arm, or give up.
The judge writes this string, so it can be anything; anything unrecognized is a recorded
verdict error rather than an exception, since every arm has already been billed by here.
"""
label: Final = position.strip().upper()
if len(label) != 1:
return None
index: Final = ord(label) - ord("A")
if index < 0 or index >= len(order):
return None
return runs[order[index]].label
@ -226,7 +235,7 @@ async def judge_runs(
if parsed is None:
return SessionReplayVerdictResponse(error=f"unparseable judge response: {raw[:200]}")
position, confidence, reasoning = parsed
if position == "tie":
if position.strip().casefold() == "tie":
return SessionReplayVerdictResponse(winner="tie", confidence=confidence, reasoning=reasoning)
unmasked: Final = _unmask(position, order, runs)
if unmasked is None:

View file

@ -18,19 +18,21 @@ from pydantic import BaseModel, ConfigDict
TRUNCATION_MARKER: Final = "litellm_truncated"
_PROXY_INJECTED_KEYS: Final = frozenset(
REPLAYABLE_SAMPLING_KEYS: Final = frozenset(
{
"headers",
"litellm_metadata",
"litellm_session_id",
"litellm_trace_id",
"metadata",
"provider_specific_header",
"secret_fields",
"max_tokens",
"temperature",
"top_p",
"top_k",
"stop_sequences",
"thinking",
"tool_choice",
"output_config",
"service_tier",
}
)
_STRUCTURAL_KEYS: Final = frozenset({"model", "messages", "system", "tools", "stream"})
UNANSWERED_TOOL_RESULT: Final = "[no recorded result]"
_SYSTEM_REMINDER: Final = re.compile(r"<system-reminder>.*?</system-reminder>", re.DOTALL)
@ -42,7 +44,7 @@ class ContentBlock(BaseModel):
text: str | None = None
id: str | None = None
tool_use_id: str | None = None
content: str | None = None
content: str | tuple[ContentBlock, ...] | None = None
class ToolDefinition(BaseModel):
@ -91,13 +93,19 @@ class ReplayTranscript:
def _count_truncations(value: object) -> int:
if isinstance(value, str):
return 1 if TRUNCATION_MARKER in value else 0
if isinstance(value, dict):
return sum(_count_truncations(item) for item in value.values())
if isinstance(value, (list, tuple)):
return sum(_count_truncations(item) for item in value)
return 0
"""Walk the stored body iteratively; a recursive walk trips the CPU-spike CI gate."""
pending: Final[list[object]] = [value] # mutable-ok: explicit stack replaces recursion
marked: Final[list[str]] = [] # mutable-ok: one entry per truncated string
while pending:
current = pending.pop()
if isinstance(current, str):
if TRUNCATION_MARKER in current:
marked.append(current)
elif isinstance(current, dict):
pending.extend(current.values())
elif isinstance(current, (list, tuple)):
pending.extend(current)
return len(marked)
def _as_system_blocks(system: str | tuple[ContentBlock, ...]) -> tuple[ContentBlock, ...]:
@ -119,6 +127,10 @@ def build_transcript(recorded_body: RecordedBody) -> ReplayTranscript:
`role: "system"` messages are hoisted rather than dropped: the Anthropic Messages API
rejects that role inside `messages`, but the text is real context the recorded model saw.
Sampling parameters are allowlisted, never filtered. The recorded body is caller-controlled
and keeps transport fields the snapshot does not strip, `api_base` among them, so forwarding
everything unrecognized would let a seeded session redirect an admin's replay traffic.
"""
hoisted: Final = tuple(
_hoisted_system_block(message) for message in recorded_body.messages if message.role == "system"
@ -130,7 +142,7 @@ def build_transcript(recorded_body: RecordedBody) -> ReplayTranscript:
sampling: Final = tuple(
(key, value)
for key, value in recorded_body.model_dump(exclude_none=True).items()
if key not in _PROXY_INJECTED_KEYS and key not in _STRUCTURAL_KEYS
if key in REPLAYABLE_SAMPLING_KEYS
)
return ReplayTranscript(
system=_as_system_blocks(recorded_body.system) + hoisted,
@ -150,30 +162,31 @@ def tool_use_ids(assistant_blocks: Sequence[ContentBlock]) -> tuple[str, ...]:
def attach_turn(turn: RecordedMessage, pending_tool_use_ids: Sequence[str]) -> RecordedMessage | None:
"""Bind a recorded user turn onto the arm's own trajectory, or drop it as unattachable.
Recorded `tool_result` blocks carry the original run's `tool_use_id`s, which name
`tool_use` blocks this arm never emitted. When the arm did call tools the results are
rebound onto its ids in order; when it called none there is nothing to bind to, and
injecting the recording's tool output as prose would feed the arm answers to questions
it never asked, so the turn is dropped instead.
The Anthropic Messages API requires every `tool_use` an assistant emits to be answered by a
`tool_result` carrying that exact id, in the very next user message. The recording answers
the ids the ORIGINAL run emitted, so once an arm diverges the two no longer line up in
either direction, and both mismatches are a provider 400 rather than a quality question:
- the arm called tools the recording does not answer, including the case where the next
recorded turn is ordinary text, so every unanswered id gets a stub
- the recording answers tools the arm never called, so the surplus results are dropped
rather than piled onto the last pending id, which would repeat one id
A turn carrying only tool results while the arm called nothing has nothing to bind to, and
replaying the recording's tool output as prose would hand the arm answers to questions it
never asked, so that turn is dropped.
"""
blocks: Final = turn.blocks()
results: Final = tuple(block for block in blocks if block.type == "tool_result")
if not results:
return turn
others: Final = tuple(block for block in blocks if block.type != "tool_result")
if not pending_tool_use_ids:
return None
return None if results else turn
rebound: Final = tuple(
block.model_copy(
update={"tool_use_id": pending_tool_use_ids[min(index, len(pending_tool_use_ids) - 1)]}
) # mutable-ok: pydantic model_copy update mapping
if block.type == "tool_result"
else block
for index, block in enumerate(blocks)
result.model_copy(update={"tool_use_id": tool_id}) # mutable-ok: pydantic model_copy update mapping
for tool_id, result in zip(pending_tool_use_ids, results)
)
answered: Final = frozenset(block.tool_use_id for block in rebound if block.type == "tool_result")
stubs: Final = tuple(
ContentBlock(type="tool_result", tool_use_id=tool_id, content="[no recorded result]")
for tool_id in pending_tool_use_ids
if tool_id not in answered
ContentBlock(type="tool_result", tool_use_id=tool_id, content=UNANSWERED_TOOL_RESULT)
for tool_id in pending_tool_use_ids[len(rebound) :]
)
return RecordedMessage(role="user", content=rebound + stubs)
return RecordedMessage(role="user", content=rebound + stubs + others)

View file

@ -213,3 +213,34 @@ async def test_jsonb_columns_returned_as_text_are_still_parsed():
assert isinstance(result, _SourceRequest)
assert len(result.body.messages) == 1
def test_finished_job_whose_result_comes_back_as_text_is_still_parsed():
"""Same driver behaviour as the source body: a jsonb column can arrive as raw JSON text,
and treating it as absent would silently drop the arms, verdict and fidelity from GET."""
result = {
"human_asks": ["plan my week"],
"arms": [{"label": "auto_router", "model": "r", "final_text": "a", "turns": []}],
"verdict": {"winner": "auto_router", "confidence": 0.72, "reasoning": "more complete"},
"fidelity": {
"truncated_strings": 6,
"recorded_prompt_tokens": 41920,
"replayed_first_turn_prompt_tokens": 16016,
},
}
response = _job_response(
{
"id": "job-3",
"session_id": "s",
"status": "completed",
"judge_model": "j",
"max_turns": 5,
"result": json.dumps(result),
}
)
assert response.verdict is not None and response.verdict.winner == "auto_router"
assert response.fidelity is not None and response.fidelity.truncated_strings == 6
assert tuple(arm.label for arm in response.arms) == ("auto_router",)
assert response.human_asks == ("plan my week",)

View file

@ -303,3 +303,50 @@ async def test_arm_token_totals_survive_serialization():
assert run.model_dump()["prompt_tokens"] == 20
assert run.model_dump()["output_tokens"] == 10
@pytest.mark.asyncio
async def test_failed_turn_does_not_leave_an_unanswered_user_message_in_history():
"""A failed call produces no assistant reply, so keeping its user turn would send two
consecutive user messages on the next turn and rebind results onto ids no longer pending."""
caller = _RecordingCaller([_tool_response("toolu_ARM"), RuntimeError("upstream 500"), _text_response("ok")])
await run_arm(_transcript(), SessionReplayArmSpec(label="a", model="m"), max_turns=10, session_id="s", call_model=caller)
for request in caller.requests:
roles = [message["role"] for message in request["messages"]]
assert all(a != b for a, b in zip(roles, roles[1:])), roles
@pytest.mark.asyncio
@pytest.mark.parametrize("winner", ["Tie", "TIE", "CANDIDATE A", "", "neither", "A "])
async def test_any_judge_winner_string_is_a_recorded_outcome_never_an_exception(winner):
"""Every arm is already billed by the time the judge answers, so an unparseable winner must
not take the whole job down with it."""
caller = _RecordingCaller([_text_response("x"), _text_response("y")])
run = await run_arm(
_transcript(), SessionReplayArmSpec(label="a", model="m"), max_turns=10, session_id="s", call_model=caller
)
async def call_judge(system_prompt, user_prompt):
return '{"winner": "%s", "confidence": 0.5, "reasoning": "r"}' % winner
verdict = await judge_runs(("ask",), (run, run), call_judge, random.Random(0))
assert verdict.winner in (None, "tie", "a")
@pytest.mark.asyncio
async def test_tie_is_recognised_whatever_case_the_judge_used():
caller = _RecordingCaller([_text_response("x"), _text_response("y")])
run = await run_arm(
_transcript(), SessionReplayArmSpec(label="a", model="m"), max_turns=10, session_id="s", call_model=caller
)
async def call_judge(system_prompt, user_prompt):
return '{"winner": "Tie", "confidence": 0.4, "reasoning": "same"}'
verdict = await judge_runs(("ask",), (run, run), call_judge, random.Random(0))
assert verdict.winner == "tie"
assert verdict.error is None

View file

@ -1,5 +1,6 @@
from litellm.proxy.session_replay.transcript import (
RecordedBody,
RecordedMessage,
attach_turn,
build_transcript,
tool_use_ids,
@ -119,11 +120,49 @@ def test_tool_calls_the_recording_never_answered_get_stubbed():
assert [block.tool_use_id for block in attached.blocks()] == ["toolu_ARM", "toolu_EXTRA"]
def test_plain_user_turn_passes_through_untouched():
def test_plain_user_turn_passes_through_when_no_tool_call_is_outstanding():
transcript = _transcript()
plain = next(turn for turn in transcript.user_turns if "tool_result" not in turn.block_types())
assert attach_turn(plain, ("toolu_ARM",)) is plain
assert attach_turn(plain, ()) is plain
def test_plain_user_turn_still_answers_an_outstanding_tool_call():
"""Anthropic rejects a plain user message straight after an unanswered tool_use, so an arm
that called a tool where the recording next has ordinary text would 400 on the following turn."""
transcript = _transcript()
plain = next(turn for turn in transcript.user_turns if "tool_result" not in turn.block_types())
attached = attach_turn(plain, ("toolu_ARM",))
assert attached is not None
kinds = [block.type for block in attached.blocks()]
assert kinds[0] == "tool_result"
assert attached.blocks()[0].tool_use_id == "toolu_ARM"
assert "text" in kinds
assert "Plan my week" in attached.text()
def test_surplus_recorded_results_are_dropped_not_piled_onto_the_last_id():
"""Mapping extras onto the final pending id repeats one tool_use_id in a single message,
which the provider rejects."""
turn = RecordedMessage.model_validate(
{
"role": "user",
"content": [
{"type": "tool_result", "tool_use_id": "orig_1", "content": "a"},
{"type": "tool_result", "tool_use_id": "orig_2", "content": "b"},
{"type": "tool_result", "tool_use_id": "orig_3", "content": "c"},
],
}
)
attached = attach_turn(turn, ("toolu_ARM",))
assert attached is not None
ids = [block.tool_use_id for block in attached.blocks()]
assert ids == ["toolu_ARM"]
assert len(ids) == len(set(ids))
def test_tool_use_ids_reads_only_tool_use_blocks():
@ -132,3 +171,47 @@ def test_tool_use_ids_reads_only_tool_use_blocks():
assert tool_use_ids(assistant_blocks) == ("toolu_ORIGINAL",)
assert tool_use_ids(transcript.system) == ()
def test_transport_fields_never_reach_the_replay():
"""The recorded body is caller-controlled and the spend-log snapshot strips api_key but not
api_base, so forwarding unrecognized keys would let a seeded session redirect an admin's
replay traffic to an attacker-chosen endpoint."""
hostile = dict(RECORDED_BODY) | {
"api_base": "https://attacker.example",
"api_key": "sk-leak",
"base_url": "https://attacker.example",
"custom_llm_provider": "openai",
"extra_headers": {"x-exfil": "1"},
"vertex_project": "someone-elses",
}
transcript = build_transcript(RecordedBody.model_validate(hostile))
keys = {key for key, _ in transcript.sampling_params}
assert keys.isdisjoint(
{"api_base", "api_key", "base_url", "custom_llm_provider", "extra_headers", "vertex_project"}
)
assert {"max_tokens", "thinking"} <= keys
def test_tool_result_content_may_be_a_list_of_blocks():
"""Anthropic tool results carry either a string or a list of content blocks, and rejecting
the list form would make every session using it unreplayable."""
turn = RecordedMessage.model_validate(
{
"role": "user",
"content": [
{
"type": "tool_result",
"tool_use_id": "orig_1",
"content": [{"type": "text", "text": "file body"}],
}
],
}
)
attached = attach_turn(turn, ("toolu_ARM",))
assert attached is not None
assert attached.blocks()[0].tool_use_id == "toolu_ARM"