feat(session_replay): replay a recorded session against model arms and judge it

This commit is contained in:
tin 2026-09-10 13:26:55 -07:00
parent 692a311efb
commit 2225c6d433
17 changed files with 2151 additions and 0 deletions

View file

@ -143,6 +143,7 @@ jobs:
test-path: >-
tests/test_litellm/proxy/analytics_endpoints
tests/test_litellm/proxy/management_endpoints
tests/test_litellm/proxy/session_replay
tests/test_litellm/proxy/list_api
tests/test_litellm/proxy/memory
tests/test_litellm/proxy/guardrails

View file

@ -0,0 +1,28 @@
CREATE TABLE IF NOT EXISTS "LiteLLM_SessionReplayJob" (
"id" TEXT NOT NULL,
"session_id" TEXT NOT NULL,
"status" TEXT NOT NULL DEFAULT 'running',
"arms" JSONB NOT NULL,
"judge_model" TEXT NOT NULL,
"max_turns" INTEGER NOT NULL,
"source_request_id" TEXT,
"turns_completed" INTEGER NOT NULL DEFAULT 0,
"result" JSONB,
"error" TEXT,
"created_by" TEXT,
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"finished_at" TIMESTAMP(3),
CONSTRAINT "LiteLLM_SessionReplayJob_pkey" PRIMARY KEY ("id")
);
CREATE INDEX IF NOT EXISTS "LiteLLM_SessionReplayJob_session_id_idx" ON "LiteLLM_SessionReplayJob"("session_id");
CREATE INDEX IF NOT EXISTS "LiteLLM_SessionReplayJob_created_at_idx" ON "LiteLLM_SessionReplayJob"("created_at");
-- A replay issues real billable calls for every recorded turn on every arm, so two
-- concurrent starts on one session double that spend for no extra signal. Partial
-- indexes are not expressible in schema.prisma, so this lives here only.
CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_SessionReplayJob_one_active_per_session"
ON "LiteLLM_SessionReplayJob"("session_id") WHERE "status" = 'running';

View file

@ -1592,6 +1592,33 @@ model LiteLLM_ShadowEvalFunnel {
withheld Int @default(0)
}
// ---------------------------------------------------------------------------
// Session Replay
//
// One row per replay job: a recorded session re-run against several model arms,
// whose end results are judged blind. Single writer (the job's own task), so
// progress and result live on the row rather than in append-only attempt rows.
// ---------------------------------------------------------------------------
model LiteLLM_SessionReplayJob {
id String @id @default(cuid())
session_id String
status String @default("running") // running | completed | failed | stopped
arms Json
judge_model String
max_turns Int
source_request_id String? // the recorded request whose body seeded the transcript
turns_completed Int @default(0)
result Json?
error String?
created_by String?
created_at DateTime @default(now())
updated_at DateTime @default(now()) @updatedAt
finished_at DateTime?
@@index([session_id])
@@index([created_at])
}
// ---------------------------------------------------------------------------
// Workflow Run Tracking
//

View file

@ -0,0 +1,460 @@
"""Admin endpoints to replay a recorded session against several model arms and judge it."""
from __future__ import annotations
import asyncio
import json
import random
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from typing import TYPE_CHECKING, Annotated, Final, Protocol
import fastapi
from fastapi import APIRouter, Depends, HTTPException, status
from prisma import Json
from litellm._logging import verbose_proxy_logger
from litellm.integrations.shadow_eval_logger import (
_request_mutating_guardrail_ran, # pyright: ignore[reportPrivateUsage] # one owner for the guardrail-mode reading
)
from litellm.litellm_core_utils.internal_call_metadata import sanitized_forwardable_call_metadata
from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.management_endpoints.auto_router_endpoints import (
_is_unique_violation, # pyright: ignore[reportPrivateUsage] # one owner for the unique-violation read
)
from litellm.proxy.session_replay.executor import replay_session
from litellm.proxy.session_replay.transcript import RecordedBody, ReplayTranscript, build_transcript
from litellm.types.management_endpoints.session_replay_endpoints import (
SessionReplayArmResponse,
SessionReplayArmSpec,
SessionReplayFidelityResponse,
SessionReplayJobListResponse,
SessionReplayJobResponse,
SessionReplayVerdictResponse,
StartSessionReplayRequest,
)
from litellm.types.utils import SESSION_REPLAY_ARM_CALL_ORIGIN, SESSION_REPLAY_JUDGE_CALL_ORIGIN
if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient
from litellm.router import Router
router: Final = APIRouter()
MAX_CONCURRENT_SESSION_REPLAYS: Final = 4
STALE_JOB_AFTER: Final = timedelta(minutes=30)
_REPLAYABLE_CALL_TYPE: Final = "anthropic_messages"
class _SessionReplayJobRow(Protocol):
@property
def id(self) -> str: ...
class _SessionReplayJobTable(Protocol):
async def create(self, data: Mapping[str, object]) -> _SessionReplayJobRow: ...
async def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> object: ...
async def update_many(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> int: ...
def _session_replay_jobs(prisma_client: PrismaClient) -> _SessionReplayJobTable:
return prisma_client.db.litellm_sessionreplayjob # pyright: ignore[reportAttributeAccessIssue] # generated client
async def _query_raw(prisma_client: PrismaClient, query: str, *args: object) -> Sequence[Mapping[str, object]]:
return await prisma_client.db.query_raw(query, *args)
@dataclass(frozen=True, slots=True)
class _SourceRequest:
request_id: str
body: RecordedBody
recorded_prompt_tokens: int
@dataclass(frozen=True, slots=True)
class _SourceUnavailable:
status_code: int
reason: str
def _as_mapping(value: object) -> Mapping[str, object] | None:
"""Read a jsonb column back, which query_raw hands over as a dict or as raw JSON text."""
if isinstance(value, Mapping):
return value
if not isinstance(value, str):
return None
try:
decoded: Final = json.loads(value)
except json.JSONDecodeError:
return None
return decoded if isinstance(decoded, Mapping) else None
async def _load_source_request(prisma_client: PrismaClient, session_id: str) -> _SourceRequest | _SourceUnavailable:
"""Pick the recorded request whose stored body carries the fullest transcript.
Turn N's body already embeds turns 1..N-1, so the row with the most prompt tokens is
the whole conversation and every earlier row is a prefix of it.
"""
rows: Final = await _query_raw(
prisma_client,
"""
SELECT request_id, proxy_server_request, prompt_tokens, metadata
FROM "LiteLLM_SpendLogs"
WHERE session_id = $1 AND call_type = $2
ORDER BY prompt_tokens DESC
LIMIT 1
""",
session_id,
_REPLAYABLE_CALL_TYPE,
)
if not rows:
return _SourceUnavailable(
status_code=status.HTTP_404_NOT_FOUND,
reason=f"no {_REPLAYABLE_CALL_TYPE} spend logs found for session {session_id}",
)
row: Final = rows[0]
recorded: Final = _as_mapping(row.get("proxy_server_request"))
if recorded is None or not recorded.get("messages"):
return _SourceUnavailable(
status_code=status.HTTP_409_CONFLICT,
reason=(
"the recorded request body was not stored, so there is nothing to replay. "
"Set general_settings.store_prompts_in_spend_logs to record it"
),
)
metadata: Final = _as_mapping(row.get("metadata"))
if metadata is not None and _request_mutating_guardrail_ran(metadata):
return _SourceUnavailable(
status_code=status.HTTP_409_CONFLICT,
reason="a request-mutating guardrail ran on this session, so the stored body predates the rewrite",
)
prompt_tokens: Final = row.get("prompt_tokens")
return _SourceRequest(
request_id=str(row.get("request_id") or ""),
body=RecordedBody.model_validate(recorded),
recorded_prompt_tokens=prompt_tokens if isinstance(prompt_tokens, int) else 0,
)
def _arm_caller(llm_router: Router, parent_metadata: Mapping[str, object]):
async def call(request: Mapping[str, object]) -> Mapping[str, object]:
return await llm_router.aanthropic_messages(
**request,
metadata=sanitized_forwardable_call_metadata(parent_metadata, SESSION_REPLAY_ARM_CALL_ORIGIN),
num_retries=0,
fallbacks=[], # mutable-ok: SDK kwarg
)
return call
def _judge_caller(llm_router: Router, judge_model: str, parent_metadata: Mapping[str, object]):
async def call(system_prompt: str, user_prompt: str) -> str:
response: Final = await llm_router.aanthropic_messages(
model=judge_model,
max_tokens=800,
system=[{"type": "text", "text": system_prompt}], # mutable-ok: SDK kwarg
messages=[{"role": "user", "content": [{"type": "text", "text": user_prompt}]}], # mutable-ok: SDK kwarg
stream=False,
metadata=sanitized_forwardable_call_metadata(parent_metadata, SESSION_REPLAY_JUDGE_CALL_ORIGIN),
num_retries=0,
fallbacks=[], # mutable-ok: SDK kwarg
)
blocks: Final = response.get("content") if isinstance(response, Mapping) else None
if not isinstance(blocks, Sequence):
return ""
return "".join(
str(block.get("text") or "")
for block in blocks
if isinstance(block, Mapping) and block.get("type") == "text"
)
return call
async def _reap_stale_jobs(prisma_client: PrismaClient) -> None:
"""Fail jobs whose pod died mid-run.
Progress is written after every turn, so a running job with a stale updated_at is
abandoned. Without this the partial unique index would lock that session out forever.
"""
cutoff: Final = datetime.now(timezone.utc) - STALE_JOB_AFTER
await _session_replay_jobs(prisma_client).update_many(
where={"status": "running", "updated_at": {"lt": cutoff}}, # mutable-ok: Prisma filter
data={ # mutable-ok: Prisma update input
"status": "failed",
"error": "abandoned: no progress recorded",
"finished_at": datetime.now(timezone.utc),
},
)
async def _run_job(
job_id: str,
transcript: ReplayTranscript,
arms: Sequence[SessionReplayArmSpec],
request: StartSessionReplayRequest,
recorded_prompt_tokens: int,
llm_router: Router,
prisma_client: PrismaClient,
parent_metadata: Mapping[str, object],
) -> None:
jobs: Final = _session_replay_jobs(prisma_client)
async def on_progress(turns_completed: int) -> None:
await jobs.update(
where={"id": job_id}, data={"turns_completed": turns_completed}
) # mutable-ok: Prisma update input
try:
outcome: Final = await replay_session(
transcript=transcript,
arms=arms,
max_turns=request.max_turns,
session_id=request.session_id,
recorded_prompt_tokens=recorded_prompt_tokens,
call_model=_arm_caller(llm_router, parent_metadata),
call_judge=_judge_caller(llm_router, request.judge_model, parent_metadata),
rng=random.Random(),
on_progress=on_progress,
)
except Exception as exc: # noqa: BLE001 # the job records its own failure rather than dying silently
verbose_proxy_logger.exception("session replay job %s failed", job_id)
await jobs.update(
where={"id": job_id}, # mutable-ok: Prisma update input
data={ # mutable-ok: Prisma update input
"status": "failed",
"error": str(exc)[:1000],
"finished_at": datetime.now(timezone.utc),
},
)
return
await jobs.update(
where={"id": job_id}, # mutable-ok: Prisma update input
data={ # mutable-ok: Prisma update input
"status": "completed",
"result": Json(outcome.model_dump()),
"turns_completed": sum(len(run.turns) for run in outcome.arms),
"finished_at": datetime.now(timezone.utc),
},
)
def _require_admin_writer(user_api_key_dict: UserAPIKeyAuth, action: str) -> None:
if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
raise HTTPException(status_code=403, detail=f"Only a proxy admin can {action}")
def _require_admin_viewer(user_api_key_dict: UserAPIKeyAuth, action: str) -> None:
if user_api_key_dict.user_role not in (
LitellmUserRoles.PROXY_ADMIN,
LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
):
raise HTTPException(status_code=403, detail=f"Only proxy admin roles can {action}")
def _job_response(row: Mapping[str, object]) -> SessionReplayJobResponse:
result: Final = row.get("result")
payload: Final = (
result if isinstance(result, Mapping) else {}
) # mutable-ok: empty stand-in for an unfinished job's result
verdict: Final = payload.get("verdict")
fidelity: Final = payload.get("fidelity")
arms: Final = payload.get("arms")
human_asks: Final = payload.get("human_asks")
return SessionReplayJobResponse(
job_id=str(row.get("id") or ""),
session_id=str(row.get("session_id") or ""),
status=str(row.get("status") or "running"), # pyright: ignore[reportArgumentType] # column is the literal set
judge_model=str(row.get("judge_model") or ""),
max_turns=int(row.get("max_turns") or 0), # pyright: ignore[reportArgumentType] # numeric column
source_request_id=str(row["source_request_id"]) if row.get("source_request_id") else None,
turns_completed=int(row.get("turns_completed") or 0), # pyright: ignore[reportArgumentType] # numeric column
created_by=str(row["created_by"]) if row.get("created_by") else None,
created_at=str(row["created_at"]) if row.get("created_at") else None,
finished_at=str(row["finished_at"]) if row.get("finished_at") else None,
error=str(row["error"]) if row.get("error") else None,
human_asks=tuple(str(ask) for ask in human_asks) if isinstance(human_asks, Sequence) else (),
arms=tuple(SessionReplayArmResponse.model_validate(arm) for arm in arms) if isinstance(arms, Sequence) else (),
verdict=SessionReplayVerdictResponse.model_validate(verdict) if isinstance(verdict, Mapping) else None,
fidelity=SessionReplayFidelityResponse.model_validate(fidelity) if isinstance(fidelity, Mapping) else None,
)
@router.post(
"/auto_router/session_replay/start",
tags=("auto router",),
dependencies=(Depends(user_api_key_auth),),
response_model=SessionReplayJobResponse,
status_code=status.HTTP_201_CREATED,
)
async def start_session_replay(
data: StartSessionReplayRequest,
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
) -> SessionReplayJobResponse:
"""
Replay a recorded session against several model arms and judge the end results blind.
The recorded user and tool_result turns are the fixed environment; each arm generates
its own assistant turns, so the arms diverge from the recording and from each other.
Judging compares the final answer each arm reached against the human turns the
recording actually contained.
Spend logs truncate stored prompts, so the replayed request is smaller than the one
that was recorded. Every job reports that gap under `fidelity`; read a verdict against
it rather than as a clean comparison.
A replay issues one billable call per recorded turn per arm, so `max_turns` bounds it
and one session can have only one running job.
"""
from litellm.proxy.proxy_server import llm_router, prisma_client
_require_admin_writer(user_api_key_dict, "start a session replay")
if prisma_client is None:
raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
if llm_router is None:
raise HTTPException(status_code=500, detail=CommonProxyErrors.no_llm_router.value)
await _reap_stale_jobs(prisma_client)
running: Final = await _query_raw(
prisma_client,
'SELECT COUNT(*)::int AS running FROM "LiteLLM_SessionReplayJob" WHERE status = $1',
"running",
)
in_flight: Final = running[0].get("running") if running else 0
if isinstance(in_flight, int) and in_flight >= MAX_CONCURRENT_SESSION_REPLAYS:
raise HTTPException(
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
detail=f"{in_flight} session replays already running, limit is {MAX_CONCURRENT_SESSION_REPLAYS}",
)
source: Final = await _load_source_request(prisma_client, data.session_id)
if isinstance(source, _SourceUnavailable):
raise HTTPException(status_code=source.status_code, detail=source.reason)
transcript: Final = build_transcript(source.body)
if not transcript.user_turns:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail="the recorded session has no user turns to replay",
)
try:
created: Final = await _session_replay_jobs(prisma_client).create(
data={ # mutable-ok: Prisma create input
"session_id": data.session_id,
"status": "running",
"arms": Json([arm.model_dump() for arm in data.arms]), # mutable-ok: Prisma Json column payload
"judge_model": data.judge_model,
"max_turns": data.max_turns,
"source_request_id": source.request_id,
"created_by": user_api_key_dict.user_id,
}
)
except Exception as exc: # noqa: BLE001 # separate the lost race from every other create failure
if _is_unique_violation(exc):
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail=f"a replay is already running for session {data.session_id}",
) from exc
verbose_proxy_logger.exception("could not create session replay job")
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"could not create the replay job: {exc}",
) from exc
parent_metadata: Final = {
"user_api_key_user_id": user_api_key_dict.user_id
} # mutable-ok: forwardable call metadata, an SDK kwarg
asyncio.create_task( # detached job; its own handler records terminal state on the row
_run_job(
job_id=created.id,
transcript=transcript,
arms=data.arms,
request=data,
recorded_prompt_tokens=source.recorded_prompt_tokens,
llm_router=llm_router,
prisma_client=prisma_client,
parent_metadata=parent_metadata,
)
)
return SessionReplayJobResponse(
job_id=created.id,
session_id=data.session_id,
status="running",
judge_model=data.judge_model,
max_turns=data.max_turns,
source_request_id=source.request_id,
created_by=user_api_key_dict.user_id,
human_asks=transcript.human_asks,
fidelity=SessionReplayFidelityResponse(
truncated_strings=transcript.truncated_strings,
recorded_prompt_tokens=source.recorded_prompt_tokens,
),
)
@router.get(
"/auto_router/session_replay/{job_id}",
tags=("auto router",),
dependencies=(Depends(user_api_key_auth),),
response_model=SessionReplayJobResponse,
)
async def get_session_replay(
job_id: str,
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
) -> SessionReplayJobResponse:
"""Read one session replay job, including its verdict once it has finished."""
from litellm.proxy.proxy_server import prisma_client
_require_admin_viewer(user_api_key_dict, "read a session replay")
if prisma_client is None:
raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
rows: Final = await _query_raw(
prisma_client,
'SELECT * FROM "LiteLLM_SessionReplayJob" WHERE id = $1',
job_id,
)
if not rows:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"no session replay job {job_id}")
return _job_response(rows[0])
@router.get(
"/auto_router/session_replay",
tags=("auto router",),
dependencies=(Depends(user_api_key_auth),),
response_model=SessionReplayJobListResponse,
)
async def list_session_replays(
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
session_id: str | None = fastapi.Query(default=None, description="Only jobs for this session"),
limit: int = fastapi.Query(default=25, ge=1, le=100),
) -> SessionReplayJobListResponse:
"""List session replay jobs, newest first."""
from litellm.proxy.proxy_server import prisma_client
_require_admin_viewer(user_api_key_dict, "list session replays")
if prisma_client is None:
raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
rows: Final = (
await _query_raw(
prisma_client,
'SELECT * FROM "LiteLLM_SessionReplayJob" WHERE session_id = $1 ORDER BY created_at DESC LIMIT $2',
session_id,
limit,
)
if session_id
else await _query_raw(
prisma_client,
'SELECT * FROM "LiteLLM_SessionReplayJob" ORDER BY created_at DESC LIMIT $1',
limit,
)
)
return SessionReplayJobListResponse(jobs=tuple(_job_response(row) for row in rows))

View file

@ -539,6 +539,9 @@ from litellm.proxy.management_endpoints.organization_endpoints import (
from litellm.proxy.management_endpoints.router_settings_endpoints import (
router as router_settings_router,
)
from litellm.proxy.management_endpoints.session_replay_endpoints import (
router as session_replay_management_router,
)
from litellm.proxy.management_endpoints.tag_management_endpoints import (
router as tag_management_router,
)
@ -18602,6 +18605,7 @@ app.include_router(budget_management_router)
app.include_router(model_management_router)
app.include_router(model_access_group_management_router)
app.include_router(auto_router_management_router)
app.include_router(session_replay_management_router)
app.include_router(tag_management_router)
app.include_router(workflow_management_router)
app.include_router(memory_router)

View file

@ -1592,6 +1592,33 @@ model LiteLLM_ShadowEvalFunnel {
withheld Int @default(0)
}
// ---------------------------------------------------------------------------
// Session Replay
//
// One row per replay job: a recorded session re-run against several model arms,
// whose end results are judged blind. Single writer (the job's own task), so
// progress and result live on the row rather than in append-only attempt rows.
// ---------------------------------------------------------------------------
model LiteLLM_SessionReplayJob {
id String @id @default(cuid())
session_id String
status String @default("running") // running | completed | failed | stopped
arms Json
judge_model String
max_turns Int
source_request_id String? // the recorded request whose body seeded the transcript
turns_completed Int @default(0)
result Json?
error String?
created_by String?
created_at DateTime @default(now())
updated_at DateTime @default(now()) @updatedAt
finished_at DateTime?
@@index([session_id])
@@index([created_at])
}
// ---------------------------------------------------------------------------
// Workflow Run Tracking
//

View file

View file

@ -0,0 +1,280 @@
"""Replay one recorded session against several model arms and judge the end results.
The environment track (recorded user and tool_result turns) is fixed; each arm generates
its own assistant turns and diverges from the recording freely. What is judged is the
final answer each arm reached, against the human asks the recording actually contained.
"""
from __future__ import annotations
import json
import random
from collections.abc import Mapping, Sequence
from typing import Final, Protocol
from pydantic import BaseModel, ConfigDict
from litellm.proxy.session_replay.transcript import (
ContentBlock,
RecordedMessage,
ReplayTranscript,
attach_turn,
tool_use_ids,
)
from litellm.types.management_endpoints.session_replay_endpoints import (
SessionReplayArmResponse,
SessionReplayArmSpec,
SessionReplayFidelityResponse,
SessionReplayResult,
SessionReplayTurnResponse,
SessionReplayVerdictResponse,
)
JUDGE_SYSTEM_PROMPT: Final = (
"You are grading candidate assistants that were each given the same task and allowed to work "
"independently. Judge only how well the final answer serves the user's request: correctness, "
"completeness, and whether it actually did the work rather than describing it. Ignore length "
"and style. Respond with JSON only, no prose: "
'{"winner": "<label>", "confidence": <0.0-1.0>, "reasoning": "<one sentence>"}. '
'Use the exact label "tie" if neither is better.'
)
_MAX_JUDGE_EXCERPT_CHARS: Final = 6000
class Usage(BaseModel):
model_config = ConfigDict(extra="allow")
input_tokens: int = 0
output_tokens: int = 0
cache_read_input_tokens: int = 0
cache_creation_input_tokens: int = 0
def prompt_tokens(self) -> int:
return self.input_tokens + self.cache_read_input_tokens + self.cache_creation_input_tokens
class ArmResponse(BaseModel):
model_config = ConfigDict(extra="allow")
model: str | None = None
stop_reason: str | None = None
content: tuple[ContentBlock, ...] = ()
usage: Usage = Usage()
def text(self) -> str:
return "".join(block.text or "" for block in self.content if block.type == "text")
class ModelCaller(Protocol):
async def __call__(self, request: Mapping[str, object]) -> Mapping[str, object]: ...
class JudgeCaller(Protocol):
async def __call__(self, system_prompt: str, user_prompt: str) -> str: ...
class ProgressSink(Protocol):
async def __call__(self, turns_completed: int) -> None: ...
async def _noop_progress(turns_completed: int) -> None:
return None
def _skipped_turn(index: int) -> SessionReplayTurnResponse:
return SessionReplayTurnResponse(turn=index, attached=False)
def _failed_turn(index: int, error: str) -> SessionReplayTurnResponse:
return SessionReplayTurnResponse(turn=index, attached=True, error=error[:500])
def _request_body(
transcript: ReplayTranscript,
arm: SessionReplayArmSpec,
messages: Sequence[RecordedMessage],
session_id: str,
) -> Mapping[str, object]:
return { # mutable-ok: Anthropic Messages wire body
**{key: value for key, value in transcript.sampling_params}, # mutable-ok: Anthropic Messages wire body
"model": arm.model,
"messages": [
message.model_dump(exclude_none=True) for message in messages
], # mutable-ok: Anthropic Messages wire body
"system": [
block.model_dump(exclude_none=True) for block in transcript.system
], # mutable-ok: Anthropic Messages wire body
"tools": [
tool.model_dump(exclude_none=True) for tool in transcript.tools
], # mutable-ok: Anthropic Messages wire body
"stream": False,
"litellm_session_id": session_id,
}
async def run_arm(
transcript: ReplayTranscript,
arm: SessionReplayArmSpec,
max_turns: int,
session_id: str,
call_model: ModelCaller,
on_progress: ProgressSink = _noop_progress,
) -> SessionReplayArmResponse:
"""Walk the recorded user turns, letting the arm build its own assistant trajectory.
A turn that fails is recorded and the walk continues: one provider error should cost a
turn, not the whole job.
"""
history: Final[list[RecordedMessage]] = [] # mutable-ok: the arm's trajectory is built turn by turn
turns: Final[list[SessionReplayTurnResponse]] = [] # mutable-ok: one append per recorded turn
pending: tuple[str, ...] = () # rebind-ok: carries the previous turn's tool_use ids forward
final_text = "" # rebind-ok: the last turn that produced text wins
for index, recorded_turn in enumerate(transcript.user_turns[:max_turns]):
attached = attach_turn(recorded_turn, pending)
if attached is None:
turns.append(_skipped_turn(index))
await on_progress(len(turns))
continue
history.append(attached)
try:
raw = await call_model(_request_body(transcript, arm, history, session_id))
response = ArmResponse.model_validate(raw)
except Exception as exc: # noqa: BLE001 # a failed turn is recorded, never fatal to the job
turns.append(_failed_turn(index, str(exc)))
await on_progress(len(turns))
continue
history.append(RecordedMessage(role="assistant", content=response.content))
pending = tool_use_ids(response.content)
final_text = response.text() or final_text
turns.append(
SessionReplayTurnResponse(
turn=index,
attached=True,
served_model=response.model,
stop_reason=response.stop_reason,
text_chars=len(response.text()),
tool_calls=len(pending),
prompt_tokens=response.usage.prompt_tokens(),
output_tokens=response.usage.output_tokens,
)
)
await on_progress(len(turns))
return SessionReplayArmResponse(
label=arm.label,
model=arm.model,
served_models=tuple(dict.fromkeys(turn.served_model for turn in turns if turn.served_model)),
turns=tuple(turns),
final_text=final_text,
)
def _judge_prompt(human_asks: Sequence[str], runs: Sequence[SessionReplayArmResponse], order: Sequence[int]) -> str:
asks: Final = "\n\n".join(ask[:_MAX_JUDGE_EXCERPT_CHARS] for ask in human_asks) or "(no human turn recorded)"
answers: Final = "\n\n".join(
f"--- CANDIDATE {chr(ord('A') + position)} ---\n{runs[arm_index].final_text[:_MAX_JUDGE_EXCERPT_CHARS]}"
for position, arm_index in enumerate(order)
)
return f"THE USER ASKED:\n{asks}\n\n{answers}"
def _unmask(position: str, order: Sequence[int], runs: Sequence[SessionReplayArmResponse]) -> str | None:
index: Final = ord(position.strip().upper()) - ord("A")
if index < 0 or index >= len(order):
return None
return runs[order[index]].label
def _parse_verdict(raw: str) -> tuple[str, float | None, str] | None:
start: Final = raw.find("{")
end: Final = raw.rfind("}")
if start < 0 or end <= start:
return None
try:
payload: Final = json.loads(raw[start : end + 1])
except json.JSONDecodeError:
return None
if not isinstance(payload, dict):
return None
winner: Final = payload.get("winner")
if not isinstance(winner, str):
return None
confidence: Final = payload.get("confidence")
reasoning: Final = payload.get("reasoning")
return (
winner,
float(confidence) if isinstance(confidence, (int, float)) else None,
reasoning if isinstance(reasoning, str) else "",
)
async def judge_runs(
human_asks: Sequence[str],
runs: Sequence[SessionReplayArmResponse],
call_judge: JudgeCaller,
rng: random.Random,
) -> SessionReplayVerdictResponse:
"""Blind pairwise judging: arms are presented as A/B in randomized order and unmasked after."""
if len(runs) < 2:
return SessionReplayVerdictResponse(error="need at least two arms to judge")
order: Final = tuple(rng.sample(range(len(runs)), len(runs)))
try:
raw: Final = await call_judge(JUDGE_SYSTEM_PROMPT, _judge_prompt(human_asks, runs, order))
except Exception as exc: # noqa: BLE001 # a failed judge is a recorded outcome, not a failed job
return SessionReplayVerdictResponse(error=str(exc)[:500])
parsed: Final = _parse_verdict(raw)
if parsed is None:
return SessionReplayVerdictResponse(error=f"unparseable judge response: {raw[:200]}")
position, confidence, reasoning = parsed
if position == "tie":
return SessionReplayVerdictResponse(winner="tie", confidence=confidence, reasoning=reasoning)
unmasked: Final = _unmask(position, order, runs)
if unmasked is None:
return SessionReplayVerdictResponse(reasoning=reasoning, error=f"judge named unknown label {position}")
return SessionReplayVerdictResponse(winner=unmasked, confidence=confidence, reasoning=reasoning)
async def _run_arms(
transcript: ReplayTranscript,
arms: Sequence[SessionReplayArmSpec],
max_turns: int,
session_id: str,
call_model: ModelCaller,
on_progress: ProgressSink,
) -> tuple[SessionReplayArmResponse, ...]:
"""Progress counts turns across the whole job, not per arm, so the heartbeat a stale-job
reaper reads never goes backwards when the next arm starts."""
runs: Final[list[SessionReplayArmResponse]] = [] # mutable-ok: one append per finished arm
for arm in arms:
done_before = sum(len(run.turns) for run in runs)
async def cumulative(turns_completed: int, offset: int = done_before) -> None:
await on_progress(offset + turns_completed)
runs.append(await run_arm(transcript, arm, max_turns, f"{session_id}-{arm.label}", call_model, cumulative))
return tuple(runs)
async def replay_session(
transcript: ReplayTranscript,
arms: Sequence[SessionReplayArmSpec],
max_turns: int,
session_id: str,
recorded_prompt_tokens: int,
call_model: ModelCaller,
call_judge: JudgeCaller,
rng: random.Random,
on_progress: ProgressSink = _noop_progress,
) -> SessionReplayResult:
runs: Final = await _run_arms(transcript, arms, max_turns, session_id, call_model, on_progress)
first_turn_tokens: Final = next((turn.prompt_tokens for run in runs for turn in run.turns if turn.prompt_tokens), 0)
return SessionReplayResult(
human_asks=transcript.human_asks,
arms=runs,
verdict=await judge_runs(transcript.human_asks, runs, call_judge, rng),
fidelity=SessionReplayFidelityResponse(
truncated_strings=transcript.truncated_strings,
recorded_prompt_tokens=recorded_prompt_tokens,
replayed_first_turn_prompt_tokens=first_turn_tokens,
),
)

View file

@ -0,0 +1,179 @@
"""Rebuild a replayable Anthropic Messages request from a recorded spend-log body.
A recorded `proxy_server_request` is a post-guardrail snapshot of the proxy's mutated
request dict, not the caller's wire body, so it needs three repairs before a provider
will accept it back: stray `role: "system"` messages belong in the top-level `system`
array, proxy-injected keys must go, and every string longer than
`MAX_STRING_LENGTH_PROMPT_IN_DB` was truncated on the way in.
"""
from __future__ import annotations
import re
from collections.abc import Sequence
from dataclasses import dataclass
from typing import Final
from pydantic import BaseModel, ConfigDict
TRUNCATION_MARKER: Final = "litellm_truncated"
_PROXY_INJECTED_KEYS: Final = frozenset(
{
"headers",
"litellm_metadata",
"litellm_session_id",
"litellm_trace_id",
"metadata",
"provider_specific_header",
"secret_fields",
}
)
_STRUCTURAL_KEYS: Final = frozenset({"model", "messages", "system", "tools", "stream"})
_SYSTEM_REMINDER: Final = re.compile(r"<system-reminder>.*?</system-reminder>", re.DOTALL)
class ContentBlock(BaseModel):
model_config = ConfigDict(extra="allow")
type: str
text: str | None = None
id: str | None = None
tool_use_id: str | None = None
content: str | None = None
class ToolDefinition(BaseModel):
model_config = ConfigDict(extra="allow")
name: str
description: str | None = None
class RecordedMessage(BaseModel):
model_config = ConfigDict(extra="allow")
role: str
content: str | tuple[ContentBlock, ...]
def blocks(self) -> tuple[ContentBlock, ...]:
if isinstance(self.content, str):
return (ContentBlock(type="text", text=self.content),)
return self.content
def block_types(self) -> frozenset[str]:
return frozenset(block.type for block in self.blocks())
def text(self) -> str:
return "".join(block.text or "" for block in self.blocks() if block.type == "text")
class RecordedBody(BaseModel):
model_config = ConfigDict(extra="allow")
model: str | None = None
messages: tuple[RecordedMessage, ...] = ()
system: str | tuple[ContentBlock, ...] = ()
tools: tuple[ToolDefinition, ...] = ()
@dataclass(frozen=True, slots=True)
class ReplayTranscript:
system: tuple[ContentBlock, ...]
tools: tuple[ToolDefinition, ...]
user_turns: tuple[RecordedMessage, ...]
sampling_params: tuple[tuple[str, object], ...]
human_asks: tuple[str, ...]
truncated_strings: int
recorded_model: str | None
def _count_truncations(value: object) -> int:
if isinstance(value, str):
return 1 if TRUNCATION_MARKER in value else 0
if isinstance(value, dict):
return sum(_count_truncations(item) for item in value.values())
if isinstance(value, (list, tuple)):
return sum(_count_truncations(item) for item in value)
return 0
def _as_system_blocks(system: str | tuple[ContentBlock, ...]) -> tuple[ContentBlock, ...]:
if isinstance(system, str):
return (ContentBlock(type="text", text=system),)
return system
def _hoisted_system_block(message: RecordedMessage) -> ContentBlock:
return ContentBlock(type="text", text=message.text())
def human_ask(message: RecordedMessage) -> str:
return _SYSTEM_REMINDER.sub("", message.text()).strip()
def build_transcript(recorded_body: RecordedBody) -> ReplayTranscript:
"""Split a recorded body into a fixed environment track and the arm's own policy track.
`role: "system"` messages are hoisted rather than dropped: the Anthropic Messages API
rejects that role inside `messages`, but the text is real context the recorded model saw.
"""
hoisted: Final = tuple(
_hoisted_system_block(message) for message in recorded_body.messages if message.role == "system"
)
user_turns: Final = tuple(message for message in recorded_body.messages if message.role == "user")
asks: Final = tuple(
ask for message in user_turns if "tool_result" not in message.block_types() and (ask := human_ask(message))
)
sampling: Final = tuple(
(key, value)
for key, value in recorded_body.model_dump(exclude_none=True).items()
if key not in _PROXY_INJECTED_KEYS and key not in _STRUCTURAL_KEYS
)
return ReplayTranscript(
system=_as_system_blocks(recorded_body.system) + hoisted,
tools=recorded_body.tools,
user_turns=user_turns,
sampling_params=sampling,
human_asks=asks,
truncated_strings=_count_truncations(recorded_body.model_dump()),
recorded_model=recorded_body.model,
)
def tool_use_ids(assistant_blocks: Sequence[ContentBlock]) -> tuple[str, ...]:
return tuple(block.id for block in assistant_blocks if block.type == "tool_use" and block.id is not None)
def attach_turn(turn: RecordedMessage, pending_tool_use_ids: Sequence[str]) -> RecordedMessage | None:
"""Bind a recorded user turn onto the arm's own trajectory, or drop it as unattachable.
Recorded `tool_result` blocks carry the original run's `tool_use_id`s, which name
`tool_use` blocks this arm never emitted. When the arm did call tools the results are
rebound onto its ids in order; when it called none there is nothing to bind to, and
injecting the recording's tool output as prose would feed the arm answers to questions
it never asked, so the turn is dropped instead.
"""
blocks: Final = turn.blocks()
results: Final = tuple(block for block in blocks if block.type == "tool_result")
if not results:
return turn
if not pending_tool_use_ids:
return None
rebound: Final = tuple(
block.model_copy(
update={"tool_use_id": pending_tool_use_ids[min(index, len(pending_tool_use_ids) - 1)]}
) # mutable-ok: pydantic model_copy update mapping
if block.type == "tool_result"
else block
for index, block in enumerate(blocks)
)
answered: Final = frozenset(block.tool_use_id for block in rebound if block.type == "tool_result")
stubs: Final = tuple(
ContentBlock(type="tool_result", tool_use_id=tool_id, content="[no recorded result]")
for tool_id in pending_tool_use_ids
if tool_id not in answered
)
return RecordedMessage(role="user", content=rebound + stubs)

View file

@ -0,0 +1,121 @@
from __future__ import annotations
from typing import Final, Literal
from pydantic import BaseModel, Field, computed_field, field_validator
MAX_SESSION_REPLAY_TURNS: Final = 200
MAX_SESSION_REPLAY_ARMS: Final = 4
SessionReplayStatus = Literal["running", "completed", "failed", "stopped"]
class SessionReplayArmSpec(BaseModel):
label: str = Field(min_length=1, max_length=64)
model: str = Field(min_length=1)
class StartSessionReplayRequest(BaseModel):
session_id: str = Field(min_length=1)
arms: tuple[SessionReplayArmSpec, ...]
judge_model: str = Field(min_length=1)
max_turns: int = Field(default=20, ge=1, le=MAX_SESSION_REPLAY_TURNS)
@field_validator("arms")
@classmethod
def _at_least_two_distinct_arms(cls, arms: tuple[SessionReplayArmSpec, ...]) -> tuple[SessionReplayArmSpec, ...]:
if len(arms) < 2:
raise ValueError("a replay needs at least two arms to compare")
if len(arms) > MAX_SESSION_REPLAY_ARMS:
raise ValueError(f"at most {MAX_SESSION_REPLAY_ARMS} arms per replay")
labels: Final = tuple(arm.label for arm in arms)
if len(frozenset(labels)) != len(labels):
raise ValueError("arm labels must be unique")
return arms
class SessionReplayTurnResponse(BaseModel):
turn: int
attached: bool
served_model: str | None = None
stop_reason: str | None = None
text_chars: int = 0
tool_calls: int = 0
prompt_tokens: int = 0
output_tokens: int = 0
error: str | None = None
class SessionReplayArmResponse(BaseModel):
label: str
model: str
served_models: tuple[str, ...] = ()
turns: tuple[SessionReplayTurnResponse, ...] = ()
final_text: str = ""
@computed_field
@property
def prompt_tokens(self) -> int:
return sum(turn.prompt_tokens for turn in self.turns)
@computed_field
@property
def output_tokens(self) -> int:
return sum(turn.output_tokens for turn in self.turns)
class SessionReplayVerdictResponse(BaseModel):
winner: str | None = None
confidence: float | None = None
reasoning: str = ""
error: str | None = None
class SessionReplayFidelityResponse(BaseModel):
"""How much of the recorded prompt actually survived into the replay.
Spend logs truncate every stored string at MAX_STRING_LENGTH_PROMPT_IN_DB, so a replay
runs against a smaller prompt than the recording did. Reported on every job so a result
is never read as a clean comparison when it is not one.
"""
truncated_strings: int = 0
recorded_prompt_tokens: int = 0
replayed_first_turn_prompt_tokens: int = 0
@property
def prompt_retained_ratio(self) -> float | None:
if not self.recorded_prompt_tokens:
return None
return self.replayed_first_turn_prompt_tokens / self.recorded_prompt_tokens
class SessionReplayResult(BaseModel):
"""Everything a finished replay produced, stored verbatim as the job row's result."""
human_asks: tuple[str, ...] = ()
arms: tuple[SessionReplayArmResponse, ...] = ()
verdict: SessionReplayVerdictResponse = SessionReplayVerdictResponse()
fidelity: SessionReplayFidelityResponse = SessionReplayFidelityResponse()
class SessionReplayJobResponse(BaseModel):
job_id: str
session_id: str
status: SessionReplayStatus
judge_model: str
max_turns: int
source_request_id: str | None = None
turns_completed: int = 0
created_by: str | None = None
created_at: str | None = None
finished_at: str | None = None
error: str | None = None
human_asks: tuple[str, ...] = ()
arms: tuple[SessionReplayArmResponse, ...] = ()
verdict: SessionReplayVerdictResponse | None = None
fidelity: SessionReplayFidelityResponse | None = None
class SessionReplayJobListResponse(BaseModel):
jobs: tuple[SessionReplayJobResponse, ...] = ()

View file

@ -2938,6 +2938,8 @@ InternalCallOrigin = Literal[
"shadow_eval_router",
"shadow_eval_judge",
"background_response_cost_poll",
"session_replay_arm",
"session_replay_judge",
]
"""Which internal litellm feature originated a billed sub-call, so a spend log row
records that it is not traffic the caller sent."""
@ -2946,6 +2948,8 @@ AUTOROUTER_CLASSIFIER_CALL_ORIGIN: Final[InternalCallOrigin] = "autorouter_class
SHADOW_EVAL_ROUTER_CALL_ORIGIN: Final[InternalCallOrigin] = "shadow_eval_router"
SHADOW_EVAL_JUDGE_CALL_ORIGIN: Final[InternalCallOrigin] = "shadow_eval_judge"
BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN: Final[InternalCallOrigin] = "background_response_cost_poll"
SESSION_REPLAY_ARM_CALL_ORIGIN: Final[InternalCallOrigin] = "session_replay_arm"
SESSION_REPLAY_JUDGE_CALL_ORIGIN: Final[InternalCallOrigin] = "session_replay_judge"
class StandardLoggingRoutingDecision(TypedDict, total=False):

View file

@ -1592,6 +1592,33 @@ model LiteLLM_ShadowEvalFunnel {
withheld Int @default(0)
}
// ---------------------------------------------------------------------------
// Session Replay
//
// One row per replay job: a recorded session re-run against several model arms,
// whose end results are judged blind. Single writer (the job's own task), so
// progress and result live on the row rather than in append-only attempt rows.
// ---------------------------------------------------------------------------
model LiteLLM_SessionReplayJob {
id String @id @default(cuid())
session_id String
status String @default("running") // running | completed | failed | stopped
arms Json
judge_model String
max_turns Int
source_request_id String? // the recorded request whose body seeded the transcript
turns_completed Int @default(0)
result Json?
error String?
created_by String?
created_at DateTime @default(now())
updated_at DateTime @default(now()) @updatedAt
finished_at DateTime?
@@index([session_id])
@@index([created_at])
}
// ---------------------------------------------------------------------------
// Workflow Run Tracking
//

View file

@ -0,0 +1,215 @@
import json
import pytest
from fastapi import HTTPException
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.management_endpoints.session_replay_endpoints import (
_job_response,
_load_source_request,
_require_admin_viewer,
_require_admin_writer,
_SourceRequest,
_SourceUnavailable,
)
RECORDED = {
"model": "router",
"max_tokens": 1024,
"system": [{"type": "text", "text": "sys"}],
"messages": [{"role": "user", "content": [{"type": "text", "text": "hello"}]}],
}
class _FakePrisma:
def __init__(self, rows):
self.rows = rows
self.queries = []
class _Db:
def __init__(self, outer):
self.outer = outer
async def query_raw(self, query, *args):
self.outer.queries.append((query, args))
return self.outer.rows
@property
def db(self):
return self._Db(self)
@pytest.mark.asyncio
async def test_session_with_no_replayable_rows_is_a_404():
result = await _load_source_request(_FakePrisma([]), "missing-session")
assert isinstance(result, _SourceUnavailable)
assert result.status_code == 404
@pytest.mark.asyncio
async def test_session_whose_body_was_not_stored_explains_the_config_gate():
"""store_prompts_in_spend_logs off writes an empty object, so the row exists but holds
nothing to replay: that must not read as a missing session."""
result = await _load_source_request(
_FakePrisma([{"request_id": "r1", "proxy_server_request": {}, "prompt_tokens": 10, "metadata": {}}]),
"sess",
)
assert isinstance(result, _SourceUnavailable)
assert result.status_code == 409
assert "store_prompts_in_spend_logs" in result.reason
@pytest.mark.asyncio
async def test_session_rewritten_by_a_mutating_guardrail_is_refused():
"""The stored body predates the guardrail's rewrite, so replaying it would resurrect
content the guardrail stripped."""
rows = [
{
"request_id": "r1",
"proxy_server_request": RECORDED,
"prompt_tokens": 10,
"metadata": {
"standard_logging_guardrail_information": [{"guardrail_mode": "pre_call"}],
},
}
]
result = await _load_source_request(_FakePrisma(rows), "sess")
assert isinstance(result, _SourceUnavailable)
assert result.status_code == 409
assert "guardrail" in result.reason
@pytest.mark.asyncio
async def test_replayable_session_returns_the_parsed_body_and_recorded_token_count():
rows = [
{
"request_id": "req-42",
"proxy_server_request": RECORDED,
"prompt_tokens": 41920,
"metadata": {},
}
]
result = await _load_source_request(_FakePrisma(rows), "sess")
assert isinstance(result, _SourceRequest)
assert result.request_id == "req-42"
assert result.recorded_prompt_tokens == 41920
assert len(result.body.messages) == 1
@pytest.mark.asyncio
async def test_source_lookup_is_scoped_to_replayable_call_types():
"""A chat-completions row cannot be replayed through the Anthropic Messages path, so
picking one up would fail at the first turn instead of at start."""
prisma = _FakePrisma([])
await _load_source_request(prisma, "sess")
_, args = prisma.queries[0]
assert args == ("sess", "anthropic_messages")
@pytest.mark.parametrize(
"role",
[
LitellmUserRoles.INTERNAL_USER,
LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
LitellmUserRoles.TEAM,
None,
],
)
def test_only_a_proxy_admin_can_start_a_replay(role):
"""A replay reads another key's recorded prompts and spends real money on them."""
with pytest.raises(HTTPException) as excinfo:
_require_admin_writer(UserAPIKeyAuth(api_key="k", user_role=role), "start a session replay")
assert excinfo.value.status_code == 403
def test_proxy_admin_may_start_a_replay():
admin = UserAPIKeyAuth(api_key="k", user_role=LitellmUserRoles.PROXY_ADMIN)
assert _require_admin_writer(admin, "start") is None
@pytest.mark.parametrize(
"role",
[LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY],
)
def test_admin_viewers_may_read_a_replay(role):
assert _require_admin_viewer(UserAPIKeyAuth(api_key="k", user_role=role), "read") is None
def test_internal_user_cannot_read_a_replay():
with pytest.raises(HTTPException) as excinfo:
_require_admin_viewer(UserAPIKeyAuth(api_key="k", user_role=LitellmUserRoles.INTERNAL_USER), "read")
assert excinfo.value.status_code == 403
def test_finished_job_row_surfaces_verdict_and_fidelity():
row = {
"id": "job-1",
"session_id": "sess",
"status": "completed",
"judge_model": "judge",
"max_turns": 20,
"source_request_id": "req-42",
"turns_completed": 3,
"created_by": "admin",
"result": {
"human_asks": ["plan my week"],
"arms": [
{"label": "auto_router", "model": "r", "final_text": "a", "turns": []},
{"label": "baseline", "model": "b", "final_text": "c", "turns": []},
],
"verdict": {"winner": "auto_router", "confidence": 0.72, "reasoning": "more complete"},
"fidelity": {
"truncated_strings": 6,
"recorded_prompt_tokens": 41920,
"replayed_first_turn_prompt_tokens": 16016,
},
},
}
response = _job_response(row)
assert response.status == "completed"
assert response.verdict is not None and response.verdict.winner == "auto_router"
assert response.fidelity is not None
assert response.fidelity.prompt_retained_ratio == pytest.approx(16016 / 41920)
assert tuple(arm.label for arm in response.arms) == ("auto_router", "baseline")
def test_running_job_row_has_no_verdict_yet():
response = _job_response(
{"id": "job-2", "session_id": "s", "status": "running", "judge_model": "j", "max_turns": 5}
)
assert response.status == "running"
assert response.verdict is None
assert response.arms == ()
@pytest.mark.asyncio
async def test_jsonb_columns_returned_as_text_are_still_parsed():
"""query_raw hands a jsonb column back as a dict or as raw JSON text depending on the
driver, and treating the text form as absent would report every session unreplayable."""
rows = [
{
"request_id": "req-1",
"proxy_server_request": json.dumps(RECORDED),
"prompt_tokens": 100,
"metadata": "{}",
}
]
result = await _load_source_request(_FakePrisma(rows), "sess")
assert isinstance(result, _SourceRequest)
assert len(result.body.messages) == 1

View file

@ -0,0 +1,305 @@
import random
import pytest
from litellm.proxy.session_replay.executor import judge_runs, replay_session, run_arm
from litellm.proxy.session_replay.transcript import RecordedBody, build_transcript
from litellm.types.management_endpoints.session_replay_endpoints import SessionReplayArmSpec
TWO_HUMAN_TURNS = {
"model": "recorded-router",
"max_tokens": 1024,
"system": [{"type": "text", "text": "be helpful"}],
"tools": [{"name": "Read", "description": "read", "input_schema": {"type": "object", "properties": {}}}],
"messages": [
{"role": "user", "content": [{"type": "text", "text": "first ask"}]},
{"role": "assistant", "content": [{"type": "tool_use", "id": "toolu_REC", "name": "Read"}]},
{
"role": "user",
"content": [{"type": "tool_result", "tool_use_id": "toolu_REC", "content": "recorded output"}],
},
{"role": "user", "content": [{"type": "text", "text": "second ask"}]},
],
}
def _transcript(body=None):
return build_transcript(RecordedBody.model_validate(body or TWO_HUMAN_TURNS))
def _text_response(text, model="served-model"):
return {
"model": model,
"stop_reason": "end_turn",
"content": [{"type": "text", "text": text}],
"usage": {"input_tokens": 10, "output_tokens": 5},
}
def _tool_response(tool_use_id, model="served-model"):
return {
"model": model,
"stop_reason": "tool_use",
"content": [{"type": "tool_use", "id": tool_use_id, "name": "Read", "input": {}}],
"usage": {"input_tokens": 10, "output_tokens": 5},
}
class _RecordingCaller:
def __init__(self, responses):
self.responses = list(responses)
self.requests = []
async def __call__(self, request):
self.requests.append(request)
nxt = self.responses.pop(0)
if isinstance(nxt, Exception):
raise nxt
return nxt
@pytest.mark.asyncio
async def test_arm_trajectory_accumulates_its_own_assistant_turns():
"""The arm's own outputs, not the recording's, must be what later turns see: that is
what makes this an end-to-end replay rather than per-turn scoring."""
caller = _RecordingCaller([_tool_response("toolu_ARM"), _text_response("after tool"), _text_response("final")])
run = await run_arm(
_transcript(), SessionReplayArmSpec(label="a", model="m"), max_turns=10, session_id="s", call_model=caller
)
assert run.final_text == "final"
assert len(caller.requests) == 3
roles_on_last_call = [message["role"] for message in caller.requests[-1]["messages"]]
assert roles_on_last_call == ["user", "assistant", "user", "assistant", "user"]
@pytest.mark.asyncio
async def test_recorded_tool_result_is_rebound_to_the_arms_tool_use_id():
caller = _RecordingCaller([_tool_response("toolu_ARM"), _text_response("ok"), _text_response("done")])
await run_arm(
_transcript(), SessionReplayArmSpec(label="a", model="m"), max_turns=10, session_id="s", call_model=caller
)
second_call_last_message = caller.requests[1]["messages"][-1]
assert [block["tool_use_id"] for block in second_call_last_message["content"]] == ["toolu_ARM"]
@pytest.mark.asyncio
async def test_turn_whose_tool_result_cannot_attach_is_skipped_without_calling_the_model():
caller = _RecordingCaller([_text_response("no tools here"), _text_response("second answer")])
run = await run_arm(
_transcript(), SessionReplayArmSpec(label="a", model="m"), max_turns=10, session_id="s", call_model=caller
)
assert [turn.attached for turn in run.turns] == [True, False, True]
assert len(caller.requests) == 2
@pytest.mark.asyncio
async def test_a_failing_turn_is_recorded_and_the_walk_continues():
"""One provider error should cost a turn, not the whole job."""
caller = _RecordingCaller([RuntimeError("upstream 401"), _text_response("recovered")])
run = await run_arm(
_transcript(), SessionReplayArmSpec(label="a", model="m"), max_turns=10, session_id="s", call_model=caller
)
assert run.turns[0].error is not None and "upstream 401" in run.turns[0].error
assert run.final_text == "recovered"
@pytest.mark.asyncio
async def test_max_turns_bounds_the_number_of_billable_calls():
caller = _RecordingCaller([_text_response("one"), _text_response("two"), _text_response("three")])
await run_arm(
_transcript(), SessionReplayArmSpec(label="a", model="m"), max_turns=1, session_id="s", call_model=caller
)
assert len(caller.requests) == 1
@pytest.mark.asyncio
async def test_progress_is_reported_after_every_turn():
"""A job whose pod dies is detected by a stale heartbeat, so progress must advance
per turn rather than per arm."""
caller = _RecordingCaller([_text_response("one"), _text_response("two")])
seen = []
async def on_progress(turns_completed):
seen.append(turns_completed)
await run_arm(
_transcript(),
SessionReplayArmSpec(label="a", model="m"),
max_turns=10,
session_id="s",
call_model=caller,
on_progress=on_progress,
)
assert seen == [1, 2, 3]
@pytest.mark.asyncio
async def test_judge_verdict_is_unmasked_back_to_the_winning_arm_label():
"""The judge sees anonymous A/B in randomized order, so a verdict read without
unmasking would attribute wins to whichever arm happened to be shown first."""
runs = []
for label in ("alpha", "beta"):
caller = _RecordingCaller([_text_response(f"{label} answer"), _text_response(f"{label} final")])
runs.append(
await run_arm(
_transcript(),
SessionReplayArmSpec(label=label, model="m"),
max_turns=10,
session_id="s",
call_model=caller,
)
)
async def call_judge(system_prompt, user_prompt):
shown_first = "alpha" if "alpha final" in user_prompt.split("CANDIDATE B")[0] else "beta"
assert shown_first in ("alpha", "beta")
return '{"winner": "A", "confidence": 0.9, "reasoning": "clearer"}'
for seed in range(6):
verdict = await judge_runs(("ask",), runs, call_judge, random.Random(seed))
order = random.Random(seed).sample(range(2), 2)
assert verdict.winner == runs[order[0]].label
@pytest.mark.asyncio
async def test_unparseable_judge_response_is_a_recorded_outcome_not_an_exception():
caller = _RecordingCaller([_text_response("x"), _text_response("y")])
run = await run_arm(
_transcript(), SessionReplayArmSpec(label="a", model="m"), max_turns=10, session_id="s", call_model=caller
)
async def call_judge(system_prompt, user_prompt):
return "I prefer the first one"
verdict = await judge_runs(("ask",), (run, run), call_judge, random.Random(0))
assert verdict.winner is None
assert verdict.error is not None
@pytest.mark.asyncio
async def test_judge_failure_does_not_fail_the_job():
caller = _RecordingCaller([_text_response("x"), _text_response("y")])
run = await run_arm(
_transcript(), SessionReplayArmSpec(label="a", model="m"), max_turns=10, session_id="s", call_model=caller
)
async def call_judge(system_prompt, user_prompt):
raise RuntimeError("judge model unavailable")
verdict = await judge_runs(("ask",), (run, run), call_judge, random.Random(0))
assert verdict.winner is None
assert verdict.error is not None and "judge model unavailable" in verdict.error
@pytest.mark.asyncio
async def test_fidelity_reports_the_gap_between_recorded_and_replayed_prompt():
"""Spend logs truncate stored prompts, so a replay runs on a smaller prompt than the
recording did and a verdict must never be read as a clean comparison."""
async def call_model(request):
return _text_response("answer")
async def call_judge(system_prompt, user_prompt):
return '{"winner": "tie", "confidence": 0.5, "reasoning": "same"}'
outcome = await replay_session(
transcript=_transcript(),
arms=(SessionReplayArmSpec(label="a", model="m"), SessionReplayArmSpec(label="b", model="n")),
max_turns=10,
session_id="s",
recorded_prompt_tokens=41920,
call_model=call_model,
call_judge=call_judge,
rng=random.Random(0),
)
assert outcome.fidelity.recorded_prompt_tokens == 41920
assert outcome.fidelity.replayed_first_turn_prompt_tokens == 10
assert outcome.verdict.winner == "tie"
@pytest.mark.asyncio
async def test_each_arm_replays_under_its_own_session_id():
"""Arms share one recorded session, but session affinity would pin them to one tier
if they also shared a session id at replay time."""
seen = []
async def call_model(request):
seen.append(request["litellm_session_id"])
return _text_response("answer")
async def call_judge(system_prompt, user_prompt):
return '{"winner": "tie", "confidence": 0.5, "reasoning": "same"}'
await replay_session(
transcript=_transcript(),
arms=(SessionReplayArmSpec(label="a", model="m"), SessionReplayArmSpec(label="b", model="n")),
max_turns=1,
session_id="orig",
recorded_prompt_tokens=1,
call_model=call_model,
call_judge=call_judge,
rng=random.Random(0),
)
assert len(set(seen)) == 2
assert "orig" not in seen
@pytest.mark.asyncio
async def test_progress_counts_turns_across_arms_not_per_arm():
"""A stale-job reaper reads this counter as a heartbeat, so restarting it at 1 when the
second arm begins makes a live job look like it went backwards."""
async def call_model(request):
return _text_response("answer")
async def call_judge(system_prompt, user_prompt):
return '{"winner": "tie", "confidence": 0.5, "reasoning": "same"}'
seen = []
async def on_progress(turns_completed):
seen.append(turns_completed)
await replay_session(
transcript=_transcript(),
arms=(SessionReplayArmSpec(label="a", model="m"), SessionReplayArmSpec(label="b", model="n")),
max_turns=10,
session_id="s",
recorded_prompt_tokens=1,
call_model=call_model,
call_judge=call_judge,
rng=random.Random(0),
on_progress=on_progress,
)
assert seen == sorted(seen)
assert seen[-1] == len(seen)
@pytest.mark.asyncio
async def test_arm_token_totals_survive_serialization():
"""They are derived from the turns, and the job row stores the serialized result, so a
plain property would drop them from every stored and returned job."""
caller = _RecordingCaller([_text_response("one"), _text_response("two")])
run = await run_arm(
_transcript(), SessionReplayArmSpec(label="a", model="m"), max_turns=10, session_id="s", call_model=caller
)
assert run.model_dump()["prompt_tokens"] == 20
assert run.model_dump()["output_tokens"] == 10

View file

@ -0,0 +1,134 @@
from litellm.proxy.session_replay.transcript import (
RecordedBody,
attach_turn,
build_transcript,
tool_use_ids,
)
RECORDED_BODY = {
"model": "tin-claude-router",
"max_tokens": 32000,
"thinking": {"type": "adaptive"},
"stream": True,
"litellm_session_id": "sess-1",
"litellm_trace_id": "sess-1",
"headers": {"x-claude-code-session-id": "sess-1"},
"litellm_metadata": {"user_api_key_hash": "secret-hash"},
"metadata": {"user_id": "device-blob"},
"provider_specific_header": {"extra_headers": {"anthropic-beta": "claude-code"}},
"system": [{"type": "text", "text": "You are Claude Code"}],
"tools": [
{"name": "Read", "description": "Read a file", "input_schema": {"type": "object", "properties": {}}},
{
"name": "Edit",
"description": "Edit a file...[litellm_truncated]...tail",
"input_schema": {"type": "object", "properties": {}},
},
],
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "<system-reminder>\ntoday is tuesday\n</system-reminder>"},
{"type": "text", "text": "Plan my week"},
],
},
{"role": "system", "content": "Available agent types for the Agent tool"},
{
"role": "assistant",
"content": [
{"type": "text", "text": "on it"},
{"type": "tool_use", "id": "toolu_ORIGINAL", "name": "Read"},
],
},
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_ORIGINAL", "content": "file body"}]},
],
}
def _transcript():
return build_transcript(RecordedBody.model_validate(RECORDED_BODY))
def test_system_role_message_is_hoisted_out_of_messages():
"""The Anthropic Messages API rejects role=system inside messages with a 400, and the
recorded body carries one, so leaving it in place makes every replay unreplayable."""
transcript = _transcript()
assert all(turn.role == "user" for turn in transcript.user_turns)
assert "Available agent types for the Agent tool" in "".join(block.text or "" for block in transcript.system)
def test_human_asks_strip_system_reminders_and_exclude_tool_results():
transcript = _transcript()
assert transcript.human_asks == ("Plan my week",)
def test_sampling_params_keep_provider_knobs_and_drop_proxy_injected_keys():
transcript = _transcript()
keys = {key for key, _ in transcript.sampling_params}
assert {"max_tokens", "thinking"} <= keys
assert keys.isdisjoint(
{
"litellm_session_id",
"litellm_trace_id",
"headers",
"litellm_metadata",
"metadata",
"provider_specific_header",
}
)
assert keys.isdisjoint({"model", "messages", "system", "tools", "stream"})
def test_truncation_markers_are_counted_so_fidelity_loss_is_visible():
assert _transcript().truncated_strings == 1
def test_unattachable_tool_result_turn_is_dropped_not_degraded():
"""An arm that called no tools has no tool_use block for a recorded tool_result to
reference, and injecting the recording's tool output as prose would hand the arm
answers to questions it never asked."""
transcript = _transcript()
tool_turn = next(turn for turn in transcript.user_turns if "tool_result" in turn.block_types())
assert attach_turn(tool_turn, ()) is None
def test_tool_results_rebind_to_the_arms_own_tool_use_ids():
transcript = _transcript()
tool_turn = next(turn for turn in transcript.user_turns if "tool_result" in turn.block_types())
attached = attach_turn(tool_turn, ("toolu_ARM",))
assert attached is not None
assert [block.tool_use_id for block in attached.blocks()] == ["toolu_ARM"]
def test_tool_calls_the_recording_never_answered_get_stubbed():
"""Anthropic rejects an assistant tool_use with no matching tool_result, so an arm that
called more tools than the recording answered would wedge on the next turn."""
transcript = _transcript()
tool_turn = next(turn for turn in transcript.user_turns if "tool_result" in turn.block_types())
attached = attach_turn(tool_turn, ("toolu_ARM", "toolu_EXTRA"))
assert attached is not None
assert [block.tool_use_id for block in attached.blocks()] == ["toolu_ARM", "toolu_EXTRA"]
def test_plain_user_turn_passes_through_untouched():
transcript = _transcript()
plain = next(turn for turn in transcript.user_turns if "tool_result" not in turn.block_types())
assert attach_turn(plain, ("toolu_ARM",)) is plain
def test_tool_use_ids_reads_only_tool_use_blocks():
transcript = _transcript()
assistant_blocks = RecordedBody.model_validate(RECORDED_BODY).messages[2].blocks()
assert tool_use_ids(assistant_blocks) == ("toolu_ORIGINAL",)
assert tool_use_ids(transcript.system) == ()

View file

@ -1242,6 +1242,78 @@ export interface paths {
patch?: never;
trace?: never;
};
"/auto_router/session_replay": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/**
* List Session Replays
* @description List session replay jobs, newest first.
*/
get: operations["list_session_replays_auto_router_session_replay_get"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/auto_router/session_replay/start": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
get?: never;
put?: never;
/**
* Start Session Replay
* @description Replay a recorded session against several model arms and judge the end results blind.
*
* The recorded user and tool_result turns are the fixed environment; each arm generates
* its own assistant turns, so the arms diverge from the recording and from each other.
* Judging compares the final answer each arm reached against the human turns the
* recording actually contained.
*
* Spend logs truncate stored prompts, so the replayed request is smaller than the one
* that was recorded. Every job reports that gap under `fidelity`; read a verdict against
* it rather than as a clean comparison.
*
* A replay issues one billable call per recorded turn per arm, so `max_turns` bounds it
* and one session can have only one running job.
*/
post: operations["start_session_replay_auto_router_session_replay_start_post"];
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/auto_router/session_replay/{job_id}": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/**
* Get Session Replay
* @description Read one session replay job, including its verdict once it has finished.
*/
get: operations["get_session_replay_auto_router_session_replay__job_id__get"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/auto_router/shadow_eval": {
parameters: {
query?: never;
@ -36049,6 +36121,162 @@ export interface components {
/** Timeout */
timeout?: number | null;
};
/** SessionReplayArmResponse */
SessionReplayArmResponse: {
/**
* Final Text
* @default
*/
final_text: string;
/** Label */
label: string;
/** Model */
model: string;
/** Output Tokens */
readonly output_tokens: number;
/** Prompt Tokens */
readonly prompt_tokens: number;
/**
* Served Models
* @default []
*/
served_models: string[];
/**
* Turns
* @default []
*/
turns: components["schemas"]["SessionReplayTurnResponse"][];
};
/** SessionReplayArmSpec */
SessionReplayArmSpec: {
/** Label */
label: string;
/** Model */
model: string;
};
/**
* SessionReplayFidelityResponse
* @description How much of the recorded prompt actually survived into the replay.
*
* Spend logs truncate every stored string at MAX_STRING_LENGTH_PROMPT_IN_DB, so a replay
* runs against a smaller prompt than the recording did. Reported on every job so a result
* is never read as a clean comparison when it is not one.
*/
SessionReplayFidelityResponse: {
/**
* Recorded Prompt Tokens
* @default 0
*/
recorded_prompt_tokens: number;
/**
* Replayed First Turn Prompt Tokens
* @default 0
*/
replayed_first_turn_prompt_tokens: number;
/**
* Truncated Strings
* @default 0
*/
truncated_strings: number;
};
/** SessionReplayJobListResponse */
SessionReplayJobListResponse: {
/**
* Jobs
* @default []
*/
jobs: components["schemas"]["SessionReplayJobResponse"][];
};
/** SessionReplayJobResponse */
SessionReplayJobResponse: {
/**
* Arms
* @default []
*/
arms: components["schemas"]["SessionReplayArmResponse"][];
/** Created At */
created_at?: string | null;
/** Created By */
created_by?: string | null;
/** Error */
error?: string | null;
fidelity?: components["schemas"]["SessionReplayFidelityResponse"] | null;
/** Finished At */
finished_at?: string | null;
/**
* Human Asks
* @default []
*/
human_asks: string[];
/** Job Id */
job_id: string;
/** Judge Model */
judge_model: string;
/** Max Turns */
max_turns: number;
/** Session Id */
session_id: string;
/** Source Request Id */
source_request_id?: string | null;
/**
* Status
* @enum {string}
*/
status: "running" | "completed" | "failed" | "stopped";
/**
* Turns Completed
* @default 0
*/
turns_completed: number;
verdict?: components["schemas"]["SessionReplayVerdictResponse"] | null;
};
/** SessionReplayTurnResponse */
SessionReplayTurnResponse: {
/** Attached */
attached: boolean;
/** Error */
error?: string | null;
/**
* Output Tokens
* @default 0
*/
output_tokens: number;
/**
* Prompt Tokens
* @default 0
*/
prompt_tokens: number;
/** Served Model */
served_model?: string | null;
/** Stop Reason */
stop_reason?: string | null;
/**
* Text Chars
* @default 0
*/
text_chars: number;
/**
* Tool Calls
* @default 0
*/
tool_calls: number;
/** Turn */
turn: number;
};
/** SessionReplayVerdictResponse */
SessionReplayVerdictResponse: {
/** Confidence */
confidence?: number | null;
/** Error */
error?: string | null;
/**
* Reasoning
* @default
*/
reasoning: string;
/** Winner */
winner?: string | null;
};
/**
* ShadowEvalJobResponse
* @description A shadow-eval job over one or more targets, each with its own budget and stop state;
@ -36486,6 +36714,20 @@ export interface components {
/** Simple Medium */
simple_medium: number;
};
/** StartSessionReplayRequest */
StartSessionReplayRequest: {
/** Arms */
arms: components["schemas"]["SessionReplayArmSpec"][];
/** Judge Model */
judge_model: string;
/**
* Max Turns
* @default 20
*/
max_turns: number;
/** Session Id */
session_id: string;
};
/**
* StartShadowEvalRequest
* @description Start duplicating one or more targets' traffic for blind comparison against an auto-router.
@ -41782,6 +42024,103 @@ export interface operations {
};
};
};
list_session_replays_auto_router_session_replay_get: {
parameters: {
query?: {
/** @description Only jobs for this session */
session_id?: string | null;
limit?: number;
};
header?: never;
path?: never;
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["SessionReplayJobListResponse"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
start_session_replay_auto_router_session_replay_start_post: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody: {
content: {
"application/json": components["schemas"]["StartSessionReplayRequest"];
};
};
responses: {
/** @description Successful Response */
201: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["SessionReplayJobResponse"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
get_session_replay_auto_router_session_replay__job_id__get: {
parameters: {
query?: never;
header?: never;
path: {
job_id: string;
};
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["SessionReplayJobResponse"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
list_shadow_eval_jobs_auto_router_shadow_eval_get: {
parameters: {
query?: {