mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
feat(session_replay): replay a recorded session against model arms and judge it
This commit is contained in:
parent
692a311efb
commit
2225c6d433
17 changed files with 2151 additions and 0 deletions
1
.github/workflows/test-unit.yml
vendored
1
.github/workflows/test-unit.yml
vendored
|
|
@ -143,6 +143,7 @@ jobs:
|
|||
test-path: >-
|
||||
tests/test_litellm/proxy/analytics_endpoints
|
||||
tests/test_litellm/proxy/management_endpoints
|
||||
tests/test_litellm/proxy/session_replay
|
||||
tests/test_litellm/proxy/list_api
|
||||
tests/test_litellm/proxy/memory
|
||||
tests/test_litellm/proxy/guardrails
|
||||
|
|
|
|||
|
|
@ -0,0 +1,28 @@
|
|||
CREATE TABLE IF NOT EXISTS "LiteLLM_SessionReplayJob" (
|
||||
"id" TEXT NOT NULL,
|
||||
"session_id" TEXT NOT NULL,
|
||||
"status" TEXT NOT NULL DEFAULT 'running',
|
||||
"arms" JSONB NOT NULL,
|
||||
"judge_model" TEXT NOT NULL,
|
||||
"max_turns" INTEGER NOT NULL,
|
||||
"source_request_id" TEXT,
|
||||
"turns_completed" INTEGER NOT NULL DEFAULT 0,
|
||||
"result" JSONB,
|
||||
"error" TEXT,
|
||||
"created_by" TEXT,
|
||||
"created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"finished_at" TIMESTAMP(3),
|
||||
|
||||
CONSTRAINT "LiteLLM_SessionReplayJob_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS "LiteLLM_SessionReplayJob_session_id_idx" ON "LiteLLM_SessionReplayJob"("session_id");
|
||||
|
||||
CREATE INDEX IF NOT EXISTS "LiteLLM_SessionReplayJob_created_at_idx" ON "LiteLLM_SessionReplayJob"("created_at");
|
||||
|
||||
-- A replay issues real billable calls for every recorded turn on every arm, so two
|
||||
-- concurrent starts on one session double that spend for no extra signal. Partial
|
||||
-- indexes are not expressible in schema.prisma, so this lives here only.
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_SessionReplayJob_one_active_per_session"
|
||||
ON "LiteLLM_SessionReplayJob"("session_id") WHERE "status" = 'running';
|
||||
|
|
@ -1592,6 +1592,33 @@ model LiteLLM_ShadowEvalFunnel {
|
|||
withheld Int @default(0)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Session Replay
|
||||
//
|
||||
// One row per replay job: a recorded session re-run against several model arms,
|
||||
// whose end results are judged blind. Single writer (the job's own task), so
|
||||
// progress and result live on the row rather than in append-only attempt rows.
|
||||
// ---------------------------------------------------------------------------
|
||||
model LiteLLM_SessionReplayJob {
|
||||
id String @id @default(cuid())
|
||||
session_id String
|
||||
status String @default("running") // running | completed | failed | stopped
|
||||
arms Json
|
||||
judge_model String
|
||||
max_turns Int
|
||||
source_request_id String? // the recorded request whose body seeded the transcript
|
||||
turns_completed Int @default(0)
|
||||
result Json?
|
||||
error String?
|
||||
created_by String?
|
||||
created_at DateTime @default(now())
|
||||
updated_at DateTime @default(now()) @updatedAt
|
||||
finished_at DateTime?
|
||||
|
||||
@@index([session_id])
|
||||
@@index([created_at])
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Workflow Run Tracking
|
||||
//
|
||||
|
|
|
|||
460
litellm/proxy/management_endpoints/session_replay_endpoints.py
Normal file
460
litellm/proxy/management_endpoints/session_replay_endpoints.py
Normal file
|
|
@ -0,0 +1,460 @@
|
|||
"""Admin endpoints to replay a recorded session against several model arms and judge it."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import random
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import TYPE_CHECKING, Annotated, Final, Protocol
|
||||
|
||||
import fastapi
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from prisma import Json
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.integrations.shadow_eval_logger import (
|
||||
_request_mutating_guardrail_ran, # pyright: ignore[reportPrivateUsage] # one owner for the guardrail-mode reading
|
||||
)
|
||||
from litellm.litellm_core_utils.internal_call_metadata import sanitized_forwardable_call_metadata
|
||||
from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.management_endpoints.auto_router_endpoints import (
|
||||
_is_unique_violation, # pyright: ignore[reportPrivateUsage] # one owner for the unique-violation read
|
||||
)
|
||||
from litellm.proxy.session_replay.executor import replay_session
|
||||
from litellm.proxy.session_replay.transcript import RecordedBody, ReplayTranscript, build_transcript
|
||||
from litellm.types.management_endpoints.session_replay_endpoints import (
|
||||
SessionReplayArmResponse,
|
||||
SessionReplayArmSpec,
|
||||
SessionReplayFidelityResponse,
|
||||
SessionReplayJobListResponse,
|
||||
SessionReplayJobResponse,
|
||||
SessionReplayVerdictResponse,
|
||||
StartSessionReplayRequest,
|
||||
)
|
||||
from litellm.types.utils import SESSION_REPLAY_ARM_CALL_ORIGIN, SESSION_REPLAY_JUDGE_CALL_ORIGIN
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
from litellm.router import Router
|
||||
|
||||
router: Final = APIRouter()
|
||||
|
||||
MAX_CONCURRENT_SESSION_REPLAYS: Final = 4
|
||||
STALE_JOB_AFTER: Final = timedelta(minutes=30)
|
||||
_REPLAYABLE_CALL_TYPE: Final = "anthropic_messages"
|
||||
|
||||
|
||||
class _SessionReplayJobRow(Protocol):
|
||||
@property
|
||||
def id(self) -> str: ...
|
||||
|
||||
|
||||
class _SessionReplayJobTable(Protocol):
|
||||
async def create(self, data: Mapping[str, object]) -> _SessionReplayJobRow: ...
|
||||
|
||||
async def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> object: ...
|
||||
|
||||
async def update_many(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> int: ...
|
||||
|
||||
|
||||
def _session_replay_jobs(prisma_client: PrismaClient) -> _SessionReplayJobTable:
|
||||
return prisma_client.db.litellm_sessionreplayjob # pyright: ignore[reportAttributeAccessIssue] # generated client
|
||||
|
||||
|
||||
async def _query_raw(prisma_client: PrismaClient, query: str, *args: object) -> Sequence[Mapping[str, object]]:
|
||||
return await prisma_client.db.query_raw(query, *args)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _SourceRequest:
|
||||
request_id: str
|
||||
body: RecordedBody
|
||||
recorded_prompt_tokens: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _SourceUnavailable:
|
||||
status_code: int
|
||||
reason: str
|
||||
|
||||
|
||||
def _as_mapping(value: object) -> Mapping[str, object] | None:
|
||||
"""Read a jsonb column back, which query_raw hands over as a dict or as raw JSON text."""
|
||||
if isinstance(value, Mapping):
|
||||
return value
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
try:
|
||||
decoded: Final = json.loads(value)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
return decoded if isinstance(decoded, Mapping) else None
|
||||
|
||||
|
||||
async def _load_source_request(prisma_client: PrismaClient, session_id: str) -> _SourceRequest | _SourceUnavailable:
|
||||
"""Pick the recorded request whose stored body carries the fullest transcript.
|
||||
|
||||
Turn N's body already embeds turns 1..N-1, so the row with the most prompt tokens is
|
||||
the whole conversation and every earlier row is a prefix of it.
|
||||
"""
|
||||
rows: Final = await _query_raw(
|
||||
prisma_client,
|
||||
"""
|
||||
SELECT request_id, proxy_server_request, prompt_tokens, metadata
|
||||
FROM "LiteLLM_SpendLogs"
|
||||
WHERE session_id = $1 AND call_type = $2
|
||||
ORDER BY prompt_tokens DESC
|
||||
LIMIT 1
|
||||
""",
|
||||
session_id,
|
||||
_REPLAYABLE_CALL_TYPE,
|
||||
)
|
||||
if not rows:
|
||||
return _SourceUnavailable(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
reason=f"no {_REPLAYABLE_CALL_TYPE} spend logs found for session {session_id}",
|
||||
)
|
||||
row: Final = rows[0]
|
||||
recorded: Final = _as_mapping(row.get("proxy_server_request"))
|
||||
if recorded is None or not recorded.get("messages"):
|
||||
return _SourceUnavailable(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
reason=(
|
||||
"the recorded request body was not stored, so there is nothing to replay. "
|
||||
"Set general_settings.store_prompts_in_spend_logs to record it"
|
||||
),
|
||||
)
|
||||
metadata: Final = _as_mapping(row.get("metadata"))
|
||||
if metadata is not None and _request_mutating_guardrail_ran(metadata):
|
||||
return _SourceUnavailable(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
reason="a request-mutating guardrail ran on this session, so the stored body predates the rewrite",
|
||||
)
|
||||
prompt_tokens: Final = row.get("prompt_tokens")
|
||||
return _SourceRequest(
|
||||
request_id=str(row.get("request_id") or ""),
|
||||
body=RecordedBody.model_validate(recorded),
|
||||
recorded_prompt_tokens=prompt_tokens if isinstance(prompt_tokens, int) else 0,
|
||||
)
|
||||
|
||||
|
||||
def _arm_caller(llm_router: Router, parent_metadata: Mapping[str, object]):
|
||||
async def call(request: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return await llm_router.aanthropic_messages(
|
||||
**request,
|
||||
metadata=sanitized_forwardable_call_metadata(parent_metadata, SESSION_REPLAY_ARM_CALL_ORIGIN),
|
||||
num_retries=0,
|
||||
fallbacks=[], # mutable-ok: SDK kwarg
|
||||
)
|
||||
|
||||
return call
|
||||
|
||||
|
||||
def _judge_caller(llm_router: Router, judge_model: str, parent_metadata: Mapping[str, object]):
|
||||
async def call(system_prompt: str, user_prompt: str) -> str:
|
||||
response: Final = await llm_router.aanthropic_messages(
|
||||
model=judge_model,
|
||||
max_tokens=800,
|
||||
system=[{"type": "text", "text": system_prompt}], # mutable-ok: SDK kwarg
|
||||
messages=[{"role": "user", "content": [{"type": "text", "text": user_prompt}]}], # mutable-ok: SDK kwarg
|
||||
stream=False,
|
||||
metadata=sanitized_forwardable_call_metadata(parent_metadata, SESSION_REPLAY_JUDGE_CALL_ORIGIN),
|
||||
num_retries=0,
|
||||
fallbacks=[], # mutable-ok: SDK kwarg
|
||||
)
|
||||
blocks: Final = response.get("content") if isinstance(response, Mapping) else None
|
||||
if not isinstance(blocks, Sequence):
|
||||
return ""
|
||||
return "".join(
|
||||
str(block.get("text") or "")
|
||||
for block in blocks
|
||||
if isinstance(block, Mapping) and block.get("type") == "text"
|
||||
)
|
||||
|
||||
return call
|
||||
|
||||
|
||||
async def _reap_stale_jobs(prisma_client: PrismaClient) -> None:
|
||||
"""Fail jobs whose pod died mid-run.
|
||||
|
||||
Progress is written after every turn, so a running job with a stale updated_at is
|
||||
abandoned. Without this the partial unique index would lock that session out forever.
|
||||
"""
|
||||
cutoff: Final = datetime.now(timezone.utc) - STALE_JOB_AFTER
|
||||
await _session_replay_jobs(prisma_client).update_many(
|
||||
where={"status": "running", "updated_at": {"lt": cutoff}}, # mutable-ok: Prisma filter
|
||||
data={ # mutable-ok: Prisma update input
|
||||
"status": "failed",
|
||||
"error": "abandoned: no progress recorded",
|
||||
"finished_at": datetime.now(timezone.utc),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def _run_job(
|
||||
job_id: str,
|
||||
transcript: ReplayTranscript,
|
||||
arms: Sequence[SessionReplayArmSpec],
|
||||
request: StartSessionReplayRequest,
|
||||
recorded_prompt_tokens: int,
|
||||
llm_router: Router,
|
||||
prisma_client: PrismaClient,
|
||||
parent_metadata: Mapping[str, object],
|
||||
) -> None:
|
||||
jobs: Final = _session_replay_jobs(prisma_client)
|
||||
|
||||
async def on_progress(turns_completed: int) -> None:
|
||||
await jobs.update(
|
||||
where={"id": job_id}, data={"turns_completed": turns_completed}
|
||||
) # mutable-ok: Prisma update input
|
||||
|
||||
try:
|
||||
outcome: Final = await replay_session(
|
||||
transcript=transcript,
|
||||
arms=arms,
|
||||
max_turns=request.max_turns,
|
||||
session_id=request.session_id,
|
||||
recorded_prompt_tokens=recorded_prompt_tokens,
|
||||
call_model=_arm_caller(llm_router, parent_metadata),
|
||||
call_judge=_judge_caller(llm_router, request.judge_model, parent_metadata),
|
||||
rng=random.Random(),
|
||||
on_progress=on_progress,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 # the job records its own failure rather than dying silently
|
||||
verbose_proxy_logger.exception("session replay job %s failed", job_id)
|
||||
await jobs.update(
|
||||
where={"id": job_id}, # mutable-ok: Prisma update input
|
||||
data={ # mutable-ok: Prisma update input
|
||||
"status": "failed",
|
||||
"error": str(exc)[:1000],
|
||||
"finished_at": datetime.now(timezone.utc),
|
||||
},
|
||||
)
|
||||
return
|
||||
await jobs.update(
|
||||
where={"id": job_id}, # mutable-ok: Prisma update input
|
||||
data={ # mutable-ok: Prisma update input
|
||||
"status": "completed",
|
||||
"result": Json(outcome.model_dump()),
|
||||
"turns_completed": sum(len(run.turns) for run in outcome.arms),
|
||||
"finished_at": datetime.now(timezone.utc),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _require_admin_writer(user_api_key_dict: UserAPIKeyAuth, action: str) -> None:
|
||||
if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
|
||||
raise HTTPException(status_code=403, detail=f"Only a proxy admin can {action}")
|
||||
|
||||
|
||||
def _require_admin_viewer(user_api_key_dict: UserAPIKeyAuth, action: str) -> None:
|
||||
if user_api_key_dict.user_role not in (
|
||||
LitellmUserRoles.PROXY_ADMIN,
|
||||
LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
|
||||
):
|
||||
raise HTTPException(status_code=403, detail=f"Only proxy admin roles can {action}")
|
||||
|
||||
|
||||
def _job_response(row: Mapping[str, object]) -> SessionReplayJobResponse:
|
||||
result: Final = row.get("result")
|
||||
payload: Final = (
|
||||
result if isinstance(result, Mapping) else {}
|
||||
) # mutable-ok: empty stand-in for an unfinished job's result
|
||||
verdict: Final = payload.get("verdict")
|
||||
fidelity: Final = payload.get("fidelity")
|
||||
arms: Final = payload.get("arms")
|
||||
human_asks: Final = payload.get("human_asks")
|
||||
return SessionReplayJobResponse(
|
||||
job_id=str(row.get("id") or ""),
|
||||
session_id=str(row.get("session_id") or ""),
|
||||
status=str(row.get("status") or "running"), # pyright: ignore[reportArgumentType] # column is the literal set
|
||||
judge_model=str(row.get("judge_model") or ""),
|
||||
max_turns=int(row.get("max_turns") or 0), # pyright: ignore[reportArgumentType] # numeric column
|
||||
source_request_id=str(row["source_request_id"]) if row.get("source_request_id") else None,
|
||||
turns_completed=int(row.get("turns_completed") or 0), # pyright: ignore[reportArgumentType] # numeric column
|
||||
created_by=str(row["created_by"]) if row.get("created_by") else None,
|
||||
created_at=str(row["created_at"]) if row.get("created_at") else None,
|
||||
finished_at=str(row["finished_at"]) if row.get("finished_at") else None,
|
||||
error=str(row["error"]) if row.get("error") else None,
|
||||
human_asks=tuple(str(ask) for ask in human_asks) if isinstance(human_asks, Sequence) else (),
|
||||
arms=tuple(SessionReplayArmResponse.model_validate(arm) for arm in arms) if isinstance(arms, Sequence) else (),
|
||||
verdict=SessionReplayVerdictResponse.model_validate(verdict) if isinstance(verdict, Mapping) else None,
|
||||
fidelity=SessionReplayFidelityResponse.model_validate(fidelity) if isinstance(fidelity, Mapping) else None,
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/auto_router/session_replay/start",
|
||||
tags=("auto router",),
|
||||
dependencies=(Depends(user_api_key_auth),),
|
||||
response_model=SessionReplayJobResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
async def start_session_replay(
|
||||
data: StartSessionReplayRequest,
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
) -> SessionReplayJobResponse:
|
||||
"""
|
||||
Replay a recorded session against several model arms and judge the end results blind.
|
||||
|
||||
The recorded user and tool_result turns are the fixed environment; each arm generates
|
||||
its own assistant turns, so the arms diverge from the recording and from each other.
|
||||
Judging compares the final answer each arm reached against the human turns the
|
||||
recording actually contained.
|
||||
|
||||
Spend logs truncate stored prompts, so the replayed request is smaller than the one
|
||||
that was recorded. Every job reports that gap under `fidelity`; read a verdict against
|
||||
it rather than as a clean comparison.
|
||||
|
||||
A replay issues one billable call per recorded turn per arm, so `max_turns` bounds it
|
||||
and one session can have only one running job.
|
||||
"""
|
||||
from litellm.proxy.proxy_server import llm_router, prisma_client
|
||||
|
||||
_require_admin_writer(user_api_key_dict, "start a session replay")
|
||||
if prisma_client is None:
|
||||
raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
|
||||
if llm_router is None:
|
||||
raise HTTPException(status_code=500, detail=CommonProxyErrors.no_llm_router.value)
|
||||
|
||||
await _reap_stale_jobs(prisma_client)
|
||||
running: Final = await _query_raw(
|
||||
prisma_client,
|
||||
'SELECT COUNT(*)::int AS running FROM "LiteLLM_SessionReplayJob" WHERE status = $1',
|
||||
"running",
|
||||
)
|
||||
in_flight: Final = running[0].get("running") if running else 0
|
||||
if isinstance(in_flight, int) and in_flight >= MAX_CONCURRENT_SESSION_REPLAYS:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
|
||||
detail=f"{in_flight} session replays already running, limit is {MAX_CONCURRENT_SESSION_REPLAYS}",
|
||||
)
|
||||
|
||||
source: Final = await _load_source_request(prisma_client, data.session_id)
|
||||
if isinstance(source, _SourceUnavailable):
|
||||
raise HTTPException(status_code=source.status_code, detail=source.reason)
|
||||
|
||||
transcript: Final = build_transcript(source.body)
|
||||
if not transcript.user_turns:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail="the recorded session has no user turns to replay",
|
||||
)
|
||||
|
||||
try:
|
||||
created: Final = await _session_replay_jobs(prisma_client).create(
|
||||
data={ # mutable-ok: Prisma create input
|
||||
"session_id": data.session_id,
|
||||
"status": "running",
|
||||
"arms": Json([arm.model_dump() for arm in data.arms]), # mutable-ok: Prisma Json column payload
|
||||
"judge_model": data.judge_model,
|
||||
"max_turns": data.max_turns,
|
||||
"source_request_id": source.request_id,
|
||||
"created_by": user_api_key_dict.user_id,
|
||||
}
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 # separate the lost race from every other create failure
|
||||
if _is_unique_violation(exc):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail=f"a replay is already running for session {data.session_id}",
|
||||
) from exc
|
||||
verbose_proxy_logger.exception("could not create session replay job")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"could not create the replay job: {exc}",
|
||||
) from exc
|
||||
|
||||
parent_metadata: Final = {
|
||||
"user_api_key_user_id": user_api_key_dict.user_id
|
||||
} # mutable-ok: forwardable call metadata, an SDK kwarg
|
||||
asyncio.create_task( # detached job; its own handler records terminal state on the row
|
||||
_run_job(
|
||||
job_id=created.id,
|
||||
transcript=transcript,
|
||||
arms=data.arms,
|
||||
request=data,
|
||||
recorded_prompt_tokens=source.recorded_prompt_tokens,
|
||||
llm_router=llm_router,
|
||||
prisma_client=prisma_client,
|
||||
parent_metadata=parent_metadata,
|
||||
)
|
||||
)
|
||||
return SessionReplayJobResponse(
|
||||
job_id=created.id,
|
||||
session_id=data.session_id,
|
||||
status="running",
|
||||
judge_model=data.judge_model,
|
||||
max_turns=data.max_turns,
|
||||
source_request_id=source.request_id,
|
||||
created_by=user_api_key_dict.user_id,
|
||||
human_asks=transcript.human_asks,
|
||||
fidelity=SessionReplayFidelityResponse(
|
||||
truncated_strings=transcript.truncated_strings,
|
||||
recorded_prompt_tokens=source.recorded_prompt_tokens,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/auto_router/session_replay/{job_id}",
|
||||
tags=("auto router",),
|
||||
dependencies=(Depends(user_api_key_auth),),
|
||||
response_model=SessionReplayJobResponse,
|
||||
)
|
||||
async def get_session_replay(
|
||||
job_id: str,
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
) -> SessionReplayJobResponse:
|
||||
"""Read one session replay job, including its verdict once it has finished."""
|
||||
from litellm.proxy.proxy_server import prisma_client
|
||||
|
||||
_require_admin_viewer(user_api_key_dict, "read a session replay")
|
||||
if prisma_client is None:
|
||||
raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
|
||||
rows: Final = await _query_raw(
|
||||
prisma_client,
|
||||
'SELECT * FROM "LiteLLM_SessionReplayJob" WHERE id = $1',
|
||||
job_id,
|
||||
)
|
||||
if not rows:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"no session replay job {job_id}")
|
||||
return _job_response(rows[0])
|
||||
|
||||
|
||||
@router.get(
|
||||
"/auto_router/session_replay",
|
||||
tags=("auto router",),
|
||||
dependencies=(Depends(user_api_key_auth),),
|
||||
response_model=SessionReplayJobListResponse,
|
||||
)
|
||||
async def list_session_replays(
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
session_id: str | None = fastapi.Query(default=None, description="Only jobs for this session"),
|
||||
limit: int = fastapi.Query(default=25, ge=1, le=100),
|
||||
) -> SessionReplayJobListResponse:
|
||||
"""List session replay jobs, newest first."""
|
||||
from litellm.proxy.proxy_server import prisma_client
|
||||
|
||||
_require_admin_viewer(user_api_key_dict, "list session replays")
|
||||
if prisma_client is None:
|
||||
raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
|
||||
rows: Final = (
|
||||
await _query_raw(
|
||||
prisma_client,
|
||||
'SELECT * FROM "LiteLLM_SessionReplayJob" WHERE session_id = $1 ORDER BY created_at DESC LIMIT $2',
|
||||
session_id,
|
||||
limit,
|
||||
)
|
||||
if session_id
|
||||
else await _query_raw(
|
||||
prisma_client,
|
||||
'SELECT * FROM "LiteLLM_SessionReplayJob" ORDER BY created_at DESC LIMIT $1',
|
||||
limit,
|
||||
)
|
||||
)
|
||||
return SessionReplayJobListResponse(jobs=tuple(_job_response(row) for row in rows))
|
||||
|
|
@ -539,6 +539,9 @@ from litellm.proxy.management_endpoints.organization_endpoints import (
|
|||
from litellm.proxy.management_endpoints.router_settings_endpoints import (
|
||||
router as router_settings_router,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.session_replay_endpoints import (
|
||||
router as session_replay_management_router,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.tag_management_endpoints import (
|
||||
router as tag_management_router,
|
||||
)
|
||||
|
|
@ -18602,6 +18605,7 @@ app.include_router(budget_management_router)
|
|||
app.include_router(model_management_router)
|
||||
app.include_router(model_access_group_management_router)
|
||||
app.include_router(auto_router_management_router)
|
||||
app.include_router(session_replay_management_router)
|
||||
app.include_router(tag_management_router)
|
||||
app.include_router(workflow_management_router)
|
||||
app.include_router(memory_router)
|
||||
|
|
|
|||
|
|
@ -1592,6 +1592,33 @@ model LiteLLM_ShadowEvalFunnel {
|
|||
withheld Int @default(0)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Session Replay
|
||||
//
|
||||
// One row per replay job: a recorded session re-run against several model arms,
|
||||
// whose end results are judged blind. Single writer (the job's own task), so
|
||||
// progress and result live on the row rather than in append-only attempt rows.
|
||||
// ---------------------------------------------------------------------------
|
||||
model LiteLLM_SessionReplayJob {
|
||||
id String @id @default(cuid())
|
||||
session_id String
|
||||
status String @default("running") // running | completed | failed | stopped
|
||||
arms Json
|
||||
judge_model String
|
||||
max_turns Int
|
||||
source_request_id String? // the recorded request whose body seeded the transcript
|
||||
turns_completed Int @default(0)
|
||||
result Json?
|
||||
error String?
|
||||
created_by String?
|
||||
created_at DateTime @default(now())
|
||||
updated_at DateTime @default(now()) @updatedAt
|
||||
finished_at DateTime?
|
||||
|
||||
@@index([session_id])
|
||||
@@index([created_at])
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Workflow Run Tracking
|
||||
//
|
||||
|
|
|
|||
0
litellm/proxy/session_replay/__init__.py
Normal file
0
litellm/proxy/session_replay/__init__.py
Normal file
280
litellm/proxy/session_replay/executor.py
Normal file
280
litellm/proxy/session_replay/executor.py
Normal file
|
|
@ -0,0 +1,280 @@
|
|||
"""Replay one recorded session against several model arms and judge the end results.
|
||||
|
||||
The environment track (recorded user and tool_result turns) is fixed; each arm generates
|
||||
its own assistant turns and diverges from the recording freely. What is judged is the
|
||||
final answer each arm reached, against the human asks the recording actually contained.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import random
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final, Protocol
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from litellm.proxy.session_replay.transcript import (
|
||||
ContentBlock,
|
||||
RecordedMessage,
|
||||
ReplayTranscript,
|
||||
attach_turn,
|
||||
tool_use_ids,
|
||||
)
|
||||
from litellm.types.management_endpoints.session_replay_endpoints import (
|
||||
SessionReplayArmResponse,
|
||||
SessionReplayArmSpec,
|
||||
SessionReplayFidelityResponse,
|
||||
SessionReplayResult,
|
||||
SessionReplayTurnResponse,
|
||||
SessionReplayVerdictResponse,
|
||||
)
|
||||
|
||||
JUDGE_SYSTEM_PROMPT: Final = (
|
||||
"You are grading candidate assistants that were each given the same task and allowed to work "
|
||||
"independently. Judge only how well the final answer serves the user's request: correctness, "
|
||||
"completeness, and whether it actually did the work rather than describing it. Ignore length "
|
||||
"and style. Respond with JSON only, no prose: "
|
||||
'{"winner": "<label>", "confidence": <0.0-1.0>, "reasoning": "<one sentence>"}. '
|
||||
'Use the exact label "tie" if neither is better.'
|
||||
)
|
||||
|
||||
_MAX_JUDGE_EXCERPT_CHARS: Final = 6000
|
||||
|
||||
|
||||
class Usage(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
input_tokens: int = 0
|
||||
output_tokens: int = 0
|
||||
cache_read_input_tokens: int = 0
|
||||
cache_creation_input_tokens: int = 0
|
||||
|
||||
def prompt_tokens(self) -> int:
|
||||
return self.input_tokens + self.cache_read_input_tokens + self.cache_creation_input_tokens
|
||||
|
||||
|
||||
class ArmResponse(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
model: str | None = None
|
||||
stop_reason: str | None = None
|
||||
content: tuple[ContentBlock, ...] = ()
|
||||
usage: Usage = Usage()
|
||||
|
||||
def text(self) -> str:
|
||||
return "".join(block.text or "" for block in self.content if block.type == "text")
|
||||
|
||||
|
||||
class ModelCaller(Protocol):
|
||||
async def __call__(self, request: Mapping[str, object]) -> Mapping[str, object]: ...
|
||||
|
||||
|
||||
class JudgeCaller(Protocol):
|
||||
async def __call__(self, system_prompt: str, user_prompt: str) -> str: ...
|
||||
|
||||
|
||||
class ProgressSink(Protocol):
|
||||
async def __call__(self, turns_completed: int) -> None: ...
|
||||
|
||||
|
||||
async def _noop_progress(turns_completed: int) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _skipped_turn(index: int) -> SessionReplayTurnResponse:
|
||||
return SessionReplayTurnResponse(turn=index, attached=False)
|
||||
|
||||
|
||||
def _failed_turn(index: int, error: str) -> SessionReplayTurnResponse:
|
||||
return SessionReplayTurnResponse(turn=index, attached=True, error=error[:500])
|
||||
|
||||
|
||||
def _request_body(
|
||||
transcript: ReplayTranscript,
|
||||
arm: SessionReplayArmSpec,
|
||||
messages: Sequence[RecordedMessage],
|
||||
session_id: str,
|
||||
) -> Mapping[str, object]:
|
||||
return { # mutable-ok: Anthropic Messages wire body
|
||||
**{key: value for key, value in transcript.sampling_params}, # mutable-ok: Anthropic Messages wire body
|
||||
"model": arm.model,
|
||||
"messages": [
|
||||
message.model_dump(exclude_none=True) for message in messages
|
||||
], # mutable-ok: Anthropic Messages wire body
|
||||
"system": [
|
||||
block.model_dump(exclude_none=True) for block in transcript.system
|
||||
], # mutable-ok: Anthropic Messages wire body
|
||||
"tools": [
|
||||
tool.model_dump(exclude_none=True) for tool in transcript.tools
|
||||
], # mutable-ok: Anthropic Messages wire body
|
||||
"stream": False,
|
||||
"litellm_session_id": session_id,
|
||||
}
|
||||
|
||||
|
||||
async def run_arm(
|
||||
transcript: ReplayTranscript,
|
||||
arm: SessionReplayArmSpec,
|
||||
max_turns: int,
|
||||
session_id: str,
|
||||
call_model: ModelCaller,
|
||||
on_progress: ProgressSink = _noop_progress,
|
||||
) -> SessionReplayArmResponse:
|
||||
"""Walk the recorded user turns, letting the arm build its own assistant trajectory.
|
||||
|
||||
A turn that fails is recorded and the walk continues: one provider error should cost a
|
||||
turn, not the whole job.
|
||||
"""
|
||||
history: Final[list[RecordedMessage]] = [] # mutable-ok: the arm's trajectory is built turn by turn
|
||||
turns: Final[list[SessionReplayTurnResponse]] = [] # mutable-ok: one append per recorded turn
|
||||
pending: tuple[str, ...] = () # rebind-ok: carries the previous turn's tool_use ids forward
|
||||
final_text = "" # rebind-ok: the last turn that produced text wins
|
||||
for index, recorded_turn in enumerate(transcript.user_turns[:max_turns]):
|
||||
attached = attach_turn(recorded_turn, pending)
|
||||
if attached is None:
|
||||
turns.append(_skipped_turn(index))
|
||||
await on_progress(len(turns))
|
||||
continue
|
||||
history.append(attached)
|
||||
try:
|
||||
raw = await call_model(_request_body(transcript, arm, history, session_id))
|
||||
response = ArmResponse.model_validate(raw)
|
||||
except Exception as exc: # noqa: BLE001 # a failed turn is recorded, never fatal to the job
|
||||
turns.append(_failed_turn(index, str(exc)))
|
||||
await on_progress(len(turns))
|
||||
continue
|
||||
history.append(RecordedMessage(role="assistant", content=response.content))
|
||||
pending = tool_use_ids(response.content)
|
||||
final_text = response.text() or final_text
|
||||
turns.append(
|
||||
SessionReplayTurnResponse(
|
||||
turn=index,
|
||||
attached=True,
|
||||
served_model=response.model,
|
||||
stop_reason=response.stop_reason,
|
||||
text_chars=len(response.text()),
|
||||
tool_calls=len(pending),
|
||||
prompt_tokens=response.usage.prompt_tokens(),
|
||||
output_tokens=response.usage.output_tokens,
|
||||
)
|
||||
)
|
||||
await on_progress(len(turns))
|
||||
return SessionReplayArmResponse(
|
||||
label=arm.label,
|
||||
model=arm.model,
|
||||
served_models=tuple(dict.fromkeys(turn.served_model for turn in turns if turn.served_model)),
|
||||
turns=tuple(turns),
|
||||
final_text=final_text,
|
||||
)
|
||||
|
||||
|
||||
def _judge_prompt(human_asks: Sequence[str], runs: Sequence[SessionReplayArmResponse], order: Sequence[int]) -> str:
|
||||
asks: Final = "\n\n".join(ask[:_MAX_JUDGE_EXCERPT_CHARS] for ask in human_asks) or "(no human turn recorded)"
|
||||
answers: Final = "\n\n".join(
|
||||
f"--- CANDIDATE {chr(ord('A') + position)} ---\n{runs[arm_index].final_text[:_MAX_JUDGE_EXCERPT_CHARS]}"
|
||||
for position, arm_index in enumerate(order)
|
||||
)
|
||||
return f"THE USER ASKED:\n{asks}\n\n{answers}"
|
||||
|
||||
|
||||
def _unmask(position: str, order: Sequence[int], runs: Sequence[SessionReplayArmResponse]) -> str | None:
|
||||
index: Final = ord(position.strip().upper()) - ord("A")
|
||||
if index < 0 or index >= len(order):
|
||||
return None
|
||||
return runs[order[index]].label
|
||||
|
||||
|
||||
def _parse_verdict(raw: str) -> tuple[str, float | None, str] | None:
|
||||
start: Final = raw.find("{")
|
||||
end: Final = raw.rfind("}")
|
||||
if start < 0 or end <= start:
|
||||
return None
|
||||
try:
|
||||
payload: Final = json.loads(raw[start : end + 1])
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
winner: Final = payload.get("winner")
|
||||
if not isinstance(winner, str):
|
||||
return None
|
||||
confidence: Final = payload.get("confidence")
|
||||
reasoning: Final = payload.get("reasoning")
|
||||
return (
|
||||
winner,
|
||||
float(confidence) if isinstance(confidence, (int, float)) else None,
|
||||
reasoning if isinstance(reasoning, str) else "",
|
||||
)
|
||||
|
||||
|
||||
async def judge_runs(
|
||||
human_asks: Sequence[str],
|
||||
runs: Sequence[SessionReplayArmResponse],
|
||||
call_judge: JudgeCaller,
|
||||
rng: random.Random,
|
||||
) -> SessionReplayVerdictResponse:
|
||||
"""Blind pairwise judging: arms are presented as A/B in randomized order and unmasked after."""
|
||||
if len(runs) < 2:
|
||||
return SessionReplayVerdictResponse(error="need at least two arms to judge")
|
||||
order: Final = tuple(rng.sample(range(len(runs)), len(runs)))
|
||||
try:
|
||||
raw: Final = await call_judge(JUDGE_SYSTEM_PROMPT, _judge_prompt(human_asks, runs, order))
|
||||
except Exception as exc: # noqa: BLE001 # a failed judge is a recorded outcome, not a failed job
|
||||
return SessionReplayVerdictResponse(error=str(exc)[:500])
|
||||
parsed: Final = _parse_verdict(raw)
|
||||
if parsed is None:
|
||||
return SessionReplayVerdictResponse(error=f"unparseable judge response: {raw[:200]}")
|
||||
position, confidence, reasoning = parsed
|
||||
if position == "tie":
|
||||
return SessionReplayVerdictResponse(winner="tie", confidence=confidence, reasoning=reasoning)
|
||||
unmasked: Final = _unmask(position, order, runs)
|
||||
if unmasked is None:
|
||||
return SessionReplayVerdictResponse(reasoning=reasoning, error=f"judge named unknown label {position}")
|
||||
return SessionReplayVerdictResponse(winner=unmasked, confidence=confidence, reasoning=reasoning)
|
||||
|
||||
|
||||
async def _run_arms(
|
||||
transcript: ReplayTranscript,
|
||||
arms: Sequence[SessionReplayArmSpec],
|
||||
max_turns: int,
|
||||
session_id: str,
|
||||
call_model: ModelCaller,
|
||||
on_progress: ProgressSink,
|
||||
) -> tuple[SessionReplayArmResponse, ...]:
|
||||
"""Progress counts turns across the whole job, not per arm, so the heartbeat a stale-job
|
||||
reaper reads never goes backwards when the next arm starts."""
|
||||
runs: Final[list[SessionReplayArmResponse]] = [] # mutable-ok: one append per finished arm
|
||||
for arm in arms:
|
||||
done_before = sum(len(run.turns) for run in runs)
|
||||
|
||||
async def cumulative(turns_completed: int, offset: int = done_before) -> None:
|
||||
await on_progress(offset + turns_completed)
|
||||
|
||||
runs.append(await run_arm(transcript, arm, max_turns, f"{session_id}-{arm.label}", call_model, cumulative))
|
||||
return tuple(runs)
|
||||
|
||||
|
||||
async def replay_session(
|
||||
transcript: ReplayTranscript,
|
||||
arms: Sequence[SessionReplayArmSpec],
|
||||
max_turns: int,
|
||||
session_id: str,
|
||||
recorded_prompt_tokens: int,
|
||||
call_model: ModelCaller,
|
||||
call_judge: JudgeCaller,
|
||||
rng: random.Random,
|
||||
on_progress: ProgressSink = _noop_progress,
|
||||
) -> SessionReplayResult:
|
||||
runs: Final = await _run_arms(transcript, arms, max_turns, session_id, call_model, on_progress)
|
||||
first_turn_tokens: Final = next((turn.prompt_tokens for run in runs for turn in run.turns if turn.prompt_tokens), 0)
|
||||
return SessionReplayResult(
|
||||
human_asks=transcript.human_asks,
|
||||
arms=runs,
|
||||
verdict=await judge_runs(transcript.human_asks, runs, call_judge, rng),
|
||||
fidelity=SessionReplayFidelityResponse(
|
||||
truncated_strings=transcript.truncated_strings,
|
||||
recorded_prompt_tokens=recorded_prompt_tokens,
|
||||
replayed_first_turn_prompt_tokens=first_turn_tokens,
|
||||
),
|
||||
)
|
||||
179
litellm/proxy/session_replay/transcript.py
Normal file
179
litellm/proxy/session_replay/transcript.py
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
"""Rebuild a replayable Anthropic Messages request from a recorded spend-log body.
|
||||
|
||||
A recorded `proxy_server_request` is a post-guardrail snapshot of the proxy's mutated
|
||||
request dict, not the caller's wire body, so it needs three repairs before a provider
|
||||
will accept it back: stray `role: "system"` messages belong in the top-level `system`
|
||||
array, proxy-injected keys must go, and every string longer than
|
||||
`MAX_STRING_LENGTH_PROMPT_IN_DB` was truncated on the way in.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
TRUNCATION_MARKER: Final = "litellm_truncated"
|
||||
|
||||
_PROXY_INJECTED_KEYS: Final = frozenset(
|
||||
{
|
||||
"headers",
|
||||
"litellm_metadata",
|
||||
"litellm_session_id",
|
||||
"litellm_trace_id",
|
||||
"metadata",
|
||||
"provider_specific_header",
|
||||
"secret_fields",
|
||||
}
|
||||
)
|
||||
|
||||
_STRUCTURAL_KEYS: Final = frozenset({"model", "messages", "system", "tools", "stream"})
|
||||
|
||||
_SYSTEM_REMINDER: Final = re.compile(r"<system-reminder>.*?</system-reminder>", re.DOTALL)
|
||||
|
||||
|
||||
class ContentBlock(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
type: str
|
||||
text: str | None = None
|
||||
id: str | None = None
|
||||
tool_use_id: str | None = None
|
||||
content: str | None = None
|
||||
|
||||
|
||||
class ToolDefinition(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
name: str
|
||||
description: str | None = None
|
||||
|
||||
|
||||
class RecordedMessage(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
role: str
|
||||
content: str | tuple[ContentBlock, ...]
|
||||
|
||||
def blocks(self) -> tuple[ContentBlock, ...]:
|
||||
if isinstance(self.content, str):
|
||||
return (ContentBlock(type="text", text=self.content),)
|
||||
return self.content
|
||||
|
||||
def block_types(self) -> frozenset[str]:
|
||||
return frozenset(block.type for block in self.blocks())
|
||||
|
||||
def text(self) -> str:
|
||||
return "".join(block.text or "" for block in self.blocks() if block.type == "text")
|
||||
|
||||
|
||||
class RecordedBody(BaseModel):
|
||||
model_config = ConfigDict(extra="allow")
|
||||
|
||||
model: str | None = None
|
||||
messages: tuple[RecordedMessage, ...] = ()
|
||||
system: str | tuple[ContentBlock, ...] = ()
|
||||
tools: tuple[ToolDefinition, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReplayTranscript:
|
||||
system: tuple[ContentBlock, ...]
|
||||
tools: tuple[ToolDefinition, ...]
|
||||
user_turns: tuple[RecordedMessage, ...]
|
||||
sampling_params: tuple[tuple[str, object], ...]
|
||||
human_asks: tuple[str, ...]
|
||||
truncated_strings: int
|
||||
recorded_model: str | None
|
||||
|
||||
|
||||
def _count_truncations(value: object) -> int:
|
||||
if isinstance(value, str):
|
||||
return 1 if TRUNCATION_MARKER in value else 0
|
||||
if isinstance(value, dict):
|
||||
return sum(_count_truncations(item) for item in value.values())
|
||||
if isinstance(value, (list, tuple)):
|
||||
return sum(_count_truncations(item) for item in value)
|
||||
return 0
|
||||
|
||||
|
||||
def _as_system_blocks(system: str | tuple[ContentBlock, ...]) -> tuple[ContentBlock, ...]:
|
||||
if isinstance(system, str):
|
||||
return (ContentBlock(type="text", text=system),)
|
||||
return system
|
||||
|
||||
|
||||
def _hoisted_system_block(message: RecordedMessage) -> ContentBlock:
|
||||
return ContentBlock(type="text", text=message.text())
|
||||
|
||||
|
||||
def human_ask(message: RecordedMessage) -> str:
|
||||
return _SYSTEM_REMINDER.sub("", message.text()).strip()
|
||||
|
||||
|
||||
def build_transcript(recorded_body: RecordedBody) -> ReplayTranscript:
|
||||
"""Split a recorded body into a fixed environment track and the arm's own policy track.
|
||||
|
||||
`role: "system"` messages are hoisted rather than dropped: the Anthropic Messages API
|
||||
rejects that role inside `messages`, but the text is real context the recorded model saw.
|
||||
"""
|
||||
hoisted: Final = tuple(
|
||||
_hoisted_system_block(message) for message in recorded_body.messages if message.role == "system"
|
||||
)
|
||||
user_turns: Final = tuple(message for message in recorded_body.messages if message.role == "user")
|
||||
asks: Final = tuple(
|
||||
ask for message in user_turns if "tool_result" not in message.block_types() and (ask := human_ask(message))
|
||||
)
|
||||
sampling: Final = tuple(
|
||||
(key, value)
|
||||
for key, value in recorded_body.model_dump(exclude_none=True).items()
|
||||
if key not in _PROXY_INJECTED_KEYS and key not in _STRUCTURAL_KEYS
|
||||
)
|
||||
return ReplayTranscript(
|
||||
system=_as_system_blocks(recorded_body.system) + hoisted,
|
||||
tools=recorded_body.tools,
|
||||
user_turns=user_turns,
|
||||
sampling_params=sampling,
|
||||
human_asks=asks,
|
||||
truncated_strings=_count_truncations(recorded_body.model_dump()),
|
||||
recorded_model=recorded_body.model,
|
||||
)
|
||||
|
||||
|
||||
def tool_use_ids(assistant_blocks: Sequence[ContentBlock]) -> tuple[str, ...]:
|
||||
return tuple(block.id for block in assistant_blocks if block.type == "tool_use" and block.id is not None)
|
||||
|
||||
|
||||
def attach_turn(turn: RecordedMessage, pending_tool_use_ids: Sequence[str]) -> RecordedMessage | None:
|
||||
"""Bind a recorded user turn onto the arm's own trajectory, or drop it as unattachable.
|
||||
|
||||
Recorded `tool_result` blocks carry the original run's `tool_use_id`s, which name
|
||||
`tool_use` blocks this arm never emitted. When the arm did call tools the results are
|
||||
rebound onto its ids in order; when it called none there is nothing to bind to, and
|
||||
injecting the recording's tool output as prose would feed the arm answers to questions
|
||||
it never asked, so the turn is dropped instead.
|
||||
"""
|
||||
blocks: Final = turn.blocks()
|
||||
results: Final = tuple(block for block in blocks if block.type == "tool_result")
|
||||
if not results:
|
||||
return turn
|
||||
if not pending_tool_use_ids:
|
||||
return None
|
||||
rebound: Final = tuple(
|
||||
block.model_copy(
|
||||
update={"tool_use_id": pending_tool_use_ids[min(index, len(pending_tool_use_ids) - 1)]}
|
||||
) # mutable-ok: pydantic model_copy update mapping
|
||||
if block.type == "tool_result"
|
||||
else block
|
||||
for index, block in enumerate(blocks)
|
||||
)
|
||||
answered: Final = frozenset(block.tool_use_id for block in rebound if block.type == "tool_result")
|
||||
stubs: Final = tuple(
|
||||
ContentBlock(type="tool_result", tool_use_id=tool_id, content="[no recorded result]")
|
||||
for tool_id in pending_tool_use_ids
|
||||
if tool_id not in answered
|
||||
)
|
||||
return RecordedMessage(role="user", content=rebound + stubs)
|
||||
121
litellm/types/management_endpoints/session_replay_endpoints.py
Normal file
121
litellm/types/management_endpoints/session_replay_endpoints.py
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Final, Literal
|
||||
|
||||
from pydantic import BaseModel, Field, computed_field, field_validator
|
||||
|
||||
MAX_SESSION_REPLAY_TURNS: Final = 200
|
||||
MAX_SESSION_REPLAY_ARMS: Final = 4
|
||||
|
||||
SessionReplayStatus = Literal["running", "completed", "failed", "stopped"]
|
||||
|
||||
|
||||
class SessionReplayArmSpec(BaseModel):
|
||||
label: str = Field(min_length=1, max_length=64)
|
||||
model: str = Field(min_length=1)
|
||||
|
||||
|
||||
class StartSessionReplayRequest(BaseModel):
|
||||
session_id: str = Field(min_length=1)
|
||||
arms: tuple[SessionReplayArmSpec, ...]
|
||||
judge_model: str = Field(min_length=1)
|
||||
max_turns: int = Field(default=20, ge=1, le=MAX_SESSION_REPLAY_TURNS)
|
||||
|
||||
@field_validator("arms")
|
||||
@classmethod
|
||||
def _at_least_two_distinct_arms(cls, arms: tuple[SessionReplayArmSpec, ...]) -> tuple[SessionReplayArmSpec, ...]:
|
||||
if len(arms) < 2:
|
||||
raise ValueError("a replay needs at least two arms to compare")
|
||||
if len(arms) > MAX_SESSION_REPLAY_ARMS:
|
||||
raise ValueError(f"at most {MAX_SESSION_REPLAY_ARMS} arms per replay")
|
||||
labels: Final = tuple(arm.label for arm in arms)
|
||||
if len(frozenset(labels)) != len(labels):
|
||||
raise ValueError("arm labels must be unique")
|
||||
return arms
|
||||
|
||||
|
||||
class SessionReplayTurnResponse(BaseModel):
|
||||
turn: int
|
||||
attached: bool
|
||||
served_model: str | None = None
|
||||
stop_reason: str | None = None
|
||||
text_chars: int = 0
|
||||
tool_calls: int = 0
|
||||
prompt_tokens: int = 0
|
||||
output_tokens: int = 0
|
||||
error: str | None = None
|
||||
|
||||
|
||||
class SessionReplayArmResponse(BaseModel):
|
||||
label: str
|
||||
model: str
|
||||
served_models: tuple[str, ...] = ()
|
||||
turns: tuple[SessionReplayTurnResponse, ...] = ()
|
||||
final_text: str = ""
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def prompt_tokens(self) -> int:
|
||||
return sum(turn.prompt_tokens for turn in self.turns)
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def output_tokens(self) -> int:
|
||||
return sum(turn.output_tokens for turn in self.turns)
|
||||
|
||||
|
||||
class SessionReplayVerdictResponse(BaseModel):
|
||||
winner: str | None = None
|
||||
confidence: float | None = None
|
||||
reasoning: str = ""
|
||||
error: str | None = None
|
||||
|
||||
|
||||
class SessionReplayFidelityResponse(BaseModel):
|
||||
"""How much of the recorded prompt actually survived into the replay.
|
||||
|
||||
Spend logs truncate every stored string at MAX_STRING_LENGTH_PROMPT_IN_DB, so a replay
|
||||
runs against a smaller prompt than the recording did. Reported on every job so a result
|
||||
is never read as a clean comparison when it is not one.
|
||||
"""
|
||||
|
||||
truncated_strings: int = 0
|
||||
recorded_prompt_tokens: int = 0
|
||||
replayed_first_turn_prompt_tokens: int = 0
|
||||
|
||||
@property
|
||||
def prompt_retained_ratio(self) -> float | None:
|
||||
if not self.recorded_prompt_tokens:
|
||||
return None
|
||||
return self.replayed_first_turn_prompt_tokens / self.recorded_prompt_tokens
|
||||
|
||||
|
||||
class SessionReplayResult(BaseModel):
|
||||
"""Everything a finished replay produced, stored verbatim as the job row's result."""
|
||||
|
||||
human_asks: tuple[str, ...] = ()
|
||||
arms: tuple[SessionReplayArmResponse, ...] = ()
|
||||
verdict: SessionReplayVerdictResponse = SessionReplayVerdictResponse()
|
||||
fidelity: SessionReplayFidelityResponse = SessionReplayFidelityResponse()
|
||||
|
||||
|
||||
class SessionReplayJobResponse(BaseModel):
|
||||
job_id: str
|
||||
session_id: str
|
||||
status: SessionReplayStatus
|
||||
judge_model: str
|
||||
max_turns: int
|
||||
source_request_id: str | None = None
|
||||
turns_completed: int = 0
|
||||
created_by: str | None = None
|
||||
created_at: str | None = None
|
||||
finished_at: str | None = None
|
||||
error: str | None = None
|
||||
human_asks: tuple[str, ...] = ()
|
||||
arms: tuple[SessionReplayArmResponse, ...] = ()
|
||||
verdict: SessionReplayVerdictResponse | None = None
|
||||
fidelity: SessionReplayFidelityResponse | None = None
|
||||
|
||||
|
||||
class SessionReplayJobListResponse(BaseModel):
|
||||
jobs: tuple[SessionReplayJobResponse, ...] = ()
|
||||
|
|
@ -2938,6 +2938,8 @@ InternalCallOrigin = Literal[
|
|||
"shadow_eval_router",
|
||||
"shadow_eval_judge",
|
||||
"background_response_cost_poll",
|
||||
"session_replay_arm",
|
||||
"session_replay_judge",
|
||||
]
|
||||
"""Which internal litellm feature originated a billed sub-call, so a spend log row
|
||||
records that it is not traffic the caller sent."""
|
||||
|
|
@ -2946,6 +2948,8 @@ AUTOROUTER_CLASSIFIER_CALL_ORIGIN: Final[InternalCallOrigin] = "autorouter_class
|
|||
SHADOW_EVAL_ROUTER_CALL_ORIGIN: Final[InternalCallOrigin] = "shadow_eval_router"
|
||||
SHADOW_EVAL_JUDGE_CALL_ORIGIN: Final[InternalCallOrigin] = "shadow_eval_judge"
|
||||
BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN: Final[InternalCallOrigin] = "background_response_cost_poll"
|
||||
SESSION_REPLAY_ARM_CALL_ORIGIN: Final[InternalCallOrigin] = "session_replay_arm"
|
||||
SESSION_REPLAY_JUDGE_CALL_ORIGIN: Final[InternalCallOrigin] = "session_replay_judge"
|
||||
|
||||
|
||||
class StandardLoggingRoutingDecision(TypedDict, total=False):
|
||||
|
|
|
|||
|
|
@ -1592,6 +1592,33 @@ model LiteLLM_ShadowEvalFunnel {
|
|||
withheld Int @default(0)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Session Replay
|
||||
//
|
||||
// One row per replay job: a recorded session re-run against several model arms,
|
||||
// whose end results are judged blind. Single writer (the job's own task), so
|
||||
// progress and result live on the row rather than in append-only attempt rows.
|
||||
// ---------------------------------------------------------------------------
|
||||
model LiteLLM_SessionReplayJob {
|
||||
id String @id @default(cuid())
|
||||
session_id String
|
||||
status String @default("running") // running | completed | failed | stopped
|
||||
arms Json
|
||||
judge_model String
|
||||
max_turns Int
|
||||
source_request_id String? // the recorded request whose body seeded the transcript
|
||||
turns_completed Int @default(0)
|
||||
result Json?
|
||||
error String?
|
||||
created_by String?
|
||||
created_at DateTime @default(now())
|
||||
updated_at DateTime @default(now()) @updatedAt
|
||||
finished_at DateTime?
|
||||
|
||||
@@index([session_id])
|
||||
@@index([created_at])
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Workflow Run Tracking
|
||||
//
|
||||
|
|
|
|||
|
|
@ -0,0 +1,215 @@
|
|||
import json
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.management_endpoints.session_replay_endpoints import (
|
||||
_job_response,
|
||||
_load_source_request,
|
||||
_require_admin_viewer,
|
||||
_require_admin_writer,
|
||||
_SourceRequest,
|
||||
_SourceUnavailable,
|
||||
)
|
||||
|
||||
RECORDED = {
|
||||
"model": "router",
|
||||
"max_tokens": 1024,
|
||||
"system": [{"type": "text", "text": "sys"}],
|
||||
"messages": [{"role": "user", "content": [{"type": "text", "text": "hello"}]}],
|
||||
}
|
||||
|
||||
|
||||
class _FakePrisma:
|
||||
def __init__(self, rows):
|
||||
self.rows = rows
|
||||
self.queries = []
|
||||
|
||||
class _Db:
|
||||
def __init__(self, outer):
|
||||
self.outer = outer
|
||||
|
||||
async def query_raw(self, query, *args):
|
||||
self.outer.queries.append((query, args))
|
||||
return self.outer.rows
|
||||
|
||||
@property
|
||||
def db(self):
|
||||
return self._Db(self)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_with_no_replayable_rows_is_a_404():
|
||||
result = await _load_source_request(_FakePrisma([]), "missing-session")
|
||||
|
||||
assert isinstance(result, _SourceUnavailable)
|
||||
assert result.status_code == 404
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_whose_body_was_not_stored_explains_the_config_gate():
|
||||
"""store_prompts_in_spend_logs off writes an empty object, so the row exists but holds
|
||||
nothing to replay: that must not read as a missing session."""
|
||||
result = await _load_source_request(
|
||||
_FakePrisma([{"request_id": "r1", "proxy_server_request": {}, "prompt_tokens": 10, "metadata": {}}]),
|
||||
"sess",
|
||||
)
|
||||
|
||||
assert isinstance(result, _SourceUnavailable)
|
||||
assert result.status_code == 409
|
||||
assert "store_prompts_in_spend_logs" in result.reason
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_rewritten_by_a_mutating_guardrail_is_refused():
|
||||
"""The stored body predates the guardrail's rewrite, so replaying it would resurrect
|
||||
content the guardrail stripped."""
|
||||
rows = [
|
||||
{
|
||||
"request_id": "r1",
|
||||
"proxy_server_request": RECORDED,
|
||||
"prompt_tokens": 10,
|
||||
"metadata": {
|
||||
"standard_logging_guardrail_information": [{"guardrail_mode": "pre_call"}],
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
result = await _load_source_request(_FakePrisma(rows), "sess")
|
||||
|
||||
assert isinstance(result, _SourceUnavailable)
|
||||
assert result.status_code == 409
|
||||
assert "guardrail" in result.reason
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_replayable_session_returns_the_parsed_body_and_recorded_token_count():
|
||||
rows = [
|
||||
{
|
||||
"request_id": "req-42",
|
||||
"proxy_server_request": RECORDED,
|
||||
"prompt_tokens": 41920,
|
||||
"metadata": {},
|
||||
}
|
||||
]
|
||||
|
||||
result = await _load_source_request(_FakePrisma(rows), "sess")
|
||||
|
||||
assert isinstance(result, _SourceRequest)
|
||||
assert result.request_id == "req-42"
|
||||
assert result.recorded_prompt_tokens == 41920
|
||||
assert len(result.body.messages) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_source_lookup_is_scoped_to_replayable_call_types():
|
||||
"""A chat-completions row cannot be replayed through the Anthropic Messages path, so
|
||||
picking one up would fail at the first turn instead of at start."""
|
||||
prisma = _FakePrisma([])
|
||||
|
||||
await _load_source_request(prisma, "sess")
|
||||
|
||||
_, args = prisma.queries[0]
|
||||
assert args == ("sess", "anthropic_messages")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"role",
|
||||
[
|
||||
LitellmUserRoles.INTERNAL_USER,
|
||||
LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
|
||||
LitellmUserRoles.TEAM,
|
||||
None,
|
||||
],
|
||||
)
|
||||
def test_only_a_proxy_admin_can_start_a_replay(role):
|
||||
"""A replay reads another key's recorded prompts and spends real money on them."""
|
||||
with pytest.raises(HTTPException) as excinfo:
|
||||
_require_admin_writer(UserAPIKeyAuth(api_key="k", user_role=role), "start a session replay")
|
||||
|
||||
assert excinfo.value.status_code == 403
|
||||
|
||||
|
||||
def test_proxy_admin_may_start_a_replay():
|
||||
admin = UserAPIKeyAuth(api_key="k", user_role=LitellmUserRoles.PROXY_ADMIN)
|
||||
|
||||
assert _require_admin_writer(admin, "start") is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"role",
|
||||
[LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY],
|
||||
)
|
||||
def test_admin_viewers_may_read_a_replay(role):
|
||||
assert _require_admin_viewer(UserAPIKeyAuth(api_key="k", user_role=role), "read") is None
|
||||
|
||||
|
||||
def test_internal_user_cannot_read_a_replay():
|
||||
with pytest.raises(HTTPException) as excinfo:
|
||||
_require_admin_viewer(UserAPIKeyAuth(api_key="k", user_role=LitellmUserRoles.INTERNAL_USER), "read")
|
||||
|
||||
assert excinfo.value.status_code == 403
|
||||
|
||||
|
||||
def test_finished_job_row_surfaces_verdict_and_fidelity():
|
||||
row = {
|
||||
"id": "job-1",
|
||||
"session_id": "sess",
|
||||
"status": "completed",
|
||||
"judge_model": "judge",
|
||||
"max_turns": 20,
|
||||
"source_request_id": "req-42",
|
||||
"turns_completed": 3,
|
||||
"created_by": "admin",
|
||||
"result": {
|
||||
"human_asks": ["plan my week"],
|
||||
"arms": [
|
||||
{"label": "auto_router", "model": "r", "final_text": "a", "turns": []},
|
||||
{"label": "baseline", "model": "b", "final_text": "c", "turns": []},
|
||||
],
|
||||
"verdict": {"winner": "auto_router", "confidence": 0.72, "reasoning": "more complete"},
|
||||
"fidelity": {
|
||||
"truncated_strings": 6,
|
||||
"recorded_prompt_tokens": 41920,
|
||||
"replayed_first_turn_prompt_tokens": 16016,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
response = _job_response(row)
|
||||
|
||||
assert response.status == "completed"
|
||||
assert response.verdict is not None and response.verdict.winner == "auto_router"
|
||||
assert response.fidelity is not None
|
||||
assert response.fidelity.prompt_retained_ratio == pytest.approx(16016 / 41920)
|
||||
assert tuple(arm.label for arm in response.arms) == ("auto_router", "baseline")
|
||||
|
||||
|
||||
def test_running_job_row_has_no_verdict_yet():
|
||||
response = _job_response(
|
||||
{"id": "job-2", "session_id": "s", "status": "running", "judge_model": "j", "max_turns": 5}
|
||||
)
|
||||
|
||||
assert response.status == "running"
|
||||
assert response.verdict is None
|
||||
assert response.arms == ()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_jsonb_columns_returned_as_text_are_still_parsed():
|
||||
"""query_raw hands a jsonb column back as a dict or as raw JSON text depending on the
|
||||
driver, and treating the text form as absent would report every session unreplayable."""
|
||||
rows = [
|
||||
{
|
||||
"request_id": "req-1",
|
||||
"proxy_server_request": json.dumps(RECORDED),
|
||||
"prompt_tokens": 100,
|
||||
"metadata": "{}",
|
||||
}
|
||||
]
|
||||
|
||||
result = await _load_source_request(_FakePrisma(rows), "sess")
|
||||
|
||||
assert isinstance(result, _SourceRequest)
|
||||
assert len(result.body.messages) == 1
|
||||
0
tests/test_litellm/proxy/session_replay/__init__.py
Normal file
0
tests/test_litellm/proxy/session_replay/__init__.py
Normal file
305
tests/test_litellm/proxy/session_replay/test_executor.py
Normal file
305
tests/test_litellm/proxy/session_replay/test_executor.py
Normal file
|
|
@ -0,0 +1,305 @@
|
|||
import random
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.proxy.session_replay.executor import judge_runs, replay_session, run_arm
|
||||
from litellm.proxy.session_replay.transcript import RecordedBody, build_transcript
|
||||
from litellm.types.management_endpoints.session_replay_endpoints import SessionReplayArmSpec
|
||||
|
||||
TWO_HUMAN_TURNS = {
|
||||
"model": "recorded-router",
|
||||
"max_tokens": 1024,
|
||||
"system": [{"type": "text", "text": "be helpful"}],
|
||||
"tools": [{"name": "Read", "description": "read", "input_schema": {"type": "object", "properties": {}}}],
|
||||
"messages": [
|
||||
{"role": "user", "content": [{"type": "text", "text": "first ask"}]},
|
||||
{"role": "assistant", "content": [{"type": "tool_use", "id": "toolu_REC", "name": "Read"}]},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "tool_result", "tool_use_id": "toolu_REC", "content": "recorded output"}],
|
||||
},
|
||||
{"role": "user", "content": [{"type": "text", "text": "second ask"}]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _transcript(body=None):
|
||||
return build_transcript(RecordedBody.model_validate(body or TWO_HUMAN_TURNS))
|
||||
|
||||
|
||||
def _text_response(text, model="served-model"):
|
||||
return {
|
||||
"model": model,
|
||||
"stop_reason": "end_turn",
|
||||
"content": [{"type": "text", "text": text}],
|
||||
"usage": {"input_tokens": 10, "output_tokens": 5},
|
||||
}
|
||||
|
||||
|
||||
def _tool_response(tool_use_id, model="served-model"):
|
||||
return {
|
||||
"model": model,
|
||||
"stop_reason": "tool_use",
|
||||
"content": [{"type": "tool_use", "id": tool_use_id, "name": "Read", "input": {}}],
|
||||
"usage": {"input_tokens": 10, "output_tokens": 5},
|
||||
}
|
||||
|
||||
|
||||
class _RecordingCaller:
|
||||
def __init__(self, responses):
|
||||
self.responses = list(responses)
|
||||
self.requests = []
|
||||
|
||||
async def __call__(self, request):
|
||||
self.requests.append(request)
|
||||
nxt = self.responses.pop(0)
|
||||
if isinstance(nxt, Exception):
|
||||
raise nxt
|
||||
return nxt
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_arm_trajectory_accumulates_its_own_assistant_turns():
|
||||
"""The arm's own outputs, not the recording's, must be what later turns see: that is
|
||||
what makes this an end-to-end replay rather than per-turn scoring."""
|
||||
caller = _RecordingCaller([_tool_response("toolu_ARM"), _text_response("after tool"), _text_response("final")])
|
||||
|
||||
run = await run_arm(
|
||||
_transcript(), SessionReplayArmSpec(label="a", model="m"), max_turns=10, session_id="s", call_model=caller
|
||||
)
|
||||
|
||||
assert run.final_text == "final"
|
||||
assert len(caller.requests) == 3
|
||||
roles_on_last_call = [message["role"] for message in caller.requests[-1]["messages"]]
|
||||
assert roles_on_last_call == ["user", "assistant", "user", "assistant", "user"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recorded_tool_result_is_rebound_to_the_arms_tool_use_id():
|
||||
caller = _RecordingCaller([_tool_response("toolu_ARM"), _text_response("ok"), _text_response("done")])
|
||||
|
||||
await run_arm(
|
||||
_transcript(), SessionReplayArmSpec(label="a", model="m"), max_turns=10, session_id="s", call_model=caller
|
||||
)
|
||||
|
||||
second_call_last_message = caller.requests[1]["messages"][-1]
|
||||
assert [block["tool_use_id"] for block in second_call_last_message["content"]] == ["toolu_ARM"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_turn_whose_tool_result_cannot_attach_is_skipped_without_calling_the_model():
|
||||
caller = _RecordingCaller([_text_response("no tools here"), _text_response("second answer")])
|
||||
|
||||
run = await run_arm(
|
||||
_transcript(), SessionReplayArmSpec(label="a", model="m"), max_turns=10, session_id="s", call_model=caller
|
||||
)
|
||||
|
||||
assert [turn.attached for turn in run.turns] == [True, False, True]
|
||||
assert len(caller.requests) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_failing_turn_is_recorded_and_the_walk_continues():
|
||||
"""One provider error should cost a turn, not the whole job."""
|
||||
caller = _RecordingCaller([RuntimeError("upstream 401"), _text_response("recovered")])
|
||||
|
||||
run = await run_arm(
|
||||
_transcript(), SessionReplayArmSpec(label="a", model="m"), max_turns=10, session_id="s", call_model=caller
|
||||
)
|
||||
|
||||
assert run.turns[0].error is not None and "upstream 401" in run.turns[0].error
|
||||
assert run.final_text == "recovered"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_max_turns_bounds_the_number_of_billable_calls():
|
||||
caller = _RecordingCaller([_text_response("one"), _text_response("two"), _text_response("three")])
|
||||
|
||||
await run_arm(
|
||||
_transcript(), SessionReplayArmSpec(label="a", model="m"), max_turns=1, session_id="s", call_model=caller
|
||||
)
|
||||
|
||||
assert len(caller.requests) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_progress_is_reported_after_every_turn():
|
||||
"""A job whose pod dies is detected by a stale heartbeat, so progress must advance
|
||||
per turn rather than per arm."""
|
||||
caller = _RecordingCaller([_text_response("one"), _text_response("two")])
|
||||
seen = []
|
||||
|
||||
async def on_progress(turns_completed):
|
||||
seen.append(turns_completed)
|
||||
|
||||
await run_arm(
|
||||
_transcript(),
|
||||
SessionReplayArmSpec(label="a", model="m"),
|
||||
max_turns=10,
|
||||
session_id="s",
|
||||
call_model=caller,
|
||||
on_progress=on_progress,
|
||||
)
|
||||
|
||||
assert seen == [1, 2, 3]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_judge_verdict_is_unmasked_back_to_the_winning_arm_label():
|
||||
"""The judge sees anonymous A/B in randomized order, so a verdict read without
|
||||
unmasking would attribute wins to whichever arm happened to be shown first."""
|
||||
runs = []
|
||||
for label in ("alpha", "beta"):
|
||||
caller = _RecordingCaller([_text_response(f"{label} answer"), _text_response(f"{label} final")])
|
||||
runs.append(
|
||||
await run_arm(
|
||||
_transcript(),
|
||||
SessionReplayArmSpec(label=label, model="m"),
|
||||
max_turns=10,
|
||||
session_id="s",
|
||||
call_model=caller,
|
||||
)
|
||||
)
|
||||
|
||||
async def call_judge(system_prompt, user_prompt):
|
||||
shown_first = "alpha" if "alpha final" in user_prompt.split("CANDIDATE B")[0] else "beta"
|
||||
assert shown_first in ("alpha", "beta")
|
||||
return '{"winner": "A", "confidence": 0.9, "reasoning": "clearer"}'
|
||||
|
||||
for seed in range(6):
|
||||
verdict = await judge_runs(("ask",), runs, call_judge, random.Random(seed))
|
||||
order = random.Random(seed).sample(range(2), 2)
|
||||
assert verdict.winner == runs[order[0]].label
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unparseable_judge_response_is_a_recorded_outcome_not_an_exception():
|
||||
caller = _RecordingCaller([_text_response("x"), _text_response("y")])
|
||||
run = await run_arm(
|
||||
_transcript(), SessionReplayArmSpec(label="a", model="m"), max_turns=10, session_id="s", call_model=caller
|
||||
)
|
||||
|
||||
async def call_judge(system_prompt, user_prompt):
|
||||
return "I prefer the first one"
|
||||
|
||||
verdict = await judge_runs(("ask",), (run, run), call_judge, random.Random(0))
|
||||
|
||||
assert verdict.winner is None
|
||||
assert verdict.error is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_judge_failure_does_not_fail_the_job():
|
||||
caller = _RecordingCaller([_text_response("x"), _text_response("y")])
|
||||
run = await run_arm(
|
||||
_transcript(), SessionReplayArmSpec(label="a", model="m"), max_turns=10, session_id="s", call_model=caller
|
||||
)
|
||||
|
||||
async def call_judge(system_prompt, user_prompt):
|
||||
raise RuntimeError("judge model unavailable")
|
||||
|
||||
verdict = await judge_runs(("ask",), (run, run), call_judge, random.Random(0))
|
||||
|
||||
assert verdict.winner is None
|
||||
assert verdict.error is not None and "judge model unavailable" in verdict.error
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fidelity_reports_the_gap_between_recorded_and_replayed_prompt():
|
||||
"""Spend logs truncate stored prompts, so a replay runs on a smaller prompt than the
|
||||
recording did and a verdict must never be read as a clean comparison."""
|
||||
|
||||
async def call_model(request):
|
||||
return _text_response("answer")
|
||||
|
||||
async def call_judge(system_prompt, user_prompt):
|
||||
return '{"winner": "tie", "confidence": 0.5, "reasoning": "same"}'
|
||||
|
||||
outcome = await replay_session(
|
||||
transcript=_transcript(),
|
||||
arms=(SessionReplayArmSpec(label="a", model="m"), SessionReplayArmSpec(label="b", model="n")),
|
||||
max_turns=10,
|
||||
session_id="s",
|
||||
recorded_prompt_tokens=41920,
|
||||
call_model=call_model,
|
||||
call_judge=call_judge,
|
||||
rng=random.Random(0),
|
||||
)
|
||||
|
||||
assert outcome.fidelity.recorded_prompt_tokens == 41920
|
||||
assert outcome.fidelity.replayed_first_turn_prompt_tokens == 10
|
||||
assert outcome.verdict.winner == "tie"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_each_arm_replays_under_its_own_session_id():
|
||||
"""Arms share one recorded session, but session affinity would pin them to one tier
|
||||
if they also shared a session id at replay time."""
|
||||
seen = []
|
||||
|
||||
async def call_model(request):
|
||||
seen.append(request["litellm_session_id"])
|
||||
return _text_response("answer")
|
||||
|
||||
async def call_judge(system_prompt, user_prompt):
|
||||
return '{"winner": "tie", "confidence": 0.5, "reasoning": "same"}'
|
||||
|
||||
await replay_session(
|
||||
transcript=_transcript(),
|
||||
arms=(SessionReplayArmSpec(label="a", model="m"), SessionReplayArmSpec(label="b", model="n")),
|
||||
max_turns=1,
|
||||
session_id="orig",
|
||||
recorded_prompt_tokens=1,
|
||||
call_model=call_model,
|
||||
call_judge=call_judge,
|
||||
rng=random.Random(0),
|
||||
)
|
||||
|
||||
assert len(set(seen)) == 2
|
||||
assert "orig" not in seen
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_progress_counts_turns_across_arms_not_per_arm():
|
||||
"""A stale-job reaper reads this counter as a heartbeat, so restarting it at 1 when the
|
||||
second arm begins makes a live job look like it went backwards."""
|
||||
|
||||
async def call_model(request):
|
||||
return _text_response("answer")
|
||||
|
||||
async def call_judge(system_prompt, user_prompt):
|
||||
return '{"winner": "tie", "confidence": 0.5, "reasoning": "same"}'
|
||||
|
||||
seen = []
|
||||
|
||||
async def on_progress(turns_completed):
|
||||
seen.append(turns_completed)
|
||||
|
||||
await replay_session(
|
||||
transcript=_transcript(),
|
||||
arms=(SessionReplayArmSpec(label="a", model="m"), SessionReplayArmSpec(label="b", model="n")),
|
||||
max_turns=10,
|
||||
session_id="s",
|
||||
recorded_prompt_tokens=1,
|
||||
call_model=call_model,
|
||||
call_judge=call_judge,
|
||||
rng=random.Random(0),
|
||||
on_progress=on_progress,
|
||||
)
|
||||
|
||||
assert seen == sorted(seen)
|
||||
assert seen[-1] == len(seen)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_arm_token_totals_survive_serialization():
|
||||
"""They are derived from the turns, and the job row stores the serialized result, so a
|
||||
plain property would drop them from every stored and returned job."""
|
||||
caller = _RecordingCaller([_text_response("one"), _text_response("two")])
|
||||
|
||||
run = await run_arm(
|
||||
_transcript(), SessionReplayArmSpec(label="a", model="m"), max_turns=10, session_id="s", call_model=caller
|
||||
)
|
||||
|
||||
assert run.model_dump()["prompt_tokens"] == 20
|
||||
assert run.model_dump()["output_tokens"] == 10
|
||||
134
tests/test_litellm/proxy/session_replay/test_transcript.py
Normal file
134
tests/test_litellm/proxy/session_replay/test_transcript.py
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
from litellm.proxy.session_replay.transcript import (
|
||||
RecordedBody,
|
||||
attach_turn,
|
||||
build_transcript,
|
||||
tool_use_ids,
|
||||
)
|
||||
|
||||
RECORDED_BODY = {
|
||||
"model": "tin-claude-router",
|
||||
"max_tokens": 32000,
|
||||
"thinking": {"type": "adaptive"},
|
||||
"stream": True,
|
||||
"litellm_session_id": "sess-1",
|
||||
"litellm_trace_id": "sess-1",
|
||||
"headers": {"x-claude-code-session-id": "sess-1"},
|
||||
"litellm_metadata": {"user_api_key_hash": "secret-hash"},
|
||||
"metadata": {"user_id": "device-blob"},
|
||||
"provider_specific_header": {"extra_headers": {"anthropic-beta": "claude-code"}},
|
||||
"system": [{"type": "text", "text": "You are Claude Code"}],
|
||||
"tools": [
|
||||
{"name": "Read", "description": "Read a file", "input_schema": {"type": "object", "properties": {}}},
|
||||
{
|
||||
"name": "Edit",
|
||||
"description": "Edit a file...[litellm_truncated]...tail",
|
||||
"input_schema": {"type": "object", "properties": {}},
|
||||
},
|
||||
],
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "<system-reminder>\ntoday is tuesday\n</system-reminder>"},
|
||||
{"type": "text", "text": "Plan my week"},
|
||||
],
|
||||
},
|
||||
{"role": "system", "content": "Available agent types for the Agent tool"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "text", "text": "on it"},
|
||||
{"type": "tool_use", "id": "toolu_ORIGINAL", "name": "Read"},
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_ORIGINAL", "content": "file body"}]},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _transcript():
|
||||
return build_transcript(RecordedBody.model_validate(RECORDED_BODY))
|
||||
|
||||
|
||||
def test_system_role_message_is_hoisted_out_of_messages():
|
||||
"""The Anthropic Messages API rejects role=system inside messages with a 400, and the
|
||||
recorded body carries one, so leaving it in place makes every replay unreplayable."""
|
||||
transcript = _transcript()
|
||||
|
||||
assert all(turn.role == "user" for turn in transcript.user_turns)
|
||||
assert "Available agent types for the Agent tool" in "".join(block.text or "" for block in transcript.system)
|
||||
|
||||
|
||||
def test_human_asks_strip_system_reminders_and_exclude_tool_results():
|
||||
transcript = _transcript()
|
||||
|
||||
assert transcript.human_asks == ("Plan my week",)
|
||||
|
||||
|
||||
def test_sampling_params_keep_provider_knobs_and_drop_proxy_injected_keys():
|
||||
transcript = _transcript()
|
||||
keys = {key for key, _ in transcript.sampling_params}
|
||||
|
||||
assert {"max_tokens", "thinking"} <= keys
|
||||
assert keys.isdisjoint(
|
||||
{
|
||||
"litellm_session_id",
|
||||
"litellm_trace_id",
|
||||
"headers",
|
||||
"litellm_metadata",
|
||||
"metadata",
|
||||
"provider_specific_header",
|
||||
}
|
||||
)
|
||||
assert keys.isdisjoint({"model", "messages", "system", "tools", "stream"})
|
||||
|
||||
|
||||
def test_truncation_markers_are_counted_so_fidelity_loss_is_visible():
|
||||
assert _transcript().truncated_strings == 1
|
||||
|
||||
|
||||
def test_unattachable_tool_result_turn_is_dropped_not_degraded():
|
||||
"""An arm that called no tools has no tool_use block for a recorded tool_result to
|
||||
reference, and injecting the recording's tool output as prose would hand the arm
|
||||
answers to questions it never asked."""
|
||||
transcript = _transcript()
|
||||
tool_turn = next(turn for turn in transcript.user_turns if "tool_result" in turn.block_types())
|
||||
|
||||
assert attach_turn(tool_turn, ()) is None
|
||||
|
||||
|
||||
def test_tool_results_rebind_to_the_arms_own_tool_use_ids():
|
||||
transcript = _transcript()
|
||||
tool_turn = next(turn for turn in transcript.user_turns if "tool_result" in turn.block_types())
|
||||
|
||||
attached = attach_turn(tool_turn, ("toolu_ARM",))
|
||||
|
||||
assert attached is not None
|
||||
assert [block.tool_use_id for block in attached.blocks()] == ["toolu_ARM"]
|
||||
|
||||
|
||||
def test_tool_calls_the_recording_never_answered_get_stubbed():
|
||||
"""Anthropic rejects an assistant tool_use with no matching tool_result, so an arm that
|
||||
called more tools than the recording answered would wedge on the next turn."""
|
||||
transcript = _transcript()
|
||||
tool_turn = next(turn for turn in transcript.user_turns if "tool_result" in turn.block_types())
|
||||
|
||||
attached = attach_turn(tool_turn, ("toolu_ARM", "toolu_EXTRA"))
|
||||
|
||||
assert attached is not None
|
||||
assert [block.tool_use_id for block in attached.blocks()] == ["toolu_ARM", "toolu_EXTRA"]
|
||||
|
||||
|
||||
def test_plain_user_turn_passes_through_untouched():
|
||||
transcript = _transcript()
|
||||
plain = next(turn for turn in transcript.user_turns if "tool_result" not in turn.block_types())
|
||||
|
||||
assert attach_turn(plain, ("toolu_ARM",)) is plain
|
||||
|
||||
|
||||
def test_tool_use_ids_reads_only_tool_use_blocks():
|
||||
transcript = _transcript()
|
||||
assistant_blocks = RecordedBody.model_validate(RECORDED_BODY).messages[2].blocks()
|
||||
|
||||
assert tool_use_ids(assistant_blocks) == ("toolu_ORIGINAL",)
|
||||
assert tool_use_ids(transcript.system) == ()
|
||||
339
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
339
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -1242,6 +1242,78 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/auto_router/session_replay": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
/**
|
||||
* List Session Replays
|
||||
* @description List session replay jobs, newest first.
|
||||
*/
|
||||
get: operations["list_session_replays_auto_router_session_replay_get"];
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/auto_router/session_replay/start": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/**
|
||||
* Start Session Replay
|
||||
* @description Replay a recorded session against several model arms and judge the end results blind.
|
||||
*
|
||||
* The recorded user and tool_result turns are the fixed environment; each arm generates
|
||||
* its own assistant turns, so the arms diverge from the recording and from each other.
|
||||
* Judging compares the final answer each arm reached against the human turns the
|
||||
* recording actually contained.
|
||||
*
|
||||
* Spend logs truncate stored prompts, so the replayed request is smaller than the one
|
||||
* that was recorded. Every job reports that gap under `fidelity`; read a verdict against
|
||||
* it rather than as a clean comparison.
|
||||
*
|
||||
* A replay issues one billable call per recorded turn per arm, so `max_turns` bounds it
|
||||
* and one session can have only one running job.
|
||||
*/
|
||||
post: operations["start_session_replay_auto_router_session_replay_start_post"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/auto_router/session_replay/{job_id}": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
/**
|
||||
* Get Session Replay
|
||||
* @description Read one session replay job, including its verdict once it has finished.
|
||||
*/
|
||||
get: operations["get_session_replay_auto_router_session_replay__job_id__get"];
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/auto_router/shadow_eval": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -36049,6 +36121,162 @@ export interface components {
|
|||
/** Timeout */
|
||||
timeout?: number | null;
|
||||
};
|
||||
/** SessionReplayArmResponse */
|
||||
SessionReplayArmResponse: {
|
||||
/**
|
||||
* Final Text
|
||||
* @default
|
||||
*/
|
||||
final_text: string;
|
||||
/** Label */
|
||||
label: string;
|
||||
/** Model */
|
||||
model: string;
|
||||
/** Output Tokens */
|
||||
readonly output_tokens: number;
|
||||
/** Prompt Tokens */
|
||||
readonly prompt_tokens: number;
|
||||
/**
|
||||
* Served Models
|
||||
* @default []
|
||||
*/
|
||||
served_models: string[];
|
||||
/**
|
||||
* Turns
|
||||
* @default []
|
||||
*/
|
||||
turns: components["schemas"]["SessionReplayTurnResponse"][];
|
||||
};
|
||||
/** SessionReplayArmSpec */
|
||||
SessionReplayArmSpec: {
|
||||
/** Label */
|
||||
label: string;
|
||||
/** Model */
|
||||
model: string;
|
||||
};
|
||||
/**
|
||||
* SessionReplayFidelityResponse
|
||||
* @description How much of the recorded prompt actually survived into the replay.
|
||||
*
|
||||
* Spend logs truncate every stored string at MAX_STRING_LENGTH_PROMPT_IN_DB, so a replay
|
||||
* runs against a smaller prompt than the recording did. Reported on every job so a result
|
||||
* is never read as a clean comparison when it is not one.
|
||||
*/
|
||||
SessionReplayFidelityResponse: {
|
||||
/**
|
||||
* Recorded Prompt Tokens
|
||||
* @default 0
|
||||
*/
|
||||
recorded_prompt_tokens: number;
|
||||
/**
|
||||
* Replayed First Turn Prompt Tokens
|
||||
* @default 0
|
||||
*/
|
||||
replayed_first_turn_prompt_tokens: number;
|
||||
/**
|
||||
* Truncated Strings
|
||||
* @default 0
|
||||
*/
|
||||
truncated_strings: number;
|
||||
};
|
||||
/** SessionReplayJobListResponse */
|
||||
SessionReplayJobListResponse: {
|
||||
/**
|
||||
* Jobs
|
||||
* @default []
|
||||
*/
|
||||
jobs: components["schemas"]["SessionReplayJobResponse"][];
|
||||
};
|
||||
/** SessionReplayJobResponse */
|
||||
SessionReplayJobResponse: {
|
||||
/**
|
||||
* Arms
|
||||
* @default []
|
||||
*/
|
||||
arms: components["schemas"]["SessionReplayArmResponse"][];
|
||||
/** Created At */
|
||||
created_at?: string | null;
|
||||
/** Created By */
|
||||
created_by?: string | null;
|
||||
/** Error */
|
||||
error?: string | null;
|
||||
fidelity?: components["schemas"]["SessionReplayFidelityResponse"] | null;
|
||||
/** Finished At */
|
||||
finished_at?: string | null;
|
||||
/**
|
||||
* Human Asks
|
||||
* @default []
|
||||
*/
|
||||
human_asks: string[];
|
||||
/** Job Id */
|
||||
job_id: string;
|
||||
/** Judge Model */
|
||||
judge_model: string;
|
||||
/** Max Turns */
|
||||
max_turns: number;
|
||||
/** Session Id */
|
||||
session_id: string;
|
||||
/** Source Request Id */
|
||||
source_request_id?: string | null;
|
||||
/**
|
||||
* Status
|
||||
* @enum {string}
|
||||
*/
|
||||
status: "running" | "completed" | "failed" | "stopped";
|
||||
/**
|
||||
* Turns Completed
|
||||
* @default 0
|
||||
*/
|
||||
turns_completed: number;
|
||||
verdict?: components["schemas"]["SessionReplayVerdictResponse"] | null;
|
||||
};
|
||||
/** SessionReplayTurnResponse */
|
||||
SessionReplayTurnResponse: {
|
||||
/** Attached */
|
||||
attached: boolean;
|
||||
/** Error */
|
||||
error?: string | null;
|
||||
/**
|
||||
* Output Tokens
|
||||
* @default 0
|
||||
*/
|
||||
output_tokens: number;
|
||||
/**
|
||||
* Prompt Tokens
|
||||
* @default 0
|
||||
*/
|
||||
prompt_tokens: number;
|
||||
/** Served Model */
|
||||
served_model?: string | null;
|
||||
/** Stop Reason */
|
||||
stop_reason?: string | null;
|
||||
/**
|
||||
* Text Chars
|
||||
* @default 0
|
||||
*/
|
||||
text_chars: number;
|
||||
/**
|
||||
* Tool Calls
|
||||
* @default 0
|
||||
*/
|
||||
tool_calls: number;
|
||||
/** Turn */
|
||||
turn: number;
|
||||
};
|
||||
/** SessionReplayVerdictResponse */
|
||||
SessionReplayVerdictResponse: {
|
||||
/** Confidence */
|
||||
confidence?: number | null;
|
||||
/** Error */
|
||||
error?: string | null;
|
||||
/**
|
||||
* Reasoning
|
||||
* @default
|
||||
*/
|
||||
reasoning: string;
|
||||
/** Winner */
|
||||
winner?: string | null;
|
||||
};
|
||||
/**
|
||||
* ShadowEvalJobResponse
|
||||
* @description A shadow-eval job over one or more targets, each with its own budget and stop state;
|
||||
|
|
@ -36486,6 +36714,20 @@ export interface components {
|
|||
/** Simple Medium */
|
||||
simple_medium: number;
|
||||
};
|
||||
/** StartSessionReplayRequest */
|
||||
StartSessionReplayRequest: {
|
||||
/** Arms */
|
||||
arms: components["schemas"]["SessionReplayArmSpec"][];
|
||||
/** Judge Model */
|
||||
judge_model: string;
|
||||
/**
|
||||
* Max Turns
|
||||
* @default 20
|
||||
*/
|
||||
max_turns: number;
|
||||
/** Session Id */
|
||||
session_id: string;
|
||||
};
|
||||
/**
|
||||
* StartShadowEvalRequest
|
||||
* @description Start duplicating one or more targets' traffic for blind comparison against an auto-router.
|
||||
|
|
@ -41782,6 +42024,103 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
list_session_replays_auto_router_session_replay_get: {
|
||||
parameters: {
|
||||
query?: {
|
||||
/** @description Only jobs for this session */
|
||||
session_id?: string | null;
|
||||
limit?: number;
|
||||
};
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["SessionReplayJobListResponse"];
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
422: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["HTTPValidationError"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
start_session_replay_auto_router_session_replay_start_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["StartSessionReplayRequest"];
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
201: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["SessionReplayJobResponse"];
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
422: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["HTTPValidationError"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
get_session_replay_auto_router_session_replay__job_id__get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path: {
|
||||
job_id: string;
|
||||
};
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["SessionReplayJobResponse"];
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
422: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["HTTPValidationError"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
list_shadow_eval_jobs_auto_router_shadow_eval_get: {
|
||||
parameters: {
|
||||
query?: {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue