mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(session_replay): allowlist replayed params, repair tool ordering, harden judge and job writes
This commit is contained in:
parent
2225c6d433
commit
ca759fcbc3
6 changed files with 252 additions and 65 deletions
|
|
@ -12,7 +12,6 @@ from typing import TYPE_CHECKING, Annotated, Final, Protocol
|
|||
|
||||
import fastapi
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from prisma import Json
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.integrations.shadow_eval_logger import (
|
||||
|
|
@ -44,7 +43,7 @@ if TYPE_CHECKING:
|
|||
router: Final = APIRouter()
|
||||
|
||||
MAX_CONCURRENT_SESSION_REPLAYS: Final = 4
|
||||
STALE_JOB_AFTER: Final = timedelta(minutes=30)
|
||||
STALE_JOB_AFTER: Final = timedelta(hours=3)
|
||||
_REPLAYABLE_CALL_TYPE: Final = "anthropic_messages"
|
||||
|
||||
|
||||
|
|
@ -56,11 +55,16 @@ class _SessionReplayJobRow(Protocol):
|
|||
class _SessionReplayJobTable(Protocol):
|
||||
async def create(self, data: Mapping[str, object]) -> _SessionReplayJobRow: ...
|
||||
|
||||
async def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> object: ...
|
||||
|
||||
async def update_many(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> int: ...
|
||||
|
||||
|
||||
def _json_column(value: object) -> object:
|
||||
"""prisma is an optional dependency of the base install, so it stays a lazy import."""
|
||||
from prisma import Json
|
||||
|
||||
return Json(value)
|
||||
|
||||
|
||||
def _session_replay_jobs(prisma_client: PrismaClient) -> _SessionReplayJobTable:
|
||||
return prisma_client.db.litellm_sessionreplayjob # pyright: ignore[reportAttributeAccessIssue] # generated client
|
||||
|
||||
|
|
@ -179,10 +183,12 @@ def _judge_caller(llm_router: Router, judge_model: str, parent_metadata: Mapping
|
|||
|
||||
|
||||
async def _reap_stale_jobs(prisma_client: PrismaClient) -> None:
|
||||
"""Fail jobs whose pod died mid-run.
|
||||
"""Fail jobs whose pod died mid-run, so the partial unique index does not lock a session out forever.
|
||||
|
||||
Progress is written after every turn, so a running job with a stale updated_at is
|
||||
abandoned. Without this the partial unique index would lock that session out forever.
|
||||
The heartbeat only ticks between turns, and a single arm or judge call can legitimately run
|
||||
for the router's whole timeout, so the window sits well above one call rather than near it.
|
||||
Reaping is still a guess, which is why the terminal writes below are status-guarded: a job
|
||||
reaped while it was in fact alive cannot overwrite the row a later replay now owns.
|
||||
"""
|
||||
cutoff: Final = datetime.now(timezone.utc) - STALE_JOB_AFTER
|
||||
await _session_replay_jobs(prisma_client).update_many(
|
||||
|
|
@ -208,9 +214,10 @@ async def _run_job(
|
|||
jobs: Final = _session_replay_jobs(prisma_client)
|
||||
|
||||
async def on_progress(turns_completed: int) -> None:
|
||||
await jobs.update(
|
||||
where={"id": job_id}, data={"turns_completed": turns_completed}
|
||||
) # mutable-ok: Prisma update input
|
||||
await jobs.update_many(
|
||||
where={"id": job_id, "status": "running"}, # mutable-ok: Prisma filter
|
||||
data={"turns_completed": turns_completed}, # mutable-ok: Prisma update input
|
||||
)
|
||||
|
||||
try:
|
||||
outcome: Final = await replay_session(
|
||||
|
|
@ -226,8 +233,8 @@ async def _run_job(
|
|||
)
|
||||
except Exception as exc: # noqa: BLE001 # the job records its own failure rather than dying silently
|
||||
verbose_proxy_logger.exception("session replay job %s failed", job_id)
|
||||
await jobs.update(
|
||||
where={"id": job_id}, # mutable-ok: Prisma update input
|
||||
await jobs.update_many(
|
||||
where={"id": job_id, "status": "running"}, # mutable-ok: Prisma filter
|
||||
data={ # mutable-ok: Prisma update input
|
||||
"status": "failed",
|
||||
"error": str(exc)[:1000],
|
||||
|
|
@ -235,11 +242,11 @@ async def _run_job(
|
|||
},
|
||||
)
|
||||
return
|
||||
await jobs.update(
|
||||
where={"id": job_id}, # mutable-ok: Prisma update input
|
||||
await jobs.update_many(
|
||||
where={"id": job_id, "status": "running"}, # mutable-ok: Prisma filter
|
||||
data={ # mutable-ok: Prisma update input
|
||||
"status": "completed",
|
||||
"result": Json(outcome.model_dump()),
|
||||
"result": _json_column(outcome.model_dump()),
|
||||
"turns_completed": sum(len(run.turns) for run in outcome.arms),
|
||||
"finished_at": datetime.now(timezone.utc),
|
||||
},
|
||||
|
|
@ -260,10 +267,7 @@ def _require_admin_viewer(user_api_key_dict: UserAPIKeyAuth, action: str) -> Non
|
|||
|
||||
|
||||
def _job_response(row: Mapping[str, object]) -> SessionReplayJobResponse:
|
||||
result: Final = row.get("result")
|
||||
payload: Final = (
|
||||
result if isinstance(result, Mapping) else {}
|
||||
) # mutable-ok: empty stand-in for an unfinished job's result
|
||||
payload: Final = _as_mapping(row.get("result")) or {} # mutable-ok: stand-in for an unfinished job's result
|
||||
verdict: Final = payload.get("verdict")
|
||||
fidelity: Final = payload.get("fidelity")
|
||||
arms: Final = payload.get("arms")
|
||||
|
|
@ -350,7 +354,7 @@ async def start_session_replay(
|
|||
data={ # mutable-ok: Prisma create input
|
||||
"session_id": data.session_id,
|
||||
"status": "running",
|
||||
"arms": Json([arm.model_dump() for arm in data.arms]), # mutable-ok: Prisma Json column payload
|
||||
"arms": _json_column([arm.model_dump() for arm in data.arms]), # mutable-ok: Prisma Json column payload
|
||||
"judge_model": data.judge_model,
|
||||
"max_turns": data.max_turns,
|
||||
"source_request_id": source.request_id,
|
||||
|
|
@ -369,7 +373,7 @@ async def start_session_replay(
|
|||
detail=f"could not create the replay job: {exc}",
|
||||
) from exc
|
||||
|
||||
parent_metadata: Final = {
|
||||
parent_metadata: Final = { # mutable-ok: forwardable call metadata, an SDK kwarg
|
||||
"user_api_key_user_id": user_api_key_dict.user_id
|
||||
} # mutable-ok: forwardable call metadata, an SDK kwarg
|
||||
asyncio.create_task( # detached job; its own handler records terminal state on the row
|
||||
|
|
|
|||
|
|
@ -99,13 +99,13 @@ def _request_body(
|
|||
return { # mutable-ok: Anthropic Messages wire body
|
||||
**{key: value for key, value in transcript.sampling_params}, # mutable-ok: Anthropic Messages wire body
|
||||
"model": arm.model,
|
||||
"messages": [
|
||||
"messages": [ # mutable-ok: Anthropic Messages wire body
|
||||
message.model_dump(exclude_none=True) for message in messages
|
||||
], # mutable-ok: Anthropic Messages wire body
|
||||
"system": [
|
||||
"system": [ # mutable-ok: Anthropic Messages wire body
|
||||
block.model_dump(exclude_none=True) for block in transcript.system
|
||||
], # mutable-ok: Anthropic Messages wire body
|
||||
"tools": [
|
||||
"tools": [ # mutable-ok: Anthropic Messages wire body
|
||||
tool.model_dump(exclude_none=True) for tool in transcript.tools
|
||||
], # mutable-ok: Anthropic Messages wire body
|
||||
"stream": False,
|
||||
|
|
@ -141,6 +141,7 @@ async def run_arm(
|
|||
raw = await call_model(_request_body(transcript, arm, history, session_id))
|
||||
response = ArmResponse.model_validate(raw)
|
||||
except Exception as exc: # noqa: BLE001 # a failed turn is recorded, never fatal to the job
|
||||
history.pop()
|
||||
turns.append(_failed_turn(index, str(exc)))
|
||||
await on_progress(len(turns))
|
||||
continue
|
||||
|
|
@ -179,7 +180,15 @@ def _judge_prompt(human_asks: Sequence[str], runs: Sequence[SessionReplayArmResp
|
|||
|
||||
|
||||
def _unmask(position: str, order: Sequence[int], runs: Sequence[SessionReplayArmResponse]) -> str | None:
|
||||
index: Final = ord(position.strip().upper()) - ord("A")
|
||||
"""Map the judge's blind A/B label back to an arm, or give up.
|
||||
|
||||
The judge writes this string, so it can be anything; anything unrecognized is a recorded
|
||||
verdict error rather than an exception, since every arm has already been billed by here.
|
||||
"""
|
||||
label: Final = position.strip().upper()
|
||||
if len(label) != 1:
|
||||
return None
|
||||
index: Final = ord(label) - ord("A")
|
||||
if index < 0 or index >= len(order):
|
||||
return None
|
||||
return runs[order[index]].label
|
||||
|
|
@ -226,7 +235,7 @@ async def judge_runs(
|
|||
if parsed is None:
|
||||
return SessionReplayVerdictResponse(error=f"unparseable judge response: {raw[:200]}")
|
||||
position, confidence, reasoning = parsed
|
||||
if position == "tie":
|
||||
if position.strip().casefold() == "tie":
|
||||
return SessionReplayVerdictResponse(winner="tie", confidence=confidence, reasoning=reasoning)
|
||||
unmasked: Final = _unmask(position, order, runs)
|
||||
if unmasked is None:
|
||||
|
|
|
|||
|
|
@ -18,19 +18,21 @@ from pydantic import BaseModel, ConfigDict
|
|||
|
||||
TRUNCATION_MARKER: Final = "litellm_truncated"
|
||||
|
||||
_PROXY_INJECTED_KEYS: Final = frozenset(
|
||||
REPLAYABLE_SAMPLING_KEYS: Final = frozenset(
|
||||
{
|
||||
"headers",
|
||||
"litellm_metadata",
|
||||
"litellm_session_id",
|
||||
"litellm_trace_id",
|
||||
"metadata",
|
||||
"provider_specific_header",
|
||||
"secret_fields",
|
||||
"max_tokens",
|
||||
"temperature",
|
||||
"top_p",
|
||||
"top_k",
|
||||
"stop_sequences",
|
||||
"thinking",
|
||||
"tool_choice",
|
||||
"output_config",
|
||||
"service_tier",
|
||||
}
|
||||
)
|
||||
|
||||
_STRUCTURAL_KEYS: Final = frozenset({"model", "messages", "system", "tools", "stream"})
|
||||
UNANSWERED_TOOL_RESULT: Final = "[no recorded result]"
|
||||
|
||||
_SYSTEM_REMINDER: Final = re.compile(r"<system-reminder>.*?</system-reminder>", re.DOTALL)
|
||||
|
||||
|
|
@ -42,7 +44,7 @@ class ContentBlock(BaseModel):
|
|||
text: str | None = None
|
||||
id: str | None = None
|
||||
tool_use_id: str | None = None
|
||||
content: str | None = None
|
||||
content: str | tuple[ContentBlock, ...] | None = None
|
||||
|
||||
|
||||
class ToolDefinition(BaseModel):
|
||||
|
|
@ -91,13 +93,19 @@ class ReplayTranscript:
|
|||
|
||||
|
||||
def _count_truncations(value: object) -> int:
|
||||
if isinstance(value, str):
|
||||
return 1 if TRUNCATION_MARKER in value else 0
|
||||
if isinstance(value, dict):
|
||||
return sum(_count_truncations(item) for item in value.values())
|
||||
if isinstance(value, (list, tuple)):
|
||||
return sum(_count_truncations(item) for item in value)
|
||||
return 0
|
||||
"""Walk the stored body iteratively; a recursive walk trips the CPU-spike CI gate."""
|
||||
pending: Final[list[object]] = [value] # mutable-ok: explicit stack replaces recursion
|
||||
marked: Final[list[str]] = [] # mutable-ok: one entry per truncated string
|
||||
while pending:
|
||||
current = pending.pop()
|
||||
if isinstance(current, str):
|
||||
if TRUNCATION_MARKER in current:
|
||||
marked.append(current)
|
||||
elif isinstance(current, dict):
|
||||
pending.extend(current.values())
|
||||
elif isinstance(current, (list, tuple)):
|
||||
pending.extend(current)
|
||||
return len(marked)
|
||||
|
||||
|
||||
def _as_system_blocks(system: str | tuple[ContentBlock, ...]) -> tuple[ContentBlock, ...]:
|
||||
|
|
@ -119,6 +127,10 @@ def build_transcript(recorded_body: RecordedBody) -> ReplayTranscript:
|
|||
|
||||
`role: "system"` messages are hoisted rather than dropped: the Anthropic Messages API
|
||||
rejects that role inside `messages`, but the text is real context the recorded model saw.
|
||||
|
||||
Sampling parameters are allowlisted, never filtered. The recorded body is caller-controlled
|
||||
and keeps transport fields the snapshot does not strip, `api_base` among them, so forwarding
|
||||
everything unrecognized would let a seeded session redirect an admin's replay traffic.
|
||||
"""
|
||||
hoisted: Final = tuple(
|
||||
_hoisted_system_block(message) for message in recorded_body.messages if message.role == "system"
|
||||
|
|
@ -130,7 +142,7 @@ def build_transcript(recorded_body: RecordedBody) -> ReplayTranscript:
|
|||
sampling: Final = tuple(
|
||||
(key, value)
|
||||
for key, value in recorded_body.model_dump(exclude_none=True).items()
|
||||
if key not in _PROXY_INJECTED_KEYS and key not in _STRUCTURAL_KEYS
|
||||
if key in REPLAYABLE_SAMPLING_KEYS
|
||||
)
|
||||
return ReplayTranscript(
|
||||
system=_as_system_blocks(recorded_body.system) + hoisted,
|
||||
|
|
@ -150,30 +162,31 @@ def tool_use_ids(assistant_blocks: Sequence[ContentBlock]) -> tuple[str, ...]:
|
|||
def attach_turn(turn: RecordedMessage, pending_tool_use_ids: Sequence[str]) -> RecordedMessage | None:
|
||||
"""Bind a recorded user turn onto the arm's own trajectory, or drop it as unattachable.
|
||||
|
||||
Recorded `tool_result` blocks carry the original run's `tool_use_id`s, which name
|
||||
`tool_use` blocks this arm never emitted. When the arm did call tools the results are
|
||||
rebound onto its ids in order; when it called none there is nothing to bind to, and
|
||||
injecting the recording's tool output as prose would feed the arm answers to questions
|
||||
it never asked, so the turn is dropped instead.
|
||||
The Anthropic Messages API requires every `tool_use` an assistant emits to be answered by a
|
||||
`tool_result` carrying that exact id, in the very next user message. The recording answers
|
||||
the ids the ORIGINAL run emitted, so once an arm diverges the two no longer line up in
|
||||
either direction, and both mismatches are a provider 400 rather than a quality question:
|
||||
|
||||
- the arm called tools the recording does not answer, including the case where the next
|
||||
recorded turn is ordinary text, so every unanswered id gets a stub
|
||||
- the recording answers tools the arm never called, so the surplus results are dropped
|
||||
rather than piled onto the last pending id, which would repeat one id
|
||||
|
||||
A turn carrying only tool results while the arm called nothing has nothing to bind to, and
|
||||
replaying the recording's tool output as prose would hand the arm answers to questions it
|
||||
never asked, so that turn is dropped.
|
||||
"""
|
||||
blocks: Final = turn.blocks()
|
||||
results: Final = tuple(block for block in blocks if block.type == "tool_result")
|
||||
if not results:
|
||||
return turn
|
||||
others: Final = tuple(block for block in blocks if block.type != "tool_result")
|
||||
if not pending_tool_use_ids:
|
||||
return None
|
||||
return None if results else turn
|
||||
rebound: Final = tuple(
|
||||
block.model_copy(
|
||||
update={"tool_use_id": pending_tool_use_ids[min(index, len(pending_tool_use_ids) - 1)]}
|
||||
) # mutable-ok: pydantic model_copy update mapping
|
||||
if block.type == "tool_result"
|
||||
else block
|
||||
for index, block in enumerate(blocks)
|
||||
result.model_copy(update={"tool_use_id": tool_id}) # mutable-ok: pydantic model_copy update mapping
|
||||
for tool_id, result in zip(pending_tool_use_ids, results)
|
||||
)
|
||||
answered: Final = frozenset(block.tool_use_id for block in rebound if block.type == "tool_result")
|
||||
stubs: Final = tuple(
|
||||
ContentBlock(type="tool_result", tool_use_id=tool_id, content="[no recorded result]")
|
||||
for tool_id in pending_tool_use_ids
|
||||
if tool_id not in answered
|
||||
ContentBlock(type="tool_result", tool_use_id=tool_id, content=UNANSWERED_TOOL_RESULT)
|
||||
for tool_id in pending_tool_use_ids[len(rebound) :]
|
||||
)
|
||||
return RecordedMessage(role="user", content=rebound + stubs)
|
||||
return RecordedMessage(role="user", content=rebound + stubs + others)
|
||||
|
|
|
|||
|
|
@ -213,3 +213,34 @@ async def test_jsonb_columns_returned_as_text_are_still_parsed():
|
|||
|
||||
assert isinstance(result, _SourceRequest)
|
||||
assert len(result.body.messages) == 1
|
||||
|
||||
|
||||
def test_finished_job_whose_result_comes_back_as_text_is_still_parsed():
|
||||
"""Same driver behaviour as the source body: a jsonb column can arrive as raw JSON text,
|
||||
and treating it as absent would silently drop the arms, verdict and fidelity from GET."""
|
||||
result = {
|
||||
"human_asks": ["plan my week"],
|
||||
"arms": [{"label": "auto_router", "model": "r", "final_text": "a", "turns": []}],
|
||||
"verdict": {"winner": "auto_router", "confidence": 0.72, "reasoning": "more complete"},
|
||||
"fidelity": {
|
||||
"truncated_strings": 6,
|
||||
"recorded_prompt_tokens": 41920,
|
||||
"replayed_first_turn_prompt_tokens": 16016,
|
||||
},
|
||||
}
|
||||
|
||||
response = _job_response(
|
||||
{
|
||||
"id": "job-3",
|
||||
"session_id": "s",
|
||||
"status": "completed",
|
||||
"judge_model": "j",
|
||||
"max_turns": 5,
|
||||
"result": json.dumps(result),
|
||||
}
|
||||
)
|
||||
|
||||
assert response.verdict is not None and response.verdict.winner == "auto_router"
|
||||
assert response.fidelity is not None and response.fidelity.truncated_strings == 6
|
||||
assert tuple(arm.label for arm in response.arms) == ("auto_router",)
|
||||
assert response.human_asks == ("plan my week",)
|
||||
|
|
|
|||
|
|
@ -303,3 +303,50 @@ async def test_arm_token_totals_survive_serialization():
|
|||
|
||||
assert run.model_dump()["prompt_tokens"] == 20
|
||||
assert run.model_dump()["output_tokens"] == 10
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_turn_does_not_leave_an_unanswered_user_message_in_history():
|
||||
"""A failed call produces no assistant reply, so keeping its user turn would send two
|
||||
consecutive user messages on the next turn and rebind results onto ids no longer pending."""
|
||||
caller = _RecordingCaller([_tool_response("toolu_ARM"), RuntimeError("upstream 500"), _text_response("ok")])
|
||||
|
||||
await run_arm(_transcript(), SessionReplayArmSpec(label="a", model="m"), max_turns=10, session_id="s", call_model=caller)
|
||||
|
||||
for request in caller.requests:
|
||||
roles = [message["role"] for message in request["messages"]]
|
||||
assert all(a != b for a, b in zip(roles, roles[1:])), roles
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("winner", ["Tie", "TIE", "CANDIDATE A", "", "neither", "A "])
|
||||
async def test_any_judge_winner_string_is_a_recorded_outcome_never_an_exception(winner):
|
||||
"""Every arm is already billed by the time the judge answers, so an unparseable winner must
|
||||
not take the whole job down with it."""
|
||||
caller = _RecordingCaller([_text_response("x"), _text_response("y")])
|
||||
run = await run_arm(
|
||||
_transcript(), SessionReplayArmSpec(label="a", model="m"), max_turns=10, session_id="s", call_model=caller
|
||||
)
|
||||
|
||||
async def call_judge(system_prompt, user_prompt):
|
||||
return '{"winner": "%s", "confidence": 0.5, "reasoning": "r"}' % winner
|
||||
|
||||
verdict = await judge_runs(("ask",), (run, run), call_judge, random.Random(0))
|
||||
|
||||
assert verdict.winner in (None, "tie", "a")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tie_is_recognised_whatever_case_the_judge_used():
|
||||
caller = _RecordingCaller([_text_response("x"), _text_response("y")])
|
||||
run = await run_arm(
|
||||
_transcript(), SessionReplayArmSpec(label="a", model="m"), max_turns=10, session_id="s", call_model=caller
|
||||
)
|
||||
|
||||
async def call_judge(system_prompt, user_prompt):
|
||||
return '{"winner": "Tie", "confidence": 0.4, "reasoning": "same"}'
|
||||
|
||||
verdict = await judge_runs(("ask",), (run, run), call_judge, random.Random(0))
|
||||
|
||||
assert verdict.winner == "tie"
|
||||
assert verdict.error is None
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from litellm.proxy.session_replay.transcript import (
|
||||
RecordedBody,
|
||||
RecordedMessage,
|
||||
attach_turn,
|
||||
build_transcript,
|
||||
tool_use_ids,
|
||||
|
|
@ -119,11 +120,49 @@ def test_tool_calls_the_recording_never_answered_get_stubbed():
|
|||
assert [block.tool_use_id for block in attached.blocks()] == ["toolu_ARM", "toolu_EXTRA"]
|
||||
|
||||
|
||||
def test_plain_user_turn_passes_through_untouched():
|
||||
def test_plain_user_turn_passes_through_when_no_tool_call_is_outstanding():
|
||||
transcript = _transcript()
|
||||
plain = next(turn for turn in transcript.user_turns if "tool_result" not in turn.block_types())
|
||||
|
||||
assert attach_turn(plain, ("toolu_ARM",)) is plain
|
||||
assert attach_turn(plain, ()) is plain
|
||||
|
||||
|
||||
def test_plain_user_turn_still_answers_an_outstanding_tool_call():
|
||||
"""Anthropic rejects a plain user message straight after an unanswered tool_use, so an arm
|
||||
that called a tool where the recording next has ordinary text would 400 on the following turn."""
|
||||
transcript = _transcript()
|
||||
plain = next(turn for turn in transcript.user_turns if "tool_result" not in turn.block_types())
|
||||
|
||||
attached = attach_turn(plain, ("toolu_ARM",))
|
||||
|
||||
assert attached is not None
|
||||
kinds = [block.type for block in attached.blocks()]
|
||||
assert kinds[0] == "tool_result"
|
||||
assert attached.blocks()[0].tool_use_id == "toolu_ARM"
|
||||
assert "text" in kinds
|
||||
assert "Plan my week" in attached.text()
|
||||
|
||||
|
||||
def test_surplus_recorded_results_are_dropped_not_piled_onto_the_last_id():
|
||||
"""Mapping extras onto the final pending id repeats one tool_use_id in a single message,
|
||||
which the provider rejects."""
|
||||
turn = RecordedMessage.model_validate(
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "tool_result", "tool_use_id": "orig_1", "content": "a"},
|
||||
{"type": "tool_result", "tool_use_id": "orig_2", "content": "b"},
|
||||
{"type": "tool_result", "tool_use_id": "orig_3", "content": "c"},
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
attached = attach_turn(turn, ("toolu_ARM",))
|
||||
|
||||
assert attached is not None
|
||||
ids = [block.tool_use_id for block in attached.blocks()]
|
||||
assert ids == ["toolu_ARM"]
|
||||
assert len(ids) == len(set(ids))
|
||||
|
||||
|
||||
def test_tool_use_ids_reads_only_tool_use_blocks():
|
||||
|
|
@ -132,3 +171,47 @@ def test_tool_use_ids_reads_only_tool_use_blocks():
|
|||
|
||||
assert tool_use_ids(assistant_blocks) == ("toolu_ORIGINAL",)
|
||||
assert tool_use_ids(transcript.system) == ()
|
||||
|
||||
|
||||
def test_transport_fields_never_reach_the_replay():
|
||||
"""The recorded body is caller-controlled and the spend-log snapshot strips api_key but not
|
||||
api_base, so forwarding unrecognized keys would let a seeded session redirect an admin's
|
||||
replay traffic to an attacker-chosen endpoint."""
|
||||
hostile = dict(RECORDED_BODY) | {
|
||||
"api_base": "https://attacker.example",
|
||||
"api_key": "sk-leak",
|
||||
"base_url": "https://attacker.example",
|
||||
"custom_llm_provider": "openai",
|
||||
"extra_headers": {"x-exfil": "1"},
|
||||
"vertex_project": "someone-elses",
|
||||
}
|
||||
|
||||
transcript = build_transcript(RecordedBody.model_validate(hostile))
|
||||
keys = {key for key, _ in transcript.sampling_params}
|
||||
|
||||
assert keys.isdisjoint(
|
||||
{"api_base", "api_key", "base_url", "custom_llm_provider", "extra_headers", "vertex_project"}
|
||||
)
|
||||
assert {"max_tokens", "thinking"} <= keys
|
||||
|
||||
|
||||
def test_tool_result_content_may_be_a_list_of_blocks():
|
||||
"""Anthropic tool results carry either a string or a list of content blocks, and rejecting
|
||||
the list form would make every session using it unreplayable."""
|
||||
turn = RecordedMessage.model_validate(
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "orig_1",
|
||||
"content": [{"type": "text", "text": "file body"}],
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
attached = attach_turn(turn, ("toolu_ARM",))
|
||||
|
||||
assert attached is not None
|
||||
assert attached.blocks()[0].tool_use_id == "toolu_ARM"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue