fix(responses): replay only the newest Claude thinking block on a past turn

A Responses reasoning item does not record where each thinking block sat between a turn's server tool steps and text, so replaying all of them side by side made Claude reject follow-ups after a web search turn with a 400. Keep the newest signed block per replayed assistant turn.
This commit is contained in:
mateo-berri 2026-10-30 15:20:28 -07:00
parent 467847b2e9
commit 354ed4ba66
2 changed files with 92 additions and 11 deletions

View file

@ -1633,9 +1633,10 @@ class LiteLLMCompletionResponsesConfig:
immediately following assistant message.
Signed ``thinking_blocks`` decoded from ``encrypted_content`` travel the
same way and are placed ahead of any thinking blocks the target message
already carries, because Anthropic and Bedrock verify signatures against
the original block order.
same way, but only the newest one is replayed. A reasoning item does not
record where each block sat between the turn's server tool steps and
text, and Claude rejects a latest assistant message whose thinking
blocks sit side by side when it wrote them apart.
If the reasoning item is not followed by an assistant message (e.g. a
stateless chain replays ``reasoning`` + ``user``), the standalone
@ -1683,7 +1684,7 @@ class LiteLLMCompletionResponsesConfig:
],
) -> None:
pending_texts: Final = tuple(text for text, _ in pending_items if text)
pending_blocks: Final = tuple(block for _, blocks in pending_items for block in blocks or ())
pending_blocks: Final = tuple(chain.from_iterable(blocks or () for _, blocks in pending_items))
if pending_texts:
existing_text: Final = _reasoning_text(msg)
combined: Final = "\n".join(pending_texts + ((existing_text,) if existing_text else ()))
@ -1692,13 +1693,25 @@ class LiteLLMCompletionResponsesConfig:
else:
setattr(msg, "reasoning_content", combined) # noqa: B010 # attribute name is fixed, not dynamic
if pending_blocks:
replayed: Final = list(pending_blocks + (_thinking_blocks(msg) or ()))
replayed: Final = list((pending_blocks + (_thinking_blocks(msg) or ()))[-1:])
if isinstance(msg, dict):
cast(dict[str, object], msg)["thinking_blocks"] = replayed # cast-ok: mutable reasoning carrier
else:
setattr(msg, "thinking_blocks", replayed) # noqa: B010 # attribute name is fixed, not dynamic
_standalone: Final = LiteLLMCompletionResponsesConfig._reasoning_only_assistant_message
def _standalone(
pending_items: Sequence[
tuple[
str | None,
tuple[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, ...] | None,
]
],
) -> ChatCompletionResponseMessage:
pending_texts: Final = tuple(text for text, _ in pending_items if text)
pending_blocks: Final = tuple(chain.from_iterable(blocks or () for _, blocks in pending_items))
return LiteLLMCompletionResponsesConfig._reasoning_only_assistant_message(
"\n".join(pending_texts) or None, pending_blocks[-1:]
)
merged: list[ # mutable-ok: accumulator # rebind-ok: accumulator
AllMessageValues
@ -1729,12 +1742,13 @@ class LiteLLMCompletionResponsesConfig:
elif pending:
# Not followed by an assistant message — keep the reasoning
# standalone instead of dropping it.
merged.extend([_standalone(text, blocks) for text, blocks in pending])
merged.append(_standalone(pending))
pending = []
merged.append(msg)
merged.extend([_standalone(text, blocks) for text, blocks in pending])
if pending:
merged.append(_standalone(pending))
return merged

View file

@ -234,8 +234,8 @@ class TestEncryptedReasoningRoundTrip:
]
assert len(messages[0]["tool_calls"]) == 1
def test_replayed_blocks_precede_existing_blocks(self):
"""Signature verification depends on the original block order."""
def test_merge_keeps_only_the_newest_block(self):
"""Blocks from separate turns would otherwise sit side by side on one message."""
messages = LiteLLMCompletionResponsesConfig._merge_reasoning_only_assistant_messages(
[
{
@ -251,7 +251,74 @@ class TestEncryptedReasoningRoundTrip:
]
)
assert len(messages) == 1
assert [block["thinking"] for block in messages[0]["thinking_blocks"]] == ["older", "newer"]
assert [block["thinking"] for block in messages[0]["thinking_blocks"]] == ["newer"]
def test_web_search_turn_replays_only_its_last_thinking_block(self):
"""Claude rejects a follow-up whose replay puts blocks it wrote apart side by side."""
blocks = [
{"type": "thinking", "thinking": "", "signature": "sig-before-search-1"},
{"type": "thinking", "thinking": "", "signature": "sig-before-search-2"},
{"type": "thinking", "thinking": "", "signature": "sig-before-answer"},
]
messages = _transform_input(
[
{"role": "user", "content": "Who directed the 2006 Best Picture winner?"},
{"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": json.dumps(blocks)},
{
"type": "message",
"id": "msg_1",
"role": "assistant",
"status": "completed",
"content": [{"type": "output_text", "text": "Paul Haggis.", "annotations": []}],
},
{
"type": "web_search_call",
"id": "ws_1",
"status": "completed",
"action": {"type": "search", "query": "2006 Best Picture"},
},
{
"type": "web_search_call",
"id": "ws_2",
"status": "completed",
"action": {"type": "search", "query": "Crash 2004 film director"},
},
{"role": "user", "content": "Summarize that in one sentence."},
]
)
assistants = [message for message in messages if message["role"] == "assistant"]
assert len(assistants) == 1
assert assistants[0]["thinking_blocks"] == [blocks[-1]]
assert "Paul Haggis." in json.dumps(assistants[0]["content"])
assert messages[-1]["role"] == "user"
def test_reasoning_before_a_user_turn_replays_one_block_in_one_message(self):
"""A stateless replay must not leave blocks side by side either."""
messages = _transform_input(
[
{
"type": "reasoning",
"id": "rs_1",
"summary": [{"type": "summary_text", "text": "first"}],
"encrypted_content": json.dumps([{"type": "thinking", "thinking": "a", "signature": "sig-a"}]),
},
{
"type": "reasoning",
"id": "rs_2",
"summary": [{"type": "summary_text", "text": "second"}],
"encrypted_content": json.dumps(
[
{"type": "thinking", "thinking": "b", "signature": "sig-b"},
{"type": "thinking", "thinking": "c", "signature": "sig-c"},
]
),
},
{"role": "user", "content": "and now?"},
]
)
assert [message["role"] for message in messages] == ["assistant", "user"]
assert messages[0]["reasoning_content"] == "first\nsecond"
assert messages[0]["thinking_blocks"] == [{"type": "thinking", "thinking": "c", "signature": "sig-c"}]
def test_encrypted_only_reasoning_preserved_before_user_turn(self):
"""A signed item with no plaintext still survives a stateless replay."""