mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-12 08:42:18 +00:00
fix(responses): replay only the newest Claude thinking block on a past turn
A Responses reasoning item does not record where each thinking block sat between a turn's server tool steps and text, so replaying all of them side by side made Claude reject follow-ups after a web search turn with a 400. Keep the newest signed block per replayed assistant turn.
This commit is contained in:
parent
467847b2e9
commit
354ed4ba66
2 changed files with 92 additions and 11 deletions
|
|
@ -1633,9 +1633,10 @@ class LiteLLMCompletionResponsesConfig:
|
|||
immediately following assistant message.
|
||||
|
||||
Signed ``thinking_blocks`` decoded from ``encrypted_content`` travel the
|
||||
same way and are placed ahead of any thinking blocks the target message
|
||||
already carries, because Anthropic and Bedrock verify signatures against
|
||||
the original block order.
|
||||
same way, but only the newest one is replayed. A reasoning item does not
|
||||
record where each block sat between the turn's server tool steps and
|
||||
text, and Claude rejects a latest assistant message whose thinking
|
||||
blocks sit side by side when it wrote them apart.
|
||||
|
||||
If the reasoning item is not followed by an assistant message (e.g. a
|
||||
stateless chain replays ``reasoning`` + ``user``), the standalone
|
||||
|
|
@ -1683,7 +1684,7 @@ class LiteLLMCompletionResponsesConfig:
|
|||
],
|
||||
) -> None:
|
||||
pending_texts: Final = tuple(text for text, _ in pending_items if text)
|
||||
pending_blocks: Final = tuple(block for _, blocks in pending_items for block in blocks or ())
|
||||
pending_blocks: Final = tuple(chain.from_iterable(blocks or () for _, blocks in pending_items))
|
||||
if pending_texts:
|
||||
existing_text: Final = _reasoning_text(msg)
|
||||
combined: Final = "\n".join(pending_texts + ((existing_text,) if existing_text else ()))
|
||||
|
|
@ -1692,13 +1693,25 @@ class LiteLLMCompletionResponsesConfig:
|
|||
else:
|
||||
setattr(msg, "reasoning_content", combined) # noqa: B010 # attribute name is fixed, not dynamic
|
||||
if pending_blocks:
|
||||
replayed: Final = list(pending_blocks + (_thinking_blocks(msg) or ()))
|
||||
replayed: Final = list((pending_blocks + (_thinking_blocks(msg) or ()))[-1:])
|
||||
if isinstance(msg, dict):
|
||||
cast(dict[str, object], msg)["thinking_blocks"] = replayed # cast-ok: mutable reasoning carrier
|
||||
else:
|
||||
setattr(msg, "thinking_blocks", replayed) # noqa: B010 # attribute name is fixed, not dynamic
|
||||
|
||||
_standalone: Final = LiteLLMCompletionResponsesConfig._reasoning_only_assistant_message
|
||||
def _standalone(
|
||||
pending_items: Sequence[
|
||||
tuple[
|
||||
str | None,
|
||||
tuple[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, ...] | None,
|
||||
]
|
||||
],
|
||||
) -> ChatCompletionResponseMessage:
|
||||
pending_texts: Final = tuple(text for text, _ in pending_items if text)
|
||||
pending_blocks: Final = tuple(chain.from_iterable(blocks or () for _, blocks in pending_items))
|
||||
return LiteLLMCompletionResponsesConfig._reasoning_only_assistant_message(
|
||||
"\n".join(pending_texts) or None, pending_blocks[-1:]
|
||||
)
|
||||
|
||||
merged: list[ # mutable-ok: accumulator # rebind-ok: accumulator
|
||||
AllMessageValues
|
||||
|
|
@ -1729,12 +1742,13 @@ class LiteLLMCompletionResponsesConfig:
|
|||
elif pending:
|
||||
# Not followed by an assistant message — keep the reasoning
|
||||
# standalone instead of dropping it.
|
||||
merged.extend([_standalone(text, blocks) for text, blocks in pending])
|
||||
merged.append(_standalone(pending))
|
||||
pending = []
|
||||
|
||||
merged.append(msg)
|
||||
|
||||
merged.extend([_standalone(text, blocks) for text, blocks in pending])
|
||||
if pending:
|
||||
merged.append(_standalone(pending))
|
||||
|
||||
return merged
|
||||
|
||||
|
|
|
|||
|
|
@ -234,8 +234,8 @@ class TestEncryptedReasoningRoundTrip:
|
|||
]
|
||||
assert len(messages[0]["tool_calls"]) == 1
|
||||
|
||||
def test_replayed_blocks_precede_existing_blocks(self):
|
||||
"""Signature verification depends on the original block order."""
|
||||
def test_merge_keeps_only_the_newest_block(self):
|
||||
"""Blocks from separate turns would otherwise sit side by side on one message."""
|
||||
messages = LiteLLMCompletionResponsesConfig._merge_reasoning_only_assistant_messages(
|
||||
[
|
||||
{
|
||||
|
|
@ -251,7 +251,74 @@ class TestEncryptedReasoningRoundTrip:
|
|||
]
|
||||
)
|
||||
assert len(messages) == 1
|
||||
assert [block["thinking"] for block in messages[0]["thinking_blocks"]] == ["older", "newer"]
|
||||
assert [block["thinking"] for block in messages[0]["thinking_blocks"]] == ["newer"]
|
||||
|
||||
def test_web_search_turn_replays_only_its_last_thinking_block(self):
|
||||
"""Claude rejects a follow-up whose replay puts blocks it wrote apart side by side."""
|
||||
blocks = [
|
||||
{"type": "thinking", "thinking": "", "signature": "sig-before-search-1"},
|
||||
{"type": "thinking", "thinking": "", "signature": "sig-before-search-2"},
|
||||
{"type": "thinking", "thinking": "", "signature": "sig-before-answer"},
|
||||
]
|
||||
messages = _transform_input(
|
||||
[
|
||||
{"role": "user", "content": "Who directed the 2006 Best Picture winner?"},
|
||||
{"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": json.dumps(blocks)},
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_1",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "text": "Paul Haggis.", "annotations": []}],
|
||||
},
|
||||
{
|
||||
"type": "web_search_call",
|
||||
"id": "ws_1",
|
||||
"status": "completed",
|
||||
"action": {"type": "search", "query": "2006 Best Picture"},
|
||||
},
|
||||
{
|
||||
"type": "web_search_call",
|
||||
"id": "ws_2",
|
||||
"status": "completed",
|
||||
"action": {"type": "search", "query": "Crash 2004 film director"},
|
||||
},
|
||||
{"role": "user", "content": "Summarize that in one sentence."},
|
||||
]
|
||||
)
|
||||
assistants = [message for message in messages if message["role"] == "assistant"]
|
||||
assert len(assistants) == 1
|
||||
assert assistants[0]["thinking_blocks"] == [blocks[-1]]
|
||||
assert "Paul Haggis." in json.dumps(assistants[0]["content"])
|
||||
assert messages[-1]["role"] == "user"
|
||||
|
||||
def test_reasoning_before_a_user_turn_replays_one_block_in_one_message(self):
|
||||
"""A stateless replay must not leave blocks side by side either."""
|
||||
messages = _transform_input(
|
||||
[
|
||||
{
|
||||
"type": "reasoning",
|
||||
"id": "rs_1",
|
||||
"summary": [{"type": "summary_text", "text": "first"}],
|
||||
"encrypted_content": json.dumps([{"type": "thinking", "thinking": "a", "signature": "sig-a"}]),
|
||||
},
|
||||
{
|
||||
"type": "reasoning",
|
||||
"id": "rs_2",
|
||||
"summary": [{"type": "summary_text", "text": "second"}],
|
||||
"encrypted_content": json.dumps(
|
||||
[
|
||||
{"type": "thinking", "thinking": "b", "signature": "sig-b"},
|
||||
{"type": "thinking", "thinking": "c", "signature": "sig-c"},
|
||||
]
|
||||
),
|
||||
},
|
||||
{"role": "user", "content": "and now?"},
|
||||
]
|
||||
)
|
||||
assert [message["role"] for message in messages] == ["assistant", "user"]
|
||||
assert messages[0]["reasoning_content"] == "first\nsecond"
|
||||
assert messages[0]["thinking_blocks"] == [{"type": "thinking", "thinking": "c", "signature": "sig-c"}]
|
||||
|
||||
def test_encrypted_only_reasoning_preserved_before_user_turn(self):
|
||||
"""A signed item with no plaintext still survives a stateless replay."""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue