mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(anthropic): keep encrypted reasoning in tool-call order and avoid regressions
- Read reasoning_items with getattr: Message and Delta drop the attribute when it is unset, so plain text responses and stream chunks raised AttributeError. - Emit each streamed reasoning item when its output item completes, ahead of later tool calls, and drop it from the terminal chunk so it is collected once. Anthropic thinking blocks must precede tool_use in the assistant turn. - Pass the same reasoning summary and extra_body effort to the bridge check as the completion router, and resolve chatgpt/ and github_copilot/ by their declared provider so the routing check does not start an OAuth device flow. - Add a two-turn round trip test through the mocked Responses API, streamed and not, plus summary-driven bridge detection; give the existing round trip test the usage the response translation requires. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
4b66255950
commit
71ba36ea38
6 changed files with 202 additions and 15 deletions
|
|
@ -1979,6 +1979,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
self._tool_call_index_map: dict[int, int] = {} # mutable-ok: per-stream accumulator state
|
||||
self._structured_output: Final = structured_output
|
||||
self._first_message_item_done: bool = False
|
||||
self._streamed_reasoning_ids: set[str] = set() # mutable-ok: per-stream accumulator state
|
||||
|
||||
def _handle_string_chunk(
|
||||
self, str_line: Union[str, "BaseModel"]
|
||||
|
|
@ -2321,7 +2322,35 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
)
|
||||
if self._structured_output and _is_message_item_done(chunk):
|
||||
self._first_message_item_done = True
|
||||
return self._with_served_service_tier(self._with_stream_scoped_id(translated))
|
||||
return self._with_served_service_tier(
|
||||
self._with_stream_scoped_id(self._with_reasoning_in_output_order(chunk, translated))
|
||||
)
|
||||
|
||||
def _with_reasoning_in_output_order(
|
||||
self, chunk: Mapping[str, object], translated: "ModelResponseStream"
|
||||
) -> "ModelResponseStream":
|
||||
"""Emit each reasoning item when it completes, ahead of the tool calls that follow it, and
|
||||
drop it from the terminal chunk so stream builders still collect it exactly once. Clients
|
||||
that need reasoning before its tool call (Anthropic thinking blocks) can then order them."""
|
||||
if not translated.choices:
|
||||
return translated
|
||||
delta: Final = translated.choices[0].delta
|
||||
if _responses_event_type(chunk) == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE.value:
|
||||
reasoning_item: Final = _reasoning_item_from_output_item(chunk.get("item"))
|
||||
if reasoning_item is not None:
|
||||
self._streamed_reasoning_ids.add(reasoning_item["id"])
|
||||
setattr(delta, "reasoning_items", _as_chat_reasoning_items((reasoning_item,))) # noqa: B010
|
||||
# Keep the turn's content as the terminal chunk would leave it (null on the OpenAI route).
|
||||
delta.content = None if self._replays_openai_history else ""
|
||||
return translated
|
||||
if not self._streamed_reasoning_ids:
|
||||
return translated
|
||||
terminal_items: Final = getattr(delta, "reasoning_items", None)
|
||||
if not terminal_items:
|
||||
return translated
|
||||
remaining: Final = [item for item in terminal_items if item.get("id") not in self._streamed_reasoning_ids]
|
||||
setattr(delta, "reasoning_items", remaining or None) # noqa: B010 # Delta drops unset optional fields
|
||||
return translated
|
||||
|
||||
def _is_text_delta_after_structured_answer(self, chunk: Mapping[str, object]) -> bool:
|
||||
"""Under a structured response_format the first message item is the whole answer, so
|
||||
|
|
|
|||
|
|
@ -735,30 +735,47 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
|
||||
@staticmethod
|
||||
def _uses_responses_bridge(completion_kwargs: _CompletionKwargs) -> bool:
|
||||
from litellm.responses.bridge_check import responses_api_bridge_check, str_keyed_mapping
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
|
||||
from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases
|
||||
from litellm.responses.bridge_check import (
|
||||
bridge_reasoning_effort,
|
||||
responses_api_bridge_check,
|
||||
str_keyed_mapping,
|
||||
)
|
||||
|
||||
model: Final = completion_kwargs.get("model")
|
||||
if not isinstance(model, str):
|
||||
return False
|
||||
provider: Final = completion_kwargs.get("custom_llm_provider")
|
||||
api_base: Final = completion_kwargs.get("api_base")
|
||||
try:
|
||||
local_model, resolved_provider, _, resolved_api_base = litellm.utils.get_llm_provider(
|
||||
model=model,
|
||||
custom_llm_provider=provider if isinstance(provider, str) else None,
|
||||
api_base=api_base if isinstance(api_base, str) else None,
|
||||
)
|
||||
except (ValueError, litellm.BadRequestError):
|
||||
return False
|
||||
provider: Final = (
|
||||
provider_arg if isinstance(provider_arg := completion_kwargs.get("custom_llm_provider"), str) else None
|
||||
)
|
||||
api_base: Final = base_arg if isinstance(base_arg := completion_kwargs.get("api_base"), str) else None
|
||||
# Resolving chatgpt/ or github_copilot/ runs their OAuth device flow; a routing
|
||||
# question only needs the declared provider.
|
||||
declared: Final = declared_authenticating_provider(model, provider)
|
||||
if declared is not None:
|
||||
local_model, resolved_provider, resolved_api_base = model.removeprefix(f"{declared}/"), declared, api_base
|
||||
else:
|
||||
try:
|
||||
local_model, resolved_provider, _, resolved_api_base = litellm.utils.get_llm_provider(
|
||||
model=model, custom_llm_provider=provider, api_base=api_base
|
||||
)
|
||||
except (ValueError, litellm.BadRequestError):
|
||||
return False
|
||||
tools: Final = completion_kwargs.get("tools")
|
||||
web_search: Final = completion_kwargs.get("web_search_options")
|
||||
# Same effort and summary inputs as the completion router, so nested extra_body
|
||||
# values that route the request to the Responses API keep its encrypted history.
|
||||
effort: Final = completion_kwargs.get("reasoning_effort")
|
||||
model_info, _ = responses_api_bridge_check(
|
||||
model=local_model,
|
||||
custom_llm_provider=resolved_provider,
|
||||
tools=cast("list[dict[str, object]]", tools) if isinstance(tools, list) else None,
|
||||
web_search_options=cast(OpenAIWebSearchOptions, web_search) if isinstance(web_search, dict) else None,
|
||||
reasoning_effort=effort if isinstance(effort, str) else str_keyed_mapping(effort),
|
||||
reasoning_effort=bridge_reasoning_effort(
|
||||
effort if isinstance(effort, str) else str_keyed_mapping(effort), completion_kwargs.get("extra_body")
|
||||
),
|
||||
reasoning_summary=peek_reasoning_summary_aliases(completion_kwargs),
|
||||
api_base=resolved_api_base,
|
||||
deployment_model_info=str_keyed_mapping(completion_kwargs.get("model_info")),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -469,7 +469,7 @@ class _CombinedChunkSplitter:
|
|||
choice: Final = chunk.choices[0]
|
||||
blocks: Final = tuple(
|
||||
block
|
||||
for item in (choice.delta.reasoning_items or ())
|
||||
for item in (getattr(choice.delta, "reasoning_items", None) or ())
|
||||
if (
|
||||
block := LiteLLMAnthropicToResponsesAPIAdapter._thinking_block_from_reasoning_item(
|
||||
(), item.get("encrypted_content")
|
||||
|
|
|
|||
|
|
@ -1392,7 +1392,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
new_content.extend(dict(_COMPACTION_BLOCK.validate_python(block)) for block in compaction_blocks)
|
||||
reasoning_blocks: Final = tuple(
|
||||
block
|
||||
for item in (choice.message.reasoning_items or ())
|
||||
for item in (getattr(choice.message, "reasoning_items", None) or ())
|
||||
if (
|
||||
block := LiteLLMAnthropicToResponsesAPIAdapter._thinking_block_from_reasoning_item(
|
||||
item.get("summary", ()), item.get("encrypted_content")
|
||||
|
|
|
|||
|
|
@ -73,6 +73,7 @@ def test_responses_reasoning_round_trips_through_anthropic_tool_turn(
|
|||
},
|
||||
}
|
||||
],
|
||||
usage=Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2),
|
||||
)
|
||||
result: Final = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(response)
|
||||
assert result["content"][0] == (
|
||||
|
|
|
|||
|
|
@ -0,0 +1,140 @@
|
|||
"""Encrypted reasoning round trip for `/v1/messages` served through the chat -> Responses bridge."""
|
||||
|
||||
import json
|
||||
from typing import Final
|
||||
|
||||
import anthropic
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature
|
||||
from litellm.llms.anthropic.messages.adapters.handler import LiteLLMMessagesToCompletionTransformationHandler
|
||||
|
||||
REASONING_ITEM: Final = {"id": "rs_1", "type": "reasoning", "summary": [], "encrypted_content": "opaque-test"}
|
||||
FUNCTION_CALL_ITEM: Final = {
|
||||
"id": "fc_1",
|
||||
"type": "function_call",
|
||||
"call_id": "call_1",
|
||||
"name": "weather",
|
||||
"arguments": "{}",
|
||||
"status": "completed",
|
||||
}
|
||||
TOOL_TURN: Final = {
|
||||
"id": "resp_1",
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"status": "completed",
|
||||
"model": "gpt-5",
|
||||
"output": [REASONING_ITEM, FUNCTION_CALL_ITEM],
|
||||
"parallel_tool_calls": True,
|
||||
"tool_choice": "auto",
|
||||
"tools": [],
|
||||
"usage": {"input_tokens": 3, "output_tokens": 4, "total_tokens": 7},
|
||||
}
|
||||
ANSWER_TURN: Final = {
|
||||
**TOOL_TURN,
|
||||
"id": "resp_2",
|
||||
"output": [
|
||||
{
|
||||
"id": "msg_1",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "text": "sunny", "annotations": []}],
|
||||
}
|
||||
],
|
||||
}
|
||||
REQUEST: Final = {
|
||||
"max_tokens": 256,
|
||||
"model": "openai/responses/gpt-5",
|
||||
"tools": [{"name": "weather", "description": "Weather", "input_schema": {"type": "object", "properties": {}}}],
|
||||
"thinking": {"type": "enabled", "budget_tokens": 1024},
|
||||
}
|
||||
USER_TURN: Final = {"role": "user", "content": "Weather?"}
|
||||
TOOL_RESULT_TURN: Final = {
|
||||
"role": "user",
|
||||
"content": [{"type": "tool_result", "tool_use_id": "call_1", "content": "sunny"}],
|
||||
}
|
||||
|
||||
|
||||
def _tool_turn_sse() -> bytes:
|
||||
events: Final = (
|
||||
{"type": "response.created", "response": {**TOOL_TURN, "status": "in_progress", "output": []}},
|
||||
{
|
||||
"type": "response.output_item.added",
|
||||
"output_index": 0,
|
||||
"item": {**REASONING_ITEM, "encrypted_content": None},
|
||||
},
|
||||
{"type": "response.output_item.done", "output_index": 0, "item": REASONING_ITEM},
|
||||
{
|
||||
"type": "response.output_item.added",
|
||||
"output_index": 1,
|
||||
"item": {**FUNCTION_CALL_ITEM, "arguments": "", "status": "in_progress"},
|
||||
},
|
||||
{"type": "response.function_call_arguments.delta", "output_index": 1, "item_id": "fc_1", "delta": "{}"},
|
||||
{"type": "response.output_item.done", "output_index": 1, "item": FUNCTION_CALL_ITEM},
|
||||
{"type": "response.completed", "response": TOOL_TURN},
|
||||
)
|
||||
return b"".join(
|
||||
f"event: {event['type']}\ndata: {json.dumps({**event, 'sequence_number': n})}\n\n".encode()
|
||||
for n, event in enumerate(events)
|
||||
)
|
||||
|
||||
|
||||
def _streamed_content(sse: bytes) -> list[dict[str, object]]:
|
||||
transport: Final = httpx.MockTransport(
|
||||
lambda request: httpx.Response(200, content=sse, headers={"content-type": "text/event-stream"})
|
||||
)
|
||||
client: Final = anthropic.Anthropic(api_key="transport-only", http_client=httpx.Client(transport=transport))
|
||||
with client.messages.stream(model="claude", max_tokens=16, messages=[USER_TURN]) as stream:
|
||||
return [block.model_dump(exclude_none=True) for block in stream.get_final_message().content]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
async def test_encrypted_reasoning_precedes_tool_use_and_is_replayed_once(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, stream: bool
|
||||
) -> None:
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
|
||||
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
upstream: Final = respx_mock.post("https://api.openai.com/v1/responses")
|
||||
if stream:
|
||||
upstream.respond(200, headers={"Content-Type": "text/event-stream"}, content=_tool_turn_sse())
|
||||
else:
|
||||
upstream.respond(200, json=TOOL_TURN)
|
||||
|
||||
first: Final = await LiteLLMMessagesToCompletionTransformationHandler.async_anthropic_messages_handler(
|
||||
messages=[USER_TURN], stream=stream, **REQUEST
|
||||
)
|
||||
content: Final = _streamed_content(b"".join([chunk async for chunk in first])) if stream else first["content"]
|
||||
assert [block["type"] for block in content if block["type"] != "text"] == ["redacted_thinking", "tool_use"]
|
||||
assert content[0]["data"] == encrypted_reasoning_signature("opaque-test")
|
||||
|
||||
upstream.respond(200, json=ANSWER_TURN)
|
||||
await LiteLLMMessagesToCompletionTransformationHandler.async_anthropic_messages_handler(
|
||||
messages=[USER_TURN, {"role": "assistant", "content": content}, TOOL_RESULT_TURN], **REQUEST
|
||||
)
|
||||
replayed: Final = json.loads(upstream.calls.last.request.content)["input"]
|
||||
assert [item["type"] for item in replayed] == ["message", "reasoning", "function_call", "function_call_output"]
|
||||
assert replayed[1]["encrypted_content"] == "opaque-test"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("summary_kwargs", "bridged"),
|
||||
[
|
||||
({}, False),
|
||||
({"reasoningSummary": "auto"}, True),
|
||||
({"extra_body": {"reasoningSummary": "auto"}}, True),
|
||||
],
|
||||
)
|
||||
def test_bridge_detection_follows_summary_routing(summary_kwargs: dict[str, object], bridged: bool) -> None:
|
||||
completion_kwargs: Final = {
|
||||
"model": "gpt-5",
|
||||
"custom_llm_provider": "openai",
|
||||
"reasoning_effort": "high",
|
||||
**summary_kwargs,
|
||||
}
|
||||
assert LiteLLMMessagesToCompletionTransformationHandler._uses_responses_bridge(completion_kwargs) is bridged
|
||||
Loading…
Add table
Reference in a new issue