diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 580e55273f6..506bad0bcd4 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -1979,6 +1979,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): self._tool_call_index_map: dict[int, int] = {} # mutable-ok: per-stream accumulator state self._structured_output: Final = structured_output self._first_message_item_done: bool = False + self._streamed_reasoning_ids: set[str] = set() # mutable-ok: per-stream accumulator state def _handle_string_chunk( self, str_line: Union[str, "BaseModel"] @@ -2321,7 +2322,35 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): ) if self._structured_output and _is_message_item_done(chunk): self._first_message_item_done = True - return self._with_served_service_tier(self._with_stream_scoped_id(translated)) + return self._with_served_service_tier( + self._with_stream_scoped_id(self._with_reasoning_in_output_order(chunk, translated)) + ) + + def _with_reasoning_in_output_order( + self, chunk: Mapping[str, object], translated: "ModelResponseStream" + ) -> "ModelResponseStream": + """Emit each reasoning item when it completes, ahead of the tool calls that follow it, and + drop it from the terminal chunk so stream builders still collect it exactly once. Clients + that need reasoning before its tool call (Anthropic thinking blocks) can then order them.""" + if not translated.choices: + return translated + delta: Final = translated.choices[0].delta + if _responses_event_type(chunk) == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE.value: + reasoning_item: Final = _reasoning_item_from_output_item(chunk.get("item")) + if reasoning_item is not None: + self._streamed_reasoning_ids.add(reasoning_item["id"]) + setattr(delta, "reasoning_items", _as_chat_reasoning_items((reasoning_item,))) # noqa: B010 + # Keep the turn's content as the terminal chunk would leave it (null on the OpenAI route). + delta.content = None if self._replays_openai_history else "" + return translated + if not self._streamed_reasoning_ids: + return translated + terminal_items: Final = getattr(delta, "reasoning_items", None) + if not terminal_items: + return translated + remaining: Final = [item for item in terminal_items if item.get("id") not in self._streamed_reasoning_ids] + setattr(delta, "reasoning_items", remaining or None) # noqa: B010 # Delta drops unset optional fields + return translated def _is_text_delta_after_structured_answer(self, chunk: Mapping[str, object]) -> bool: """Under a structured response_format the first message item is the whole answer, so diff --git a/litellm/llms/anthropic/messages/adapters/handler.py b/litellm/llms/anthropic/messages/adapters/handler.py index 060e3985dc9..683b15d10ab 100644 --- a/litellm/llms/anthropic/messages/adapters/handler.py +++ b/litellm/llms/anthropic/messages/adapters/handler.py @@ -735,30 +735,47 @@ class LiteLLMMessagesToCompletionTransformationHandler: @staticmethod def _uses_responses_bridge(completion_kwargs: _CompletionKwargs) -> bool: - from litellm.responses.bridge_check import responses_api_bridge_check, str_keyed_mapping + from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider + from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases + from litellm.responses.bridge_check import ( + bridge_reasoning_effort, + responses_api_bridge_check, + str_keyed_mapping, + ) model: Final = completion_kwargs.get("model") if not isinstance(model, str): return False - provider: Final = completion_kwargs.get("custom_llm_provider") - api_base: Final = completion_kwargs.get("api_base") - try: - local_model, resolved_provider, _, resolved_api_base = litellm.utils.get_llm_provider( - model=model, - custom_llm_provider=provider if isinstance(provider, str) else None, - api_base=api_base if isinstance(api_base, str) else None, - ) - except (ValueError, litellm.BadRequestError): - return False + provider: Final = ( + provider_arg if isinstance(provider_arg := completion_kwargs.get("custom_llm_provider"), str) else None + ) + api_base: Final = base_arg if isinstance(base_arg := completion_kwargs.get("api_base"), str) else None + # Resolving chatgpt/ or github_copilot/ runs their OAuth device flow; a routing + # question only needs the declared provider. + declared: Final = declared_authenticating_provider(model, provider) + if declared is not None: + local_model, resolved_provider, resolved_api_base = model.removeprefix(f"{declared}/"), declared, api_base + else: + try: + local_model, resolved_provider, _, resolved_api_base = litellm.utils.get_llm_provider( + model=model, custom_llm_provider=provider, api_base=api_base + ) + except (ValueError, litellm.BadRequestError): + return False tools: Final = completion_kwargs.get("tools") web_search: Final = completion_kwargs.get("web_search_options") + # Same effort and summary inputs as the completion router, so nested extra_body + # values that route the request to the Responses API keep its encrypted history. effort: Final = completion_kwargs.get("reasoning_effort") model_info, _ = responses_api_bridge_check( model=local_model, custom_llm_provider=resolved_provider, tools=cast("list[dict[str, object]]", tools) if isinstance(tools, list) else None, web_search_options=cast(OpenAIWebSearchOptions, web_search) if isinstance(web_search, dict) else None, - reasoning_effort=effort if isinstance(effort, str) else str_keyed_mapping(effort), + reasoning_effort=bridge_reasoning_effort( + effort if isinstance(effort, str) else str_keyed_mapping(effort), completion_kwargs.get("extra_body") + ), + reasoning_summary=peek_reasoning_summary_aliases(completion_kwargs), api_base=resolved_api_base, deployment_model_info=str_keyed_mapping(completion_kwargs.get("model_info")), ) diff --git a/litellm/llms/anthropic/messages/adapters/streaming_iterator.py b/litellm/llms/anthropic/messages/adapters/streaming_iterator.py index e64e53fc0a5..4bc7340b0f1 100644 --- a/litellm/llms/anthropic/messages/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/adapters/streaming_iterator.py @@ -469,7 +469,7 @@ class _CombinedChunkSplitter: choice: Final = chunk.choices[0] blocks: Final = tuple( block - for item in (choice.delta.reasoning_items or ()) + for item in (getattr(choice.delta, "reasoning_items", None) or ()) if ( block := LiteLLMAnthropicToResponsesAPIAdapter._thinking_block_from_reasoning_item( (), item.get("encrypted_content") diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index a7f85e636f5..e3acc790ae7 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -1392,7 +1392,7 @@ class LiteLLMAnthropicMessagesAdapter: new_content.extend(dict(_COMPACTION_BLOCK.validate_python(block)) for block in compaction_blocks) reasoning_blocks: Final = tuple( block - for item in (choice.message.reasoning_items or ()) + for item in (getattr(choice.message, "reasoning_items", None) or ()) if ( block := LiteLLMAnthropicToResponsesAPIAdapter._thinking_block_from_reasoning_item( item.get("summary", ()), item.get("encrypted_content") diff --git a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py index 1830a5ce911..7f1d3bc52d1 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py @@ -73,6 +73,7 @@ def test_responses_reasoning_round_trips_through_anthropic_tool_turn( }, } ], + usage=Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2), ) result: Final = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(response) assert result["content"][0] == ( diff --git a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_encrypted_reasoning_roundtrip.py b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_encrypted_reasoning_roundtrip.py new file mode 100644 index 00000000000..15685e255fb --- /dev/null +++ b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_encrypted_reasoning_roundtrip.py @@ -0,0 +1,140 @@ +"""Encrypted reasoning round trip for `/v1/messages` served through the chat -> Responses bridge.""" + +import json +from typing import Final + +import anthropic +import httpx +import pytest +import respx + +import litellm +from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature +from litellm.llms.anthropic.messages.adapters.handler import LiteLLMMessagesToCompletionTransformationHandler + +REASONING_ITEM: Final = {"id": "rs_1", "type": "reasoning", "summary": [], "encrypted_content": "opaque-test"} +FUNCTION_CALL_ITEM: Final = { + "id": "fc_1", + "type": "function_call", + "call_id": "call_1", + "name": "weather", + "arguments": "{}", + "status": "completed", +} +TOOL_TURN: Final = { + "id": "resp_1", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-5", + "output": [REASONING_ITEM, FUNCTION_CALL_ITEM], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "usage": {"input_tokens": 3, "output_tokens": 4, "total_tokens": 7}, +} +ANSWER_TURN: Final = { + **TOOL_TURN, + "id": "resp_2", + "output": [ + { + "id": "msg_1", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "sunny", "annotations": []}], + } + ], +} +REQUEST: Final = { + "max_tokens": 256, + "model": "openai/responses/gpt-5", + "tools": [{"name": "weather", "description": "Weather", "input_schema": {"type": "object", "properties": {}}}], + "thinking": {"type": "enabled", "budget_tokens": 1024}, +} +USER_TURN: Final = {"role": "user", "content": "Weather?"} +TOOL_RESULT_TURN: Final = { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "call_1", "content": "sunny"}], +} + + +def _tool_turn_sse() -> bytes: + events: Final = ( + {"type": "response.created", "response": {**TOOL_TURN, "status": "in_progress", "output": []}}, + { + "type": "response.output_item.added", + "output_index": 0, + "item": {**REASONING_ITEM, "encrypted_content": None}, + }, + {"type": "response.output_item.done", "output_index": 0, "item": REASONING_ITEM}, + { + "type": "response.output_item.added", + "output_index": 1, + "item": {**FUNCTION_CALL_ITEM, "arguments": "", "status": "in_progress"}, + }, + {"type": "response.function_call_arguments.delta", "output_index": 1, "item_id": "fc_1", "delta": "{}"}, + {"type": "response.output_item.done", "output_index": 1, "item": FUNCTION_CALL_ITEM}, + {"type": "response.completed", "response": TOOL_TURN}, + ) + return b"".join( + f"event: {event['type']}\ndata: {json.dumps({**event, 'sequence_number': n})}\n\n".encode() + for n, event in enumerate(events) + ) + + +def _streamed_content(sse: bytes) -> list[dict[str, object]]: + transport: Final = httpx.MockTransport( + lambda request: httpx.Response(200, content=sse, headers={"content-type": "text/event-stream"}) + ) + client: Final = anthropic.Anthropic(api_key="transport-only", http_client=httpx.Client(transport=transport)) + with client.messages.stream(model="claude", max_tokens=16, messages=[USER_TURN]) as stream: + return [block.model_dump(exclude_none=True) for block in stream.get_final_message().content] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +async def test_encrypted_reasoning_precedes_tool_use_and_is_replayed_once( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, stream: bool +) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "sk-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post("https://api.openai.com/v1/responses") + if stream: + upstream.respond(200, headers={"Content-Type": "text/event-stream"}, content=_tool_turn_sse()) + else: + upstream.respond(200, json=TOOL_TURN) + + first: Final = await LiteLLMMessagesToCompletionTransformationHandler.async_anthropic_messages_handler( + messages=[USER_TURN], stream=stream, **REQUEST + ) + content: Final = _streamed_content(b"".join([chunk async for chunk in first])) if stream else first["content"] + assert [block["type"] for block in content if block["type"] != "text"] == ["redacted_thinking", "tool_use"] + assert content[0]["data"] == encrypted_reasoning_signature("opaque-test") + + upstream.respond(200, json=ANSWER_TURN) + await LiteLLMMessagesToCompletionTransformationHandler.async_anthropic_messages_handler( + messages=[USER_TURN, {"role": "assistant", "content": content}, TOOL_RESULT_TURN], **REQUEST + ) + replayed: Final = json.loads(upstream.calls.last.request.content)["input"] + assert [item["type"] for item in replayed] == ["message", "reasoning", "function_call", "function_call_output"] + assert replayed[1]["encrypted_content"] == "opaque-test" + + +@pytest.mark.parametrize( + ("summary_kwargs", "bridged"), + [ + ({}, False), + ({"reasoningSummary": "auto"}, True), + ({"extra_body": {"reasoningSummary": "auto"}}, True), + ], +) +def test_bridge_detection_follows_summary_routing(summary_kwargs: dict[str, object], bridged: bool) -> None: + completion_kwargs: Final = { + "model": "gpt-5", + "custom_llm_provider": "openai", + "reasoning_effort": "high", + **summary_kwargs, + } + assert LiteLLMMessagesToCompletionTransformationHandler._uses_responses_bridge(completion_kwargs) is bridged