mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(anthropic): keep every non-streamed reasoning item for Messages clients and read a deployment's reasoning_summary in the keep check
Backport of #49076 to lts/1.108.x.
Ported from cadfe1d179 (cursor/chatgpt-encrypted-reasoning-messages-1723) onto the line's pass_through/adapters layout.
This commit is contained in:
parent
d6625245b4
commit
ce35f27bfa
7 changed files with 144 additions and 24 deletions
|
|
@ -13,9 +13,7 @@ from typing import (
|
|||
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
from litellm.litellm_core_utils.core_helpers import attribute_or_none
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from litellm.types.utils import CallTypes
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm import CustomStreamWrapper, LiteLLMLoggingObj, ModelResponse
|
||||
|
|
@ -38,10 +36,6 @@ def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str:
|
|||
return f"{custom_llm_provider}/{model}"
|
||||
|
||||
|
||||
def _serves_anthropic_messages(logging_obj: "LiteLLMLoggingObj") -> bool:
|
||||
return attribute_or_none(logging_obj, "call_type") == CallTypes.anthropic_messages.value
|
||||
|
||||
|
||||
class ResponsesToCompletionBridgeHandler:
|
||||
def __init__(self):
|
||||
from .transformation import LiteLLMResponsesTransformationHandler
|
||||
|
|
@ -188,7 +182,9 @@ class ResponsesToCompletionBridgeHandler:
|
|||
model_response: Final = validated_kwargs["model_response"]
|
||||
logging_obj: Final = validated_kwargs["logging_obj"]
|
||||
custom_llm_provider: Final = validated_kwargs["custom_llm_provider"]
|
||||
serves_anthropic_messages: Final = _serves_anthropic_messages(logging_obj)
|
||||
from .transformation import serves_anthropic_messages
|
||||
|
||||
streams_reasoning_signatures: Final = serves_anthropic_messages(logging_obj)
|
||||
if kwargs.get("stream") is True and "stream" not in optional_params:
|
||||
optional_params = {**optional_params, "stream": True}
|
||||
|
||||
|
|
@ -267,7 +263,7 @@ class ResponsesToCompletionBridgeHandler:
|
|||
structured_output=is_structured_output_request(
|
||||
cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict
|
||||
),
|
||||
streams_reasoning_signatures=serves_anthropic_messages,
|
||||
streams_reasoning_signatures=streams_reasoning_signatures,
|
||||
)
|
||||
streamwrapper: Final = CustomStreamWrapper(
|
||||
completion_stream=completion_stream,
|
||||
|
|
@ -290,7 +286,9 @@ class ResponsesToCompletionBridgeHandler:
|
|||
model_response: Final = validated_kwargs["model_response"]
|
||||
logging_obj: Final = validated_kwargs["logging_obj"]
|
||||
custom_llm_provider: Final = validated_kwargs["custom_llm_provider"]
|
||||
serves_anthropic_messages: Final = _serves_anthropic_messages(logging_obj)
|
||||
from .transformation import serves_anthropic_messages
|
||||
|
||||
streams_reasoning_signatures: Final = serves_anthropic_messages(logging_obj)
|
||||
if kwargs.get("stream") is True and "stream" not in optional_params:
|
||||
optional_params = {**optional_params, "stream": True}
|
||||
|
||||
|
|
@ -372,7 +370,7 @@ class ResponsesToCompletionBridgeHandler:
|
|||
structured_output=is_structured_output_request(
|
||||
cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict
|
||||
),
|
||||
streams_reasoning_signatures=serves_anthropic_messages,
|
||||
streams_reasoning_signatures=streams_reasoning_signatures,
|
||||
)
|
||||
streamwrapper: Final = CustomStreamWrapper(
|
||||
completion_stream=completion_stream,
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ import litellm
|
|||
from litellm import ModelResponse
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none
|
||||
from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
responses_reasoning_items_from_thinking_blocks,
|
||||
|
|
@ -58,7 +58,7 @@ from litellm.types.llms.openai import (
|
|||
ResponsesAPIResponse,
|
||||
ResponsesAPIStreamEvents,
|
||||
)
|
||||
from litellm.types.utils import Delta, GenericStreamingChunk, ModelResponseStream, StreamingChoices
|
||||
from litellm.types.utils import CallTypes, Delta, GenericStreamingChunk, ModelResponseStream, StreamingChoices
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from openai.types.responses import ResponseInputImageParam, ResponseOutputItem
|
||||
|
|
@ -227,6 +227,10 @@ class _BuiltReasoningItem(TypedDict):
|
|||
summary: Sequence[_ReasoningSummaryText]
|
||||
|
||||
|
||||
def serves_anthropic_messages(logging_obj: object) -> bool:
|
||||
return attribute_or_none(logging_obj, "call_type") == CallTypes.anthropic_messages.value
|
||||
|
||||
|
||||
def _get_reasoning_items(
|
||||
msg: "AllMessageValues",
|
||||
) -> list[ChatCompletionReasoningItem]:
|
||||
|
|
@ -549,7 +553,7 @@ def _fold_choices_into_one(
|
|||
choices: Sequence["Choices"],
|
||||
tool_calls: Sequence[Mapping[str, object]],
|
||||
trailing_reasoning_content: str | None,
|
||||
trailing_reasoning_item: _BuiltReasoningItem | None,
|
||||
trailing_reasoning_items: Sequence[_BuiltReasoningItem],
|
||||
tool_search_items: Sequence[Mapping[str, object]],
|
||||
structured_output: bool = False,
|
||||
) -> "Choices":
|
||||
|
|
@ -568,7 +572,7 @@ def _fold_choices_into_one(
|
|||
reasoning_contents: Final = (*(_message_reasoning_content(m) for m in messages), trailing_reasoning_content)
|
||||
reasoning_items: Final = (
|
||||
*chain.from_iterable(_message_reasoning_items(m) for m in messages),
|
||||
*(() if trailing_reasoning_item is None else (trailing_reasoning_item,)),
|
||||
*trailing_reasoning_items,
|
||||
)
|
||||
return Choices(
|
||||
message=Message(
|
||||
|
|
@ -1222,6 +1226,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
output_items: Sequence[object],
|
||||
handle_raw_dict_callback: Callable[..., tuple["Choices | None", int]] | None = None,
|
||||
structured_output: bool = False,
|
||||
keeps_every_reasoning_item: bool = False,
|
||||
) -> list["Choices"]:
|
||||
"""
|
||||
Convert Responses API output items to chat completion choices.
|
||||
|
|
@ -1231,6 +1236,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
handle_raw_dict_callback: Optional callback for handling raw dict items
|
||||
structured_output: The chat request carried a json response_format, so only the
|
||||
first message item becomes content
|
||||
keeps_every_reasoning_item: Keep each reasoning item before an answer, not only the last
|
||||
|
||||
Returns:
|
||||
List of Choices objects
|
||||
|
|
@ -1253,7 +1259,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
choices: Final[list[Choices]] = []
|
||||
index = 0
|
||||
reasoning_content: str | None = None
|
||||
pending_reasoning_item: _BuiltReasoningItem | None = None
|
||||
pending_reasoning_items: tuple[_BuiltReasoningItem, ...] = ()
|
||||
|
||||
# Collect all tool calls to put them in a single choice
|
||||
# (Chat Completions API expects all tool calls in one message)
|
||||
|
|
@ -1262,12 +1268,19 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
|
||||
for item in output_items:
|
||||
if isinstance(item, ResponseReasoningItem):
|
||||
pending_reasoning_item = _build_reasoning_item(
|
||||
item_id=item.id,
|
||||
encrypted_content=getattr(item, "encrypted_content", None),
|
||||
summary_raw=item.summary,
|
||||
pending_reasoning_items = (
|
||||
*(pending_reasoning_items if keeps_every_reasoning_item else ()),
|
||||
_build_reasoning_item(
|
||||
item_id=item.id,
|
||||
encrypted_content=getattr(item, "encrypted_content", None),
|
||||
summary_raw=item.summary,
|
||||
),
|
||||
)
|
||||
reasoning_content = " ".join(
|
||||
s["text"]
|
||||
for s in chain.from_iterable(built["summary"] for built in pending_reasoning_items)
|
||||
if s.get("text")
|
||||
)
|
||||
reasoning_content = " ".join(s["text"] for s in pending_reasoning_item["summary"] if s.get("text"))
|
||||
|
||||
elif isinstance(item, ResponseOutputMessage):
|
||||
for content in item.content:
|
||||
|
|
@ -1284,7 +1297,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
annotations=annotations,
|
||||
reasoning_items=cast(
|
||||
list[ChatCompletionReasoningItem] | None,
|
||||
([pending_reasoning_item] if pending_reasoning_item is not None else None),
|
||||
(list(pending_reasoning_items) or None),
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -1297,7 +1310,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
)
|
||||
|
||||
reasoning_content = None # flush
|
||||
pending_reasoning_item = None # flush
|
||||
pending_reasoning_items = () # flush
|
||||
index += 1
|
||||
|
||||
elif isinstance(item, ResponseFunctionToolCall):
|
||||
|
|
@ -1362,7 +1375,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
choices=choices,
|
||||
tool_calls=accumulated_tool_calls,
|
||||
trailing_reasoning_content=reasoning_content,
|
||||
trailing_reasoning_item=pending_reasoning_item,
|
||||
trailing_reasoning_items=pending_reasoning_items,
|
||||
tool_search_items=tool_search_items,
|
||||
structured_output=structured_output,
|
||||
)
|
||||
|
|
@ -1500,6 +1513,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
structured_output=is_structured_output_request(
|
||||
cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict
|
||||
),
|
||||
keeps_every_reasoning_item=serves_anthropic_messages(logging_obj),
|
||||
)
|
||||
|
||||
response_is_incomplete: Final = raw_response.status == "incomplete" or (
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from litellm._logging import verbose_logger
|
|||
from litellm.litellm_core_utils.asyncify import run_async_function
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
|
||||
from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
is_encrypted_reasoning_block,
|
||||
thinking_blocks_with_encrypted_reasoning,
|
||||
|
|
@ -760,6 +761,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
|
|||
request_data,
|
||||
custom_llm_provider=target_provider,
|
||||
api_base=api_base if isinstance(api_base, str) else None,
|
||||
reasoning_summary=peek_reasoning_summary_aliases(provider_kwargs),
|
||||
)
|
||||
|
||||
if openai_request is None:
|
||||
|
|
|
|||
|
|
@ -228,6 +228,7 @@ class ChatRouteInputs:
|
|||
reasoning_effort: str | Mapping[str, object] | None = None
|
||||
api_base: str | None = None
|
||||
thinking: Mapping[str, object] | None = None
|
||||
reasoning_summary: object | None = None
|
||||
|
||||
|
||||
_NO_ROUTE_INPUTS: Final = ChatRouteInputs()
|
||||
|
|
@ -271,6 +272,7 @@ def _keeps_encrypted_reasoning_for_responses(
|
|||
custom_llm_provider,
|
||||
tools=route.tools,
|
||||
reasoning_effort=route.reasoning_effort,
|
||||
reasoning_summary=route.reasoning_summary,
|
||||
api_base=route.api_base,
|
||||
)
|
||||
|
||||
|
|
@ -313,7 +315,12 @@ class AnthropicAdapter:
|
|||
return result
|
||||
|
||||
def translate_completion_input_params_with_tool_mapping(
|
||||
self, kwargs, *, custom_llm_provider: str | None = None, api_base: str | None = None
|
||||
self,
|
||||
kwargs,
|
||||
*,
|
||||
custom_llm_provider: str | None = None,
|
||||
api_base: str | None = None,
|
||||
reasoning_summary: object | None = None,
|
||||
) -> tuple[ChatCompletionRequest | None, dict[str, str]]:
|
||||
"""
|
||||
Translate Anthropic request params to OpenAI format, returning tool name mapping.
|
||||
|
|
@ -348,6 +355,7 @@ class AnthropicAdapter:
|
|||
anthropic_message_request=request_body,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=api_base,
|
||||
reasoning_summary=reasoning_summary,
|
||||
)
|
||||
|
||||
return translated_body, tool_name_mapping
|
||||
|
|
@ -1295,6 +1303,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
*,
|
||||
custom_llm_provider: str | None = None,
|
||||
api_base: str | None = None,
|
||||
reasoning_summary: object | None = None,
|
||||
preserve_midturn_system: bool = False,
|
||||
) -> tuple[ChatCompletionRequest, dict[str, str]]:
|
||||
"""
|
||||
|
|
@ -1349,6 +1358,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
reasoning_effort=new_kwargs.get("reasoning_effort"),
|
||||
api_base=api_base,
|
||||
thinking=anthropic_message_request.get("thinking"),
|
||||
reasoning_summary=reasoning_summary,
|
||||
),
|
||||
)
|
||||
## ADD SYSTEM MESSAGE TO MESSAGES
|
||||
|
|
|
|||
|
|
@ -197,6 +197,7 @@ def chat_completion_ends_in_responses_api(
|
|||
*,
|
||||
tools: Sequence[Mapping[str, object]] | None = None,
|
||||
reasoning_effort: str | Mapping[str, object] | None = None,
|
||||
reasoning_summary: object | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> bool:
|
||||
"""Whether chat completion for this model is served by a Responses API.
|
||||
|
|
@ -219,6 +220,7 @@ def chat_completion_ends_in_responses_api(
|
|||
custom_llm_provider=provider,
|
||||
tools=tools,
|
||||
reasoning_effort=reasoning_effort,
|
||||
reasoning_summary=reasoning_summary,
|
||||
api_base=api_base,
|
||||
)
|
||||
return model_info.get("mode") == "responses"
|
||||
|
|
|
|||
|
|
@ -3570,6 +3570,72 @@ def test_transform_response_hands_encrypted_reasoning_to_messages_clients_only()
|
|||
assert content[0]["data"] == encrypted_reasoning_signature(encrypted)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("call_type", "kept_ids"),
|
||||
[
|
||||
pytest.param("anthropic_messages", ["rs_first", "rs_second"], id="messages_client_keeps_every_item"),
|
||||
pytest.param("acompletion", ["rs_second"], id="chat_client_keeps_the_last_item"),
|
||||
],
|
||||
)
|
||||
def test_non_streamed_tool_turn_hands_messages_clients_every_reasoning_item(
|
||||
call_type: str, kept_ids: list[str]
|
||||
) -> None:
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from litellm.types.utils import ModelResponse, Usage
|
||||
|
||||
def reasoning(item_id: str) -> ResponseReasoningItem:
|
||||
return ResponseReasoningItem(
|
||||
id=item_id, summary=[], type="reasoning", content=None, encrypted_content=f"gAAAA_{item_id}", status=None
|
||||
)
|
||||
|
||||
def call(index: int, city: str) -> ResponseFunctionToolCall:
|
||||
return ResponseFunctionToolCall(
|
||||
id=f"fc_{index}",
|
||||
type="function_call",
|
||||
status="completed",
|
||||
arguments=json.dumps({"city": city}),
|
||||
call_id=f"call_{index}",
|
||||
name="get_weather",
|
||||
)
|
||||
|
||||
raw: Final = ResponsesAPIResponse.model_construct(
|
||||
id="resp_two_reasoning",
|
||||
created_at=1,
|
||||
error=None,
|
||||
incomplete_details=None,
|
||||
model="gpt-5.4-nano",
|
||||
output=[reasoning("rs_first"), call(1, "Oslo"), reasoning("rs_second"), call(2, "Lyon")],
|
||||
status="completed",
|
||||
usage=None,
|
||||
)
|
||||
result: Final = LiteLLMResponsesTransformationHandler().transform_response(
|
||||
model="gpt-5.4-nano",
|
||||
raw_response=raw,
|
||||
model_response=ModelResponse(
|
||||
id="chatcmpl-two-reasoning",
|
||||
created=1,
|
||||
model=None,
|
||||
object="chat.completion",
|
||||
system_fingerprint=None,
|
||||
choices=[],
|
||||
usage=Usage(completion_tokens=0, prompt_tokens=0, total_tokens=0),
|
||||
),
|
||||
logging_obj=SimpleNamespace(call_type=call_type),
|
||||
request_data={"model": "gpt-5.4-nano"},
|
||||
messages=[{"role": "user", "content": "Oslo or Lyon?"}],
|
||||
optional_params={},
|
||||
litellm_params={"custom_llm_provider": "azure"},
|
||||
encoding=Mock(),
|
||||
)
|
||||
|
||||
message: Final = result.choices[0].message
|
||||
assert [item["id"] for item in message.reasoning_items] == kept_ids
|
||||
assert [tool_call.function.name for tool_call in message.tool_calls] == ["get_weather", "get_weather"]
|
||||
|
||||
|
||||
def test_streaming_function_call_tool_id_for_degenerate_call_id():
|
||||
"""In streaming, Bedrock Mantle's function_call event carries a unique ``id``
|
||||
(``fc_...``) and a non-unique, index-based ``call_id`` (``call_0``). For that
|
||||
|
|
|
|||
|
|
@ -698,6 +698,34 @@ def test_encrypted_reasoning_is_kept_when_the_request_tools_route_it_to_response
|
|||
assert bool(assistant.get("thinking_blocks")) is kept
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_params", "kept"),
|
||||
[
|
||||
pytest.param({"reasoning_summary": "auto"}, True, id="deployment_summary_bridges_to_responses"),
|
||||
pytest.param({"extra_body": {"reasoningSummary": "auto"}}, True, id="extra_body_summary_bridges_to_responses"),
|
||||
pytest.param({}, False, id="no_summary_stays_on_chat"),
|
||||
],
|
||||
)
|
||||
def test_encrypted_reasoning_is_kept_when_the_deployment_reasoning_summary_routes_it_to_responses(
|
||||
deployment_params: dict[str, object], kept: bool
|
||||
) -> None:
|
||||
from litellm.llms.anthropic.pass_through.adapters.handler import (
|
||||
LiteLLMMessagesToCompletionTransformationHandler,
|
||||
)
|
||||
|
||||
request: Final = _replayed_reasoning_request([])
|
||||
completion_kwargs, _mapping = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs(
|
||||
max_tokens=64,
|
||||
messages=cast(list[dict[str, object]], request["messages"]),
|
||||
model="azure/gpt-5.1",
|
||||
thinking={"type": "enabled", "budget_tokens": 2048},
|
||||
extra_kwargs={"api_base": "https://keep-check.openai.azure.com", **deployment_params},
|
||||
)
|
||||
|
||||
assistant: Final = cast(list[Mapping[str, object]], completion_kwargs["messages"])[1]
|
||||
assert bool(assistant.get("thinking_blocks")) is kept
|
||||
|
||||
|
||||
def test_translate_anthropic_messages_to_openai_sets_reasoning_content():
|
||||
"""Reasoning-aware chat providers read reasoning_content, so thinking text must land there.
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue