mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-14 09:03:16 +00:00
fix(anthropic): keep every non-streamed reasoning item for Messages clients on the line's bridge
This line's transform_request rewrites the logging call_type to responses, so the non-streamed reply read the Messages-client check after the rewrite and kept only the last reasoning item. The bridge handler now reads it before the request transform and passes it to transform_response, as the streamed path already does.
This commit is contained in:
mateo-berri
parent
ce35f27bfa
commit
97df7cb369
4 changed files with 114 additions and 11 deletions
|
|
@ -184,7 +184,7 @@ class ResponsesToCompletionBridgeHandler:
|
|||
custom_llm_provider: Final = validated_kwargs["custom_llm_provider"]
|
||||
from .transformation import serves_anthropic_messages
|
||||
|
||||
streams_reasoning_signatures: Final = serves_anthropic_messages(logging_obj)
|
||||
serves_messages_client: Final = serves_anthropic_messages(logging_obj)
|
||||
if kwargs.get("stream") is True and "stream" not in optional_params:
|
||||
optional_params = {**optional_params, "stream": True}
|
||||
|
||||
|
|
@ -225,6 +225,7 @@ class ResponsesToCompletionBridgeHandler:
|
|||
encoding=validated_kwargs["encoding"],
|
||||
api_key=kwargs.get("api_key"),
|
||||
json_mode=kwargs.get("json_mode"),
|
||||
keeps_every_reasoning_item=serves_messages_client,
|
||||
)
|
||||
elif isinstance(result, ModelResponse):
|
||||
if not stream:
|
||||
|
|
@ -250,6 +251,7 @@ class ResponsesToCompletionBridgeHandler:
|
|||
encoding=validated_kwargs["encoding"],
|
||||
api_key=kwargs.get("api_key"),
|
||||
json_mode=kwargs.get("json_mode"),
|
||||
keeps_every_reasoning_item=serves_messages_client,
|
||||
)
|
||||
else:
|
||||
if self._is_preformatted_cached_chat_stream(result):
|
||||
|
|
@ -263,7 +265,7 @@ class ResponsesToCompletionBridgeHandler:
|
|||
structured_output=is_structured_output_request(
|
||||
cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict
|
||||
),
|
||||
streams_reasoning_signatures=streams_reasoning_signatures,
|
||||
streams_reasoning_signatures=serves_messages_client,
|
||||
)
|
||||
streamwrapper: Final = CustomStreamWrapper(
|
||||
completion_stream=completion_stream,
|
||||
|
|
@ -288,7 +290,7 @@ class ResponsesToCompletionBridgeHandler:
|
|||
custom_llm_provider: Final = validated_kwargs["custom_llm_provider"]
|
||||
from .transformation import serves_anthropic_messages
|
||||
|
||||
streams_reasoning_signatures: Final = serves_anthropic_messages(logging_obj)
|
||||
serves_messages_client: Final = serves_anthropic_messages(logging_obj)
|
||||
if kwargs.get("stream") is True and "stream" not in optional_params:
|
||||
optional_params = {**optional_params, "stream": True}
|
||||
|
||||
|
|
@ -332,6 +334,7 @@ class ResponsesToCompletionBridgeHandler:
|
|||
encoding=validated_kwargs["encoding"],
|
||||
api_key=kwargs.get("api_key"),
|
||||
json_mode=kwargs.get("json_mode"),
|
||||
keeps_every_reasoning_item=serves_messages_client,
|
||||
)
|
||||
elif isinstance(result, ModelResponse):
|
||||
if not stream:
|
||||
|
|
@ -357,6 +360,7 @@ class ResponsesToCompletionBridgeHandler:
|
|||
encoding=validated_kwargs["encoding"],
|
||||
api_key=kwargs.get("api_key"),
|
||||
json_mode=kwargs.get("json_mode"),
|
||||
keeps_every_reasoning_item=serves_messages_client,
|
||||
)
|
||||
else:
|
||||
if self._is_preformatted_cached_chat_stream(result):
|
||||
|
|
@ -370,7 +374,7 @@ class ResponsesToCompletionBridgeHandler:
|
|||
structured_output=is_structured_output_request(
|
||||
cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict
|
||||
),
|
||||
streams_reasoning_signatures=streams_reasoning_signatures,
|
||||
streams_reasoning_signatures=serves_messages_client,
|
||||
)
|
||||
streamwrapper: Final = CustomStreamWrapper(
|
||||
completion_stream=completion_stream,
|
||||
|
|
|
|||
|
|
@ -1482,6 +1482,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
encoding: object,
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
keeps_every_reasoning_item: bool = False,
|
||||
) -> "ModelResponse":
|
||||
"""Transform Responses API response to chat completion response"""
|
||||
from litellm.responses.utils import ResponseAPILoggingUtils
|
||||
|
|
@ -1513,7 +1514,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
structured_output=is_structured_output_request(
|
||||
cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict
|
||||
),
|
||||
keeps_every_reasoning_item=serves_anthropic_messages(logging_obj),
|
||||
keeps_every_reasoning_item=keeps_every_reasoning_item,
|
||||
)
|
||||
|
||||
response_is_incomplete: Final = raw_response.status == "incomplete" or (
|
||||
|
|
|
|||
|
|
@ -335,3 +335,101 @@ async def test_acompletion_keeps_provider_native_model_id_through_responses(
|
|||
|
||||
handed_model = responses_call.call_args.kwargs["model"]
|
||||
assert _upstream_model_for(handed_model, custom_llm_provider) == expected_upstream_model
|
||||
|
||||
|
||||
def _two_reasoning_tool_turn():
|
||||
from openai.types.responses import ResponseFunctionToolCall
|
||||
from openai.types.responses.response_reasoning_item import ResponseReasoningItem
|
||||
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
|
||||
def reasoning(item_id: str) -> ResponseReasoningItem:
|
||||
return ResponseReasoningItem(
|
||||
id=item_id, summary=[], type="reasoning", encrypted_content=f"gAAAA_{item_id}"
|
||||
)
|
||||
|
||||
def call(index: int, city: str) -> ResponseFunctionToolCall:
|
||||
return ResponseFunctionToolCall(
|
||||
id=f"fc_{index}",
|
||||
type="function_call",
|
||||
status="completed",
|
||||
arguments=f'{{"city": "{city}"}}',
|
||||
call_id=f"call_{index}",
|
||||
name="get_weather",
|
||||
)
|
||||
|
||||
return ResponsesAPIResponse.model_construct(
|
||||
id="resp_two_reasoning",
|
||||
created_at=1,
|
||||
error=None,
|
||||
incomplete_details=None,
|
||||
model="gpt-5.4-nano",
|
||||
output=[reasoning("rs_first"), call(1, "Oslo"), reasoning("rs_second"), call(2, "Lyon")],
|
||||
status="completed",
|
||||
usage=None,
|
||||
)
|
||||
|
||||
|
||||
class _CompletedStream:
|
||||
def __init__(self, response):
|
||||
self.completed_response = MagicMock(response=response)
|
||||
self._hidden_params = {}
|
||||
|
||||
def __iter__(self):
|
||||
return iter(())
|
||||
|
||||
def __aiter__(self):
|
||||
return self
|
||||
|
||||
async def __anext__(self):
|
||||
raise StopAsyncIteration
|
||||
|
||||
|
||||
def _messages_client_kwargs(call_type: str):
|
||||
messages = [{"role": "user", "content": "Weather in Oslo and Lyon?"}]
|
||||
return {
|
||||
"model": "gpt-5.4-nano",
|
||||
"custom_llm_provider": "azure",
|
||||
"messages": messages,
|
||||
"optional_params": {},
|
||||
"litellm_params": {},
|
||||
"headers": {},
|
||||
"model_response": ModelResponse(),
|
||||
"logging_obj": LiteLLMLogging(
|
||||
litellm_call_id="test-call",
|
||||
call_type=call_type,
|
||||
model="gpt-5.4-nano",
|
||||
messages=messages,
|
||||
function_id="fn-id",
|
||||
stream=False,
|
||||
start_time=datetime.now(),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_async", [True, False], ids=["acompletion", "completion"])
|
||||
@pytest.mark.parametrize("arrives_streamed", [False, True], ids=["response", "collected_stream"])
|
||||
@pytest.mark.parametrize(
|
||||
("call_type", "kept_ids"),
|
||||
[
|
||||
pytest.param("anthropic_messages", ["rs_first", "rs_second"], id="messages_client"),
|
||||
pytest.param("acompletion", ["rs_second"], id="chat_client"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_streamed_bridge_hands_messages_clients_every_reasoning_item(
|
||||
is_async: bool, arrives_streamed: bool, call_type: str, kept_ids: list[str]
|
||||
):
|
||||
turn = _two_reasoning_tool_turn()
|
||||
upstream = _CompletedStream(turn) if arrives_streamed else turn
|
||||
bridge = ResponsesToCompletionBridgeHandler()
|
||||
kwargs = _messages_client_kwargs(call_type)
|
||||
|
||||
if is_async:
|
||||
with patch("litellm.aresponses", new=AsyncMock(return_value=upstream)):
|
||||
result = await bridge.acompletion(**kwargs)
|
||||
else:
|
||||
with patch("litellm.responses", return_value=upstream):
|
||||
result = bridge.completion(**kwargs)
|
||||
|
||||
assert [item["id"] for item in result.choices[0].message.reasoning_items] == kept_ids
|
||||
|
|
|
|||
|
|
@ -3571,16 +3571,15 @@ def test_transform_response_hands_encrypted_reasoning_to_messages_clients_only()
|
|||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("call_type", "kept_ids"),
|
||||
("keeps_every_reasoning_item", "kept_ids"),
|
||||
[
|
||||
pytest.param("anthropic_messages", ["rs_first", "rs_second"], id="messages_client_keeps_every_item"),
|
||||
pytest.param("acompletion", ["rs_second"], id="chat_client_keeps_the_last_item"),
|
||||
pytest.param(True, ["rs_first", "rs_second"], id="messages_client_keeps_every_item"),
|
||||
pytest.param(False, ["rs_second"], id="chat_client_keeps_the_last_item"),
|
||||
],
|
||||
)
|
||||
def test_non_streamed_tool_turn_hands_messages_clients_every_reasoning_item(
|
||||
call_type: str, kept_ids: list[str]
|
||||
keeps_every_reasoning_item: bool, kept_ids: list[str]
|
||||
) -> None:
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
|
|
@ -3623,12 +3622,13 @@ def test_non_streamed_tool_turn_hands_messages_clients_every_reasoning_item(
|
|||
choices=[],
|
||||
usage=Usage(completion_tokens=0, prompt_tokens=0, total_tokens=0),
|
||||
),
|
||||
logging_obj=SimpleNamespace(call_type=call_type),
|
||||
logging_obj=Mock(),
|
||||
request_data={"model": "gpt-5.4-nano"},
|
||||
messages=[{"role": "user", "content": "Oslo or Lyon?"}],
|
||||
optional_params={},
|
||||
litellm_params={"custom_llm_provider": "azure"},
|
||||
encoding=Mock(),
|
||||
keeps_every_reasoning_item=keeps_every_reasoning_item,
|
||||
)
|
||||
|
||||
message: Final = result.choices[0].message
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue