diff --git a/litellm/completion_extras/litellm_responses_transformation/history.py b/litellm/completion_extras/litellm_responses_transformation/history.py index e5d75553da3..1b07baef990 100644 --- a/litellm/completion_extras/litellm_responses_transformation/history.py +++ b/litellm/completion_extras/litellm_responses_transformation/history.py @@ -264,7 +264,7 @@ def _matches(items: Sequence[_OutputItem], message: _ChatMessage) -> bool: def _last_phase(items: Sequence[_OutputItem | None]) -> Phase | None: - messages: Final = tuple(item for item in items if isinstance(item, _OutputMessage) and item.phase is not None) + messages: Final = tuple(item for item in items if isinstance(item, _OutputMessage)) return messages[-1].phase if messages else None @@ -364,6 +364,23 @@ def _input_item(resolved: _Resolved, position: int, entry: _MessageEntry | _Inde return dict(resolved.tool_searches[entry.index]) +def _fits(message: _ChatMessage, entries: Sequence[_MessageEntry | _IndexedEntry]) -> bool: + tool_calls: Final = len(message.tool_calls or ()) + tool_searches: Final = len(_tool_search_entries(message)) + sizes: Final = MappingProxyType( + { + "reasoning": len(message.reasoning_items or ()), + "function_call": tool_calls, + "custom_tool_call": tool_calls, + "tool_search_call": tool_searches, + "tool_search_output": tool_searches, + } + ) + parts: Final = chain.from_iterable(entry.parts for entry in entries if isinstance(entry, _MessageEntry)) + indexed: Final = (entry for entry in entries if isinstance(entry, _IndexedEntry)) + return sum(parts) == len(message.content or "") and all(entry.index < sizes[entry.type] for entry in indexed) + + def _resolve( message: _ChatMessage, entries: Sequence[_MessageEntry | _IndexedEntry], replay_tool_search: bool ) -> ReplayedTurn | None: @@ -391,4 +408,6 @@ def replay_items(message: object, *, replay_tool_search: bool) -> ReplayedTurn | stored: Final = _stored_replay(chat) if chat is not None and chat.role == "assistant" else None if chat is None or stored is None or stored.checksum != _checksum(chat, stored.items): return None + if not _fits(chat, stored.items): + return None return _resolve(chat, stored.items, replay_tool_search) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 61a11678901..6d2202695a2 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -1275,12 +1275,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): for item in output_items: if isinstance(item, ResponseReasoningItem): - built_reasoning_item = _build_reasoning_item( - item_id=item.id, - encrypted_content=getattr(item, "encrypted_content", None), - summary_raw=item.summary, + pending_reasoning_items = ( + *(pending_reasoning_items if replay_openai_history else ()), + _build_reasoning_item( + item_id=item.id, + encrypted_content=getattr(item, "encrypted_content", None), + summary_raw=item.summary, + ), ) - pending_reasoning_items = (*pending_reasoning_items, built_reasoning_item) reasoning_content = " ".join( s["text"] for s in chain.from_iterable(built["summary"] for built in pending_reasoning_items) @@ -2269,7 +2271,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): StreamingChoices( index=0, delta=Delta( - content=None, + content=None if replays_openai_history else "", reasoning_items=terminal_reasoning_items_typed, provider_specific_fields=( completed_stream_history( diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_history.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_history.py index 3aaed84b9e4..0ea9a32d455 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_history.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_history.py @@ -7,6 +7,7 @@ import pytest import respx import litellm +from litellm.completion_extras.litellm_responses_transformation import history from litellm.exceptions import MidStreamFallbackError OPENAI_RESPONSES_URL: Final = "https://api.openai.com/v1/responses" @@ -59,7 +60,12 @@ FINAL_ANSWER: Final = { } OUTPUT: Final = (COMMENTARY, FIRST_REASONING, SECOND_REASONING, WEATHER_CALL, FINAL_ANSWER) REPLAYED: Final = ( - {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "Checking the weather."}], "phase": "commentary"}, + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "Checking the weather."}], + "phase": "commentary", + }, { "type": "reasoning", "id": "rs_first", @@ -199,8 +205,8 @@ def _assistant_turn( stream=stream, ) message: Final = ( - litellm.stream_chunk_builder(list(response), messages=[USER]) if stream else response - ).choices[0].message + (litellm.stream_chunk_builder(list(response), messages=[USER]) if stream else response).choices[0].message + ) return json.loads(json.dumps(message.model_dump())) @@ -323,6 +329,18 @@ def test_missing_or_corrupt_replay_metadata_falls_back_to_the_chat_fields( assert sent[3]["phase"] == "final_answer" +@pytest.mark.parametrize("stream", [False, True], ids=["json", "stream"]) +def test_an_unphased_answer_after_commentary_is_never_labeled_commentary(stream: bool) -> None: + unphased_answer: Final = {key: value for key, value in FINAL_ANSWER.items() if key != "phase"} + assistant: Final = _assistant_turn((COMMENTARY, WEATHER_CALL, unphased_answer), stream=stream) + + sent: Final = _next_turn_input(_edit_content(assistant)) + + assert "phase" not in _replayed_fields(assistant) + assert _types(sent[1:-1]) == ["message", "function_call"] + assert "phase" not in sent[1] + + def test_a_phase_openai_does_not_define_is_dropped_under_drop_params() -> None: assistant: Final = _assistant_turn(OUTPUT, stream=False) @@ -461,6 +479,77 @@ def test_a_non_openai_route_ignores_a_recorded_replay() -> None: assert all("phase" not in item for item in sent) +SECOND_THOUGHT: Final = {**SECOND_REASONING, "summary": [{"type": "summary_text", "text": "Lyon is likely dry."}]} +LYON_CALL: Final = {**WEATHER_CALL, "id": "fc_lyon", "call_id": "call_weather_lyon", "arguments": '{"city":"Lyon"}'} + + +@pytest.mark.parametrize( + ("model", "expected_ids", "expected_text"), + [ + (f"openai/responses/{MODEL}", ["rs_first", "rs_second"], "Paris is likely wet. Lyon is likely dry."), + (f"azure/responses/{MODEL}", ["rs_second"], "Lyon is likely dry."), + ], + ids=["openai", "azure"], +) +def test_only_the_openai_route_keeps_every_reasoning_item_of_a_tool_call_turn( + model: str, expected_ids: list[str], expected_text: str +) -> None: + assistant: Final = _assistant_turn( + (FIRST_REASONING, WEATHER_CALL, SECOND_THOUGHT, LYON_CALL), stream=False, model=model + ) + + reasoning_items: Final = assistant["reasoning_items"] + assert isinstance(reasoning_items, list) + assert [item["id"] for item in reasoning_items] == expected_ids + assert assistant["reasoning_content"] == expected_text + + +@pytest.mark.parametrize( + ("model", "expected_content"), + [(f"openai/responses/{MODEL}", None), (f"azure/responses/{MODEL}", "")], + ids=["openai", "azure"], +) +def test_only_the_openai_route_ends_a_streamed_tool_call_turn_with_null_content( + model: str, expected_content: str | None +) -> None: + assistant: Final = _assistant_turn((FIRST_REASONING, WEATHER_CALL), stream=True, model=model) + + assert assistant["content"] == expected_content + + +def _resealed(message: Mapping[str, object], items: Sequence[Mapping[str, object]]) -> Mapping[str, object]: + chat: Final = history._chat_message(message) + assert chat is not None + entries: Final = history._Replay.model_validate({"version": 1, "items": items, "checksum": ""}).items + replay: Final = {"version": 1, "items": list(items), "checksum": history._checksum(chat, entries)} + return {**message, "provider_specific_fields": _with_replay(_replayed_fields(message), replay)} + + +@pytest.mark.parametrize( + "items", + [ + ( + {"type": "message", "parts": [21]}, + {"type": "function_call", "index": 1}, + {"type": "message", "parts": [7, 11]}, + ), + ( + {"type": "message", "parts": [5]}, + {"type": "function_call", "index": 0}, + {"type": "message", "parts": [7, 11]}, + ), + ], + ids=["tool-call-index", "text-lengths"], +) +def test_a_resealed_record_that_does_not_fit_the_chat_fields_falls_back(items: Sequence[Mapping[str, object]]) -> None: + assistant: Final = _assistant_turn((COMMENTARY, WEATHER_CALL, FINAL_ANSWER), stream=False) + + sent: Final = _next_turn_input(_resealed(assistant, items)) + + assert _types(sent[1:-1]) == ["message", "function_call"] + assert sent[1]["content"] == [{"type": "output_text", "text": "Checking the weather.Paris: light rain."}] + + PATCH_CALL: Final = { "type": "custom_tool_call", "id": "ctc_patch", @@ -488,7 +577,17 @@ def test_a_custom_tool_call_replays_with_its_custom_output(stream: bool) -> None sent: Final = json.loads(upstream.calls[0].request.content)["input"] assert sent[1:] == [ - {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "Checking the weather."}], "phase": "commentary"}, - {"type": "custom_tool_call", "call_id": "call_patch_readme", "name": "apply_patch", "input": PATCH_CALL["input"]}, + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "Checking the weather."}], + "phase": "commentary", + }, + { + "type": "custom_tool_call", + "call_id": "call_patch_readme", + "name": "apply_patch", + "input": PATCH_CALL["input"], + }, {"type": "custom_tool_call_output", "call_id": "call_patch_readme", "output": "applied"}, ] diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 3c7bc8f2142..73574164d26 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -686,7 +686,7 @@ def test_streamed_tool_search_turn_replays_with_its_reasoning_item(): ], ids=["tool-call", "message"], ) -def test_every_reasoning_item_before_an_answer_reaches_the_chat_message(answer_item: object) -> None: +def test_every_reasoning_item_before_an_answer_reaches_an_openai_chat_message(answer_item: object) -> None: reasoning: Final = tuple( ResponseReasoningItem( type="reasoning", @@ -697,9 +697,9 @@ def test_every_reasoning_item_before_an_answer_reaches_the_chat_message(answer_i for name in ("first", "second") ) - message: Final = ( - LiteLLMResponsesTransformationHandler._convert_response_output_to_choices([*reasoning, answer_item])[0].message - ) + message: Final = LiteLLMResponsesTransformationHandler._convert_response_output_to_choices( + [*reasoning, answer_item], replay_openai_history=True + )[0].message assert [item["id"] for item in message.reasoning_items] == ["rs_first", "rs_second"] assert message.reasoning_content == "First. Second." @@ -1287,9 +1287,7 @@ def test_transform_request_forwards_defer_loading_only_beside_tool_search( [{"type": "tool_search", "execution": "client"}, {"type": "tool_search"}], ids=["client-only", "hosted-and-client"], ) -def test_transform_request_drops_defer_loading_beside_client_tool_search( - deferred_tool, drop_params, tool_search_tool -): +def test_transform_request_drops_defer_loading_beside_client_tool_search(deferred_tool, drop_params, tool_search_tool): tool_search_tools: Final = ( [tool_search_tool] if tool_search_tool.get("execution") == "client" @@ -4413,7 +4411,11 @@ def test_tool_search_items_replay_only_to_a_request_that_carries_tool_search(too "provider_specific_fields": { "tool_search_items": [ {"type": "tool_search_call", "id": "tsc_1", "arguments": {"paths": ["get_weather"]}}, - {"type": "tool_search_output", "id": "tso_1", "tools": [{"type": "function", "name": "get_weather"}]}, + { + "type": "tool_search_output", + "id": "tso_1", + "tools": [{"type": "function", "name": "get_weather"}], + }, ] }, },