fix(responses): keep other providers' reasoning and stream output unchanged and fall back on records that do not fit

litellm/completion_extras/litellm_responses_transformation/history.py

@@ -264,7 +264,7 @@ def _matches(items: Sequence[_OutputItem], message: _ChatMessage) -> bool:
 
 
 def _last_phase(items: Sequence[_OutputItem | None]) -> Phase | None:
-    messages: Final = tuple(item for item in items if isinstance(item, _OutputMessage) and item.phase is not None)
+    messages: Final = tuple(item for item in items if isinstance(item, _OutputMessage))
     return messages[-1].phase if messages else None
 
 
@@ -364,6 +364,23 @@ def _input_item(resolved: _Resolved, position: int, entry: _MessageEntry | _Inde
             return dict(resolved.tool_searches[entry.index])
 
 
+def _fits(message: _ChatMessage, entries: Sequence[_MessageEntry | _IndexedEntry]) -> bool:
+    tool_calls: Final = len(message.tool_calls or ())
+    tool_searches: Final = len(_tool_search_entries(message))
+    sizes: Final = MappingProxyType(
+        {
+            "reasoning": len(message.reasoning_items or ()),
+            "function_call": tool_calls,
+            "custom_tool_call": tool_calls,
+            "tool_search_call": tool_searches,
+            "tool_search_output": tool_searches,
+        }
+    )
+    parts: Final = chain.from_iterable(entry.parts for entry in entries if isinstance(entry, _MessageEntry))
+    indexed: Final = (entry for entry in entries if isinstance(entry, _IndexedEntry))
+    return sum(parts) == len(message.content or "") and all(entry.index < sizes[entry.type] for entry in indexed)
+
+
 def _resolve(
     message: _ChatMessage, entries: Sequence[_MessageEntry | _IndexedEntry], replay_tool_search: bool
 ) -> ReplayedTurn | None:
@@ -391,4 +408,6 @@ def replay_items(message: object, *, replay_tool_search: bool) -> ReplayedTurn |
     stored: Final = _stored_replay(chat) if chat is not None and chat.role == "assistant" else None
     if chat is None or stored is None or stored.checksum != _checksum(chat, stored.items):
         return None
+    if not _fits(chat, stored.items):
+        return None
     return _resolve(chat, stored.items, replay_tool_search)

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -1275,12 +1275,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
 
         for item in output_items:
             if isinstance(item, ResponseReasoningItem):
-                built_reasoning_item = _build_reasoning_item(
-                    item_id=item.id,
-                    encrypted_content=getattr(item, "encrypted_content", None),
-                    summary_raw=item.summary,
+                pending_reasoning_items = (
+                    *(pending_reasoning_items if replay_openai_history else ()),
+                    _build_reasoning_item(
+                        item_id=item.id,
+                        encrypted_content=getattr(item, "encrypted_content", None),
+                        summary_raw=item.summary,
+                    ),
                 )
-                pending_reasoning_items = (*pending_reasoning_items, built_reasoning_item)
                 reasoning_content = " ".join(
                     s["text"]
                     for s in chain.from_iterable(built["summary"] for built in pending_reasoning_items)
@@ -2269,7 +2271,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
                     StreamingChoices(
                         index=0,
                         delta=Delta(
-                            content=None,
+                            content=None if replays_openai_history else "",
                             reasoning_items=terminal_reasoning_items_typed,
                             provider_specific_fields=(
                                 completed_stream_history(

tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_history.py

@@ -7,6 +7,7 @@ import pytest
 import respx
 
 import litellm
+from litellm.completion_extras.litellm_responses_transformation import history
 from litellm.exceptions import MidStreamFallbackError
 
 OPENAI_RESPONSES_URL: Final = "https://api.openai.com/v1/responses"
@@ -59,7 +60,12 @@ FINAL_ANSWER: Final = {
 }
 OUTPUT: Final = (COMMENTARY, FIRST_REASONING, SECOND_REASONING, WEATHER_CALL, FINAL_ANSWER)
 REPLAYED: Final = (
-    {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "Checking the weather."}], "phase": "commentary"},
+    {
+        "type": "message",
+        "role": "assistant",
+        "content": [{"type": "output_text", "text": "Checking the weather."}],
+        "phase": "commentary",
+    },
     {
         "type": "reasoning",
         "id": "rs_first",
@@ -199,8 +205,8 @@ def _assistant_turn(
             stream=stream,
         )
         message: Final = (
-            litellm.stream_chunk_builder(list(response), messages=[USER]) if stream else response
-        ).choices[0].message
+            (litellm.stream_chunk_builder(list(response), messages=[USER]) if stream else response).choices[0].message
+        )
     return json.loads(json.dumps(message.model_dump()))
 
 
@@ -323,6 +329,18 @@ def test_missing_or_corrupt_replay_metadata_falls_back_to_the_chat_fields(
     assert sent[3]["phase"] == "final_answer"
 
 
+@pytest.mark.parametrize("stream", [False, True], ids=["json", "stream"])
+def test_an_unphased_answer_after_commentary_is_never_labeled_commentary(stream: bool) -> None:
+    unphased_answer: Final = {key: value for key, value in FINAL_ANSWER.items() if key != "phase"}
+    assistant: Final = _assistant_turn((COMMENTARY, WEATHER_CALL, unphased_answer), stream=stream)
+
+    sent: Final = _next_turn_input(_edit_content(assistant))
+
+    assert "phase" not in _replayed_fields(assistant)
+    assert _types(sent[1:-1]) == ["message", "function_call"]
+    assert "phase" not in sent[1]
+
+
 def test_a_phase_openai_does_not_define_is_dropped_under_drop_params() -> None:
     assistant: Final = _assistant_turn(OUTPUT, stream=False)
 
@@ -461,6 +479,77 @@ def test_a_non_openai_route_ignores_a_recorded_replay() -> None:
     assert all("phase" not in item for item in sent)
 
 
+SECOND_THOUGHT: Final = {**SECOND_REASONING, "summary": [{"type": "summary_text", "text": "Lyon is likely dry."}]}
+LYON_CALL: Final = {**WEATHER_CALL, "id": "fc_lyon", "call_id": "call_weather_lyon", "arguments": '{"city":"Lyon"}'}
+
+
+@pytest.mark.parametrize(
+    ("model", "expected_ids", "expected_text"),
+    [
+        (f"openai/responses/{MODEL}", ["rs_first", "rs_second"], "Paris is likely wet. Lyon is likely dry."),
+        (f"azure/responses/{MODEL}", ["rs_second"], "Lyon is likely dry."),
+    ],
+    ids=["openai", "azure"],
+)
+def test_only_the_openai_route_keeps_every_reasoning_item_of_a_tool_call_turn(
+    model: str, expected_ids: list[str], expected_text: str
+) -> None:
+    assistant: Final = _assistant_turn(
+        (FIRST_REASONING, WEATHER_CALL, SECOND_THOUGHT, LYON_CALL), stream=False, model=model
+    )
+
+    reasoning_items: Final = assistant["reasoning_items"]
+    assert isinstance(reasoning_items, list)
+    assert [item["id"] for item in reasoning_items] == expected_ids
+    assert assistant["reasoning_content"] == expected_text
+
+
+@pytest.mark.parametrize(
+    ("model", "expected_content"),
+    [(f"openai/responses/{MODEL}", None), (f"azure/responses/{MODEL}", "")],
+    ids=["openai", "azure"],
+)
+def test_only_the_openai_route_ends_a_streamed_tool_call_turn_with_null_content(
+    model: str, expected_content: str | None
+) -> None:
+    assistant: Final = _assistant_turn((FIRST_REASONING, WEATHER_CALL), stream=True, model=model)
+
+    assert assistant["content"] == expected_content
+
+
+def _resealed(message: Mapping[str, object], items: Sequence[Mapping[str, object]]) -> Mapping[str, object]:
+    chat: Final = history._chat_message(message)
+    assert chat is not None
+    entries: Final = history._Replay.model_validate({"version": 1, "items": items, "checksum": ""}).items
+    replay: Final = {"version": 1, "items": list(items), "checksum": history._checksum(chat, entries)}
+    return {**message, "provider_specific_fields": _with_replay(_replayed_fields(message), replay)}
+
+
+@pytest.mark.parametrize(
+    "items",
+    [
+        (
+            {"type": "message", "parts": [21]},
+            {"type": "function_call", "index": 1},
+            {"type": "message", "parts": [7, 11]},
+        ),
+        (
+            {"type": "message", "parts": [5]},
+            {"type": "function_call", "index": 0},
+            {"type": "message", "parts": [7, 11]},
+        ),
+    ],
+    ids=["tool-call-index", "text-lengths"],
+)
+def test_a_resealed_record_that_does_not_fit_the_chat_fields_falls_back(items: Sequence[Mapping[str, object]]) -> None:
+    assistant: Final = _assistant_turn((COMMENTARY, WEATHER_CALL, FINAL_ANSWER), stream=False)
+
+    sent: Final = _next_turn_input(_resealed(assistant, items))
+
+    assert _types(sent[1:-1]) == ["message", "function_call"]
+    assert sent[1]["content"] == [{"type": "output_text", "text": "Checking the weather.Paris: light rain."}]
+
+
 PATCH_CALL: Final = {
     "type": "custom_tool_call",
     "id": "ctc_patch",
@@ -488,7 +577,17 @@ def test_a_custom_tool_call_replays_with_its_custom_output(stream: bool) -> None
     sent: Final = json.loads(upstream.calls[0].request.content)["input"]
 
     assert sent[1:] == [
-        {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "Checking the weather."}], "phase": "commentary"},
-        {"type": "custom_tool_call", "call_id": "call_patch_readme", "name": "apply_patch", "input": PATCH_CALL["input"]},
+        {
+            "type": "message",
+            "role": "assistant",
+            "content": [{"type": "output_text", "text": "Checking the weather."}],
+            "phase": "commentary",
+        },
+        {
+            "type": "custom_tool_call",
+            "call_id": "call_patch_readme",
+            "name": "apply_patch",
+            "input": PATCH_CALL["input"],
+        },
         {"type": "custom_tool_call_output", "call_id": "call_patch_readme", "output": "applied"},
     ]

tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py

@@ -686,7 +686,7 @@ def test_streamed_tool_search_turn_replays_with_its_reasoning_item():
     ],
     ids=["tool-call", "message"],
 )
-def test_every_reasoning_item_before_an_answer_reaches_the_chat_message(answer_item: object) -> None:
+def test_every_reasoning_item_before_an_answer_reaches_an_openai_chat_message(answer_item: object) -> None:
     reasoning: Final = tuple(
         ResponseReasoningItem(
             type="reasoning",
@@ -697,9 +697,9 @@ def test_every_reasoning_item_before_an_answer_reaches_the_chat_message(answer_i
         for name in ("first", "second")
     )
 
-    message: Final = (
-        LiteLLMResponsesTransformationHandler._convert_response_output_to_choices([*reasoning, answer_item])[0].message
-    )
+    message: Final = LiteLLMResponsesTransformationHandler._convert_response_output_to_choices(
+        [*reasoning, answer_item], replay_openai_history=True
+    )[0].message
 
     assert [item["id"] for item in message.reasoning_items] == ["rs_first", "rs_second"]
     assert message.reasoning_content == "First. Second."
@@ -1287,9 +1287,7 @@ def test_transform_request_forwards_defer_loading_only_beside_tool_search(
     [{"type": "tool_search", "execution": "client"}, {"type": "tool_search"}],
     ids=["client-only", "hosted-and-client"],
 )
-def test_transform_request_drops_defer_loading_beside_client_tool_search(
-    deferred_tool, drop_params, tool_search_tool
-):
+def test_transform_request_drops_defer_loading_beside_client_tool_search(deferred_tool, drop_params, tool_search_tool):
     tool_search_tools: Final = (
         [tool_search_tool]
         if tool_search_tool.get("execution") == "client"
@@ -4413,7 +4411,11 @@ def test_tool_search_items_replay_only_to_a_request_that_carries_tool_search(too
             "provider_specific_fields": {
                 "tool_search_items": [
                     {"type": "tool_search_call", "id": "tsc_1", "arguments": {"paths": ["get_weather"]}},
-                    {"type": "tool_search_output", "id": "tso_1", "tools": [{"type": "function", "name": "get_weather"}]},
+                    {
+                        "type": "tool_search_output",
+                        "id": "tso_1",
+                        "tools": [{"type": "function", "name": "get_weather"}],
+                    },
                 ]
             },
         },