diff --git a/litellm/responses/litellm_completion_transformation/handler.py b/litellm/responses/litellm_completion_transformation/handler.py index 802ed6bf9c5..7b229a0f98d 100644 --- a/litellm/responses/litellm_completion_transformation/handler.py +++ b/litellm/responses/litellm_completion_transformation/handler.py @@ -10,6 +10,7 @@ from typing import Final from pydantic import TypeAdapter import litellm +from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id from litellm.responses.additional_tools import HoistedAdditionalTools, hoist_additional_tools @@ -96,7 +97,9 @@ class LiteLLMCompletionTransformationHandler: responses_api_request.get("tools"), drop_params=drops_unsupported_params(_REQUEST_KWARGS_ADAPTER.validate_python(kwargs).get("drop_params")), ) - previous_response_id: Final = responses_api_request.get("previous_response_id") + previous_response_id: Final = ( + responses_api_request["previous_response_id"] if "previous_response_id" in responses_api_request else None # noqa: SIM401 # TypedDict indexing preserves the field type + ) if _is_async and previous_response_id and has_client_tool_search(hoisted.tools): return self._async_response_api_handler_after_earlier_searches( previous_response_client_ids, @@ -113,20 +116,22 @@ class LiteLLMCompletionTransformationHandler: bridged: Final = _bridged_request(hoisted, responses_api_request, ()) bridged_input: Final = bridged.input bridged_request: Final = bridged.responses_api_request - litellm_completion_request: Final[dict] = { - **LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( - model=model, - input=bridged_input, - caller_input=input, - names_non_string_fields=True, - responses_api_request=bridged_request, - custom_llm_provider=custom_llm_provider, - stream=stream, - extra_headers=extra_headers, - **_without_bridge_transform_keywords(_REQUEST_KWARGS_ADAPTER.validate_python(kwargs)), - ), - **_chat_verbosity(bridged_request), - } + litellm_completion_request: Final = _REQUEST_KWARGS_ADAPTER.validate_python( + { + **LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model=model, + input=bridged_input, + caller_input=input, + names_non_string_fields=True, + responses_api_request=bridged_request, + custom_llm_provider=custom_llm_provider, + stream=stream, + extra_headers=extra_headers, + **_without_bridge_transform_keywords(_REQUEST_KWARGS_ADAPTER.validate_python(kwargs)), + ), + **_chat_verbosity(bridged_request), + } + ) if _is_async: return self.async_response_api_handler( @@ -137,8 +142,24 @@ class LiteLLMCompletionTransformationHandler: **kwargs, ) + completion_request: Final = _REQUEST_KWARGS_ADAPTER.validate_python( + run_async_function( + LiteLLMCompletionResponsesConfig.async_responses_api_session_handler, + previous_response_id=previous_response_id, + litellm_completion_request=litellm_completion_request, + instructions=bridged_request["instructions"] if "instructions" in bridged_request else None, # noqa: SIM401 # TypedDict indexing preserves the field type + request_kwargs=_REQUEST_KWARGS_ADAPTER.validate_python(kwargs), + previous_response_client_ids=previous_response_client_ids, + input_replaces_history=holds_compaction_summary(with_compaction_items_as_params(bridged_input)), + ) + if previous_response_id + else litellm_completion_request + ) + completion_args: Final = _REQUEST_KWARGS_ADAPTER.validate_python( + {**kwargs, **completion_request, "_skip_responses_api_bridge": True} + ) litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = litellm.completion( - **{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True}, + **completion_args, ) if isinstance(litellm_completion_response, ModelResponse): @@ -224,7 +245,7 @@ class LiteLLMCompletionTransformationHandler: responses_api_request: ResponsesAPIOptionalRequestParams, **kwargs, ) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator: - previous_response_id: Final[str | None] = responses_api_request.get("previous_response_id") + previous_response_id: Final = responses_api_request.get("previous_response_id") if previous_response_id: litellm_completion_request = await LiteLLMCompletionResponsesConfig.async_responses_api_session_handler( previous_response_id=previous_response_id, @@ -236,8 +257,11 @@ class LiteLLMCompletionTransformationHandler: input_replaces_history=holds_compaction_summary(with_compaction_items_as_params(request_input)), ) + acompletion_args: Final = _REQUEST_KWARGS_ADAPTER.validate_python( + {**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True} + ) litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = await litellm.acompletion( - **{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True}, + **acompletion_args, ) if isinstance(litellm_completion_response, ModelResponse): responses_api_response: Final[ResponsesAPIResponse] = ( diff --git a/litellm/responses/litellm_completion_transformation/session_handler.py b/litellm/responses/litellm_completion_transformation/session_handler.py index 7e6ee84e2b2..3b03e043901 100644 --- a/litellm/responses/litellm_completion_transformation/session_handler.py +++ b/litellm/responses/litellm_completion_transformation/session_handler.py @@ -320,6 +320,19 @@ class ResponsesSessionHandler: replayed_turns: Final = ( replayable_turns if _holds_compaction_point(all_spend_logs, proxy_server_requests) else len(all_spend_logs) ) + if not all_spend_logs or any( + request is None or (request.get("input") is None and request.get("messages") is None) + for index, request in enumerate(proxy_server_requests[start.index : replayed_turns], start=start.index) + if not (start.at_output and index == start.index) and not _is_a_response_read(all_spend_logs[index]) + ): + raise litellm.BadRequestError( + message=( + f"Cannot reconstruct conversation history for previous_response_id '{previous_response_id}'. " + "Resend the full conversation without previous_response_id." + ), + model="", + llm_provider="", + ) earlier_instructions: Final = _instruction_items_before(proxy_server_requests, start) replayed_history: Final = ResponsesSessionHandler._history_of_turns( all_spend_logs[start.index : replayed_turns], diff --git a/tests/unit/responses/litellm_completion_transformation/test_handler.py b/tests/unit/responses/litellm_completion_transformation/test_handler.py index 4794f3701c5..1bd8835c1d8 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_handler.py +++ b/tests/unit/responses/litellm_completion_transformation/test_handler.py @@ -13,7 +13,7 @@ kwargs lack the flag and these tests fail. """ import json -from collections.abc import Mapping, Sequence +from collections.abc import Iterator, Mapping, Sequence from types import SimpleNamespace from typing import Final from unittest.mock import patch @@ -22,7 +22,8 @@ import httpx import pytest import litellm -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.proxy import proxy_server +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.responses.litellm_completion_transformation.handler import ( LiteLLMCompletionTransformationHandler, ) @@ -209,15 +210,17 @@ async def test_bridged_follow_up_turn_keeps_the_addressed_response_id_off_the_pr client: Final = AsyncHTTPHandler() client.client = httpx.AsyncClient(transport=httpx.MockTransport(provider)) - response = await litellm.aresponses( - model="azure_ai/claude-sonnet-4-6", - api_base="https://fake-foundry-resource.services.ai.azure.com", - api_key="fake-api-key", - input="Double it", - previous_response_id="resp_turn_one", - client=client, - **{ADDRESSED_RESPONSE_ID_FIELD: "resp_turn_one"}, - ) + database: Final = SimpleNamespace(db=_StoredTurns((_stored_mcp_turn("resp_turn_one"),))) + with patch.object(proxy_server, "prisma_client", database): + response = await litellm.aresponses( + model="azure_ai/claude-sonnet-4-6", + api_base="https://fake-foundry-resource.services.ai.azure.com", + api_key="fake-api-key", + input="Double it", + previous_response_id="resp_turn_one", + client=client, + **{ADDRESSED_RESPONSE_ID_FIELD: "resp_turn_one"}, + ) assert provider.request_body is not None, "the bridged turn never reached the provider" assert ADDRESSED_RESPONSE_ID_FIELD not in provider.request_body, ( @@ -323,15 +326,148 @@ async def test_bridged_next_turn_replays_a_turn_stored_under_the_id_the_client_h ) assert provider.request_body is not None, "the bridged turn never reached the provider" - assert [ - (message["role"], json.dumps(message["content"])) for message in provider.request_body["messages"] - ] == [ + assert [(message["role"], json.dumps(message["content"])) for message in provider.request_body["messages"]] == [ ("user", json.dumps([{"type": "text", "text": "My favorite number is 7."}])), ("assistant", json.dumps([{"type": "text", "text": "Noted."}])), ("user", json.dumps([{"type": "text", "text": "Double it"}])), ] +_CHAT_COMPLETION_PAYLOAD: Final = { + "id": "chatcmpl-current-turn", + "object": "chat.completion", + "created": 1, + "model": "bridge-test", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "14"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 1, "total_tokens": 11}, +} + + +@pytest.fixture(params=["no_database", "unknown_response", "unstored_input"]) +def unavailable_response_history(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + rows: Final = ( + ({**_stored_mcp_turn("resp_missing_history"), "proxy_server_request": {}},) + if request.param == "unstored_input" + else () + ) + database: Final = None if request.param == "no_database" else SimpleNamespace(db=_StoredTurns(rows)) + monkeypatch.setattr(litellm.constants, "RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL", 0.0) + with patch.object(proxy_server, "prisma_client", database): + yield + + +@pytest.mark.parametrize("stream", [False, True]) +def test_sync_bridge_rejects_unavailable_previous_response_before_inference( + unavailable_response_history: None, stream: bool +): + provider: Final = _RecordingAnthropicHandler(_CHAT_COMPLETION_PAYLOAD) + with httpx.Client(transport=httpx.MockTransport(provider)) as http_client: + with pytest.raises(litellm.BadRequestError, match=r"previous_response_id.*resp_missing_history") as error: + litellm.responses( + model="hosted_vllm/bridge-test", + api_base="https://provider.example/v1", + api_key="test-key", + input="Double it", + previous_response_id="resp_missing_history", + use_chat_completions_api=True, + stream=stream, + client=HTTPHandler(client=http_client), + ) + + assert error.value.status_code == 400 + assert provider.request_body is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +async def test_async_bridge_rejects_unavailable_previous_response_before_inference( + unavailable_response_history: None, stream: bool +): + provider: Final = _RecordingAnthropicHandler(_CHAT_COMPLETION_PAYLOAD) + client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(provider)) + try: + with pytest.raises(litellm.BadRequestError, match=r"previous_response_id.*resp_missing_history") as error: + await litellm.aresponses( + model="hosted_vllm/bridge-test", + api_base="https://provider.example/v1", + api_key="test-key", + input="Double it", + previous_response_id="resp_missing_history", + use_chat_completions_api=True, + stream=stream, + client=client, + ) + finally: + await client.close() + + assert error.value.status_code == 400 + assert provider.request_body is None + + +@pytest.mark.parametrize("previous_response_id", [None, "resp_stored"]) +def test_sync_bridge_preserves_available_history_and_first_turns(previous_response_id: str | None): + provider: Final = _RecordingAnthropicHandler(_CHAT_COMPLETION_PAYLOAD) + database: Final = SimpleNamespace(db=_StoredTurns((_stored_mcp_turn("resp_stored"),))) + with httpx.Client(transport=httpx.MockTransport(provider)) as http_client: + with patch.object(proxy_server, "prisma_client", database): + response = litellm.responses( + model="hosted_vllm/bridge-test", + api_base="https://provider.example/v1", + api_key="test-key", + input="Double it", + previous_response_id=previous_response_id, + use_chat_completions_api=True, + client=HTTPHandler(client=http_client), + ) + + assert isinstance(response, ResponsesAPIResponse) + assert response.status == "completed" + assert provider.request_body is not None + assert provider.request_body["messages"] == ( + [ + {"role": "user", "content": "My favorite number is 7."}, + {"role": "assistant", "content": "Noted."}, + {"role": "user", "content": "Double it"}, + ] + if previous_response_id + else [{"role": "user", "content": "Double it"}] + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("previous_response_id", [None, "resp_stored"]) +async def test_async_bridge_preserves_available_history_and_first_turns(previous_response_id: str | None): + provider: Final = _RecordingAnthropicHandler(_CHAT_COMPLETION_PAYLOAD) + database: Final = SimpleNamespace(db=_StoredTurns((_stored_mcp_turn("resp_stored"),))) + client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(provider)) + try: + with patch.object(proxy_server, "prisma_client", database): + response = await litellm.aresponses( + model="hosted_vllm/bridge-test", + api_base="https://provider.example/v1", + api_key="test-key", + input="Double it", + previous_response_id=previous_response_id, + use_chat_completions_api=True, + client=client, + ) + finally: + await client.close() + + assert isinstance(response, ResponsesAPIResponse) + assert response.status == "completed" + assert provider.request_body is not None + assert provider.request_body["messages"] == ( + [ + {"role": "user", "content": "My favorite number is 7."}, + {"role": "assistant", "content": "Noted."}, + {"role": "user", "content": "Double it"}, + ] + if previous_response_id + else [{"role": "user", "content": "Double it"}] + ) + + @pytest.mark.asyncio @pytest.mark.parametrize("client_field", ["caller_input", "names_non_string_fields"]) async def test_bridged_follow_up_after_a_client_tool_search_with_a_client_field_named_like_a_bridge_keyword_reaches_the_provider( diff --git a/tests/unit/responses/litellm_completion_transformation/test_session_handler.py b/tests/unit/responses/litellm_completion_transformation/test_session_handler.py index 9a8fa19e2e4..cc05345d963 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_session_handler.py +++ b/tests/unit/responses/litellm_completion_transformation/test_session_handler.py @@ -13,6 +13,7 @@ from fastapi import HTTPException from fastapi.testclient import TestClient import litellm +from litellm.proxy import proxy_server from litellm.proxy.spend_tracking.spend_tracking_utils import ( _get_proxy_server_request_for_spend_logs_payload, ) @@ -172,25 +173,32 @@ async def test_get_chat_completion_message_history_for_previous_response_id(): @pytest.mark.asyncio -async def test_get_chat_completion_message_history_empty_spend_logs(): - """ - Test get_chat_completion_message_history_for_previous_response_id with empty spend logs - """ - with patch.object( - ResponsesSessionHandler, - "get_all_spend_logs_for_previous_response_id", - new_callable=AsyncMock, - ) as mock_get_spend_logs: - mock_get_spend_logs.return_value = [] +async def test_get_chat_completion_message_history_empty_spend_logs(instant_session_lookup_retries: None): + with patch.object(proxy_server, "prisma_client", _FakePrismaClient(results=[])): + with pytest.raises(litellm.BadRequestError, match=r"previous_response_id.*non-existent-id") as error: + await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id( + "non-existent-id" + ) - previous_response_id = "non-existent-id" - result = await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id( - previous_response_id + assert error.value.status_code == 400 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stored_request", [None, "{}", "null", {}, {"model": "bridge-test"}]) +async def test_message_history_rejects_a_stored_turn_without_its_input(stored_request: object): + stored: Final = _StoredSpendLogs( + ( + { + **_spend_log("resp_missing_input", "session-a", "Remember 7", "Noted"), + "proxy_server_request": stored_request, + }, ) - - # Verify empty result structure - assert result.get("messages") == [] - assert result.get("litellm_session_id") is None + ) + with patch.object(proxy_server, "prisma_client", SimpleNamespace(db=stored)): + with pytest.raises(litellm.BadRequestError, match=r"previous_response_id.*resp_missing_input"): + await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id( + "resp_missing_input" + ) @pytest.mark.asyncio @@ -1162,7 +1170,10 @@ async def test_an_item_a_poll_caught_half_written_replays_its_finished_text_once "resp_poll_done", "aget_responses", "completed", - [_identified_output_message("msg_answer", "One, two, three"), _identified_output_message("msg_more", "Done")], + [ + _identified_output_message("msg_answer", "One, two, three"), + _identified_output_message("msg_more", "Done"), + ], ), )