diff --git a/litellm/responses/litellm_completion_transformation/handler.py b/litellm/responses/litellm_completion_transformation/handler.py index c9d800421f7..a59907d378d 100644 --- a/litellm/responses/litellm_completion_transformation/handler.py +++ b/litellm/responses/litellm_completion_transformation/handler.py @@ -8,6 +8,7 @@ from typing import Final from pydantic import TypeAdapter import litellm +from litellm.litellm_core_utils.asyncify import run_async_function from litellm.responses.additional_tools import hoist_additional_tools from litellm.responses.litellm_completion_transformation.compaction_items import holds_compaction_summary from litellm.responses.litellm_completion_transformation.streaming_iterator import ( @@ -52,7 +53,7 @@ class LiteLLMCompletionTransformationHandler: bridged_request: Final[ResponsesAPIOptionalRequestParams] = ( {**responses_api_request, "tools": list(hoisted.tools)} if hoisted.hoisted else responses_api_request ) - litellm_completion_request: Final[dict] = ( + litellm_completion_request: Final = _REQUEST_KWARGS_ADAPTER.validate_python( LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( model=model, input=bridged_input, @@ -73,10 +74,23 @@ class LiteLLMCompletionTransformationHandler: **kwargs, ) - completion_args: Final = {} - completion_args.update(kwargs) - completion_args.update(litellm_completion_request) - completion_args["_skip_responses_api_bridge"] = True + previous_response_id: Final = bridged_request.get("previous_response_id") + completion_request: Final = _REQUEST_KWARGS_ADAPTER.validate_python( + run_async_function( + LiteLLMCompletionResponsesConfig.async_responses_api_session_handler, + previous_response_id=previous_response_id, + litellm_completion_request=litellm_completion_request, + instructions=bridged_request.get("instructions"), + request_kwargs=_REQUEST_KWARGS_ADAPTER.validate_python(kwargs), + previous_response_client_ids=previous_response_client_ids, + input_replaces_history=holds_compaction_summary(with_compaction_items_as_params(bridged_input)), + ) + if previous_response_id + else litellm_completion_request + ) + completion_args: Final = _REQUEST_KWARGS_ADAPTER.validate_python( + {**kwargs, **completion_request, "_skip_responses_api_bridge": True} + ) litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = litellm.completion( **completion_args, @@ -114,7 +128,7 @@ class LiteLLMCompletionTransformationHandler: responses_api_request: ResponsesAPIOptionalRequestParams, **kwargs, ) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator: - previous_response_id: Final[str | None] = responses_api_request.get("previous_response_id") + previous_response_id: Final = responses_api_request.get("previous_response_id") if previous_response_id: litellm_completion_request = await LiteLLMCompletionResponsesConfig.async_responses_api_session_handler( previous_response_id=previous_response_id, @@ -125,10 +139,9 @@ class LiteLLMCompletionTransformationHandler: input_replaces_history=holds_compaction_summary(with_compaction_items_as_params(request_input)), ) - acompletion_args: Final = {} - acompletion_args.update(kwargs) - acompletion_args.update(litellm_completion_request) - acompletion_args["_skip_responses_api_bridge"] = True + acompletion_args: Final = _REQUEST_KWARGS_ADAPTER.validate_python( + {**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True} + ) litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = await litellm.acompletion( **acompletion_args, diff --git a/litellm/responses/litellm_completion_transformation/session_handler.py b/litellm/responses/litellm_completion_transformation/session_handler.py index d50e9a61edc..4c660732184 100644 --- a/litellm/responses/litellm_completion_transformation/session_handler.py +++ b/litellm/responses/litellm_completion_transformation/session_handler.py @@ -217,6 +217,19 @@ class ResponsesSessionHandler: replayed_turns: Final = ( replayable_turns if _holds_compaction_point(all_spend_logs, proxy_server_requests) else len(all_spend_logs) ) + if not all_spend_logs or any( + request is None or (request.get("input") is None and request.get("messages") is None) + for index, request in enumerate(proxy_server_requests[start.index : replayed_turns], start=start.index) + if not (start.at_output and index == start.index) + ): + raise litellm.BadRequestError( + message=( + f"Cannot reconstruct conversation history for previous_response_id '{previous_response_id}'. " + "Resend the full conversation without previous_response_id." + ), + model="", + llm_provider="", + ) earlier_instructions: Final = _instruction_items_before(proxy_server_requests, start) replayed_history: Final = ResponsesSessionHandler._history_of_turns( all_spend_logs[start.index : replayed_turns], diff --git a/tests/unit/responses/litellm_completion_transformation/test_handler.py b/tests/unit/responses/litellm_completion_transformation/test_handler.py index d731bf5f41c..98a7562007e 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_handler.py +++ b/tests/unit/responses/litellm_completion_transformation/test_handler.py @@ -13,7 +13,7 @@ kwargs lack the flag and these tests fail. """ import json -from collections.abc import Mapping, Sequence +from collections.abc import Iterator, Mapping, Sequence from types import SimpleNamespace from typing import Final from unittest.mock import patch @@ -22,7 +22,8 @@ import httpx import pytest import litellm -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.proxy import proxy_server +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.responses.litellm_completion_transformation.handler import ( LiteLLMCompletionTransformationHandler, ) @@ -209,15 +210,17 @@ async def test_bridged_follow_up_turn_keeps_the_addressed_response_id_off_the_pr client: Final = AsyncHTTPHandler() client.client = httpx.AsyncClient(transport=httpx.MockTransport(provider)) - response = await litellm.aresponses( - model="azure_ai/claude-sonnet-4-6", - api_base="https://fake-foundry-resource.services.ai.azure.com", - api_key="fake-api-key", - input="Double it", - previous_response_id="resp_turn_one", - client=client, - **{ADDRESSED_RESPONSE_ID_FIELD: "resp_turn_one"}, - ) + database: Final = SimpleNamespace(db=_StoredTurns((_stored_mcp_turn("resp_turn_one"),))) + with patch.object(proxy_server, "prisma_client", database): + response = await litellm.aresponses( + model="azure_ai/claude-sonnet-4-6", + api_base="https://fake-foundry-resource.services.ai.azure.com", + api_key="fake-api-key", + input="Double it", + previous_response_id="resp_turn_one", + client=client, + **{ADDRESSED_RESPONSE_ID_FIELD: "resp_turn_one"}, + ) assert provider.request_body is not None, "the bridged turn never reached the provider" assert ADDRESSED_RESPONSE_ID_FIELD not in provider.request_body, ( @@ -303,10 +306,143 @@ async def test_bridged_next_turn_replays_a_turn_stored_under_the_id_the_client_h ) assert provider.request_body is not None, "the bridged turn never reached the provider" - assert [ - (message["role"], json.dumps(message["content"])) for message in provider.request_body["messages"] - ] == [ + assert [(message["role"], json.dumps(message["content"])) for message in provider.request_body["messages"]] == [ ("user", json.dumps([{"type": "text", "text": "My favorite number is 7."}])), ("assistant", json.dumps([{"type": "text", "text": "Noted."}])), ("user", json.dumps([{"type": "text", "text": "Double it"}])), ] + + +_CHAT_COMPLETION_PAYLOAD: Final = { + "id": "chatcmpl-current-turn", + "object": "chat.completion", + "created": 1, + "model": "bridge-test", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "14"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 1, "total_tokens": 11}, +} + + +@pytest.fixture(params=["no_database", "unknown_response", "unstored_input"]) +def unavailable_response_history(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + rows: Final = ( + ({**_stored_mcp_turn("resp_missing_history"), "proxy_server_request": {}},) + if request.param == "unstored_input" + else () + ) + database: Final = None if request.param == "no_database" else SimpleNamespace(db=_StoredTurns(rows)) + monkeypatch.setattr(litellm.constants, "RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL", 0.0) + with patch.object(proxy_server, "prisma_client", database): + yield + + +@pytest.mark.parametrize("stream", [False, True]) +def test_sync_bridge_rejects_unavailable_previous_response_before_inference( + unavailable_response_history: None, stream: bool +): + provider: Final = _RecordingAnthropicHandler(_CHAT_COMPLETION_PAYLOAD) + with httpx.Client(transport=httpx.MockTransport(provider)) as http_client: + with pytest.raises(litellm.BadRequestError, match=r"previous_response_id.*resp_missing_history") as error: + litellm.responses( + model="hosted_vllm/bridge-test", + api_base="https://provider.example/v1", + api_key="test-key", + input="Double it", + previous_response_id="resp_missing_history", + use_chat_completions_api=True, + stream=stream, + client=HTTPHandler(client=http_client), + ) + + assert error.value.status_code == 400 + assert provider.request_body is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +async def test_async_bridge_rejects_unavailable_previous_response_before_inference( + unavailable_response_history: None, stream: bool +): + provider: Final = _RecordingAnthropicHandler(_CHAT_COMPLETION_PAYLOAD) + client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(provider)) + try: + with pytest.raises(litellm.BadRequestError, match=r"previous_response_id.*resp_missing_history") as error: + await litellm.aresponses( + model="hosted_vllm/bridge-test", + api_base="https://provider.example/v1", + api_key="test-key", + input="Double it", + previous_response_id="resp_missing_history", + use_chat_completions_api=True, + stream=stream, + client=client, + ) + finally: + await client.close() + + assert error.value.status_code == 400 + assert provider.request_body is None + + +@pytest.mark.parametrize("previous_response_id", [None, "resp_stored"]) +def test_sync_bridge_preserves_available_history_and_first_turns(previous_response_id: str | None): + provider: Final = _RecordingAnthropicHandler(_CHAT_COMPLETION_PAYLOAD) + database: Final = SimpleNamespace(db=_StoredTurns((_stored_mcp_turn("resp_stored"),))) + with httpx.Client(transport=httpx.MockTransport(provider)) as http_client: + with patch.object(proxy_server, "prisma_client", database): + response = litellm.responses( + model="hosted_vllm/bridge-test", + api_base="https://provider.example/v1", + api_key="test-key", + input="Double it", + previous_response_id=previous_response_id, + use_chat_completions_api=True, + client=HTTPHandler(client=http_client), + ) + + assert isinstance(response, ResponsesAPIResponse) + assert response.status == "completed" + assert provider.request_body is not None + assert provider.request_body["messages"] == ( + [ + {"role": "user", "content": "My favorite number is 7."}, + {"role": "assistant", "content": "Noted."}, + {"role": "user", "content": "Double it"}, + ] + if previous_response_id + else [{"role": "user", "content": "Double it"}] + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("previous_response_id", [None, "resp_stored"]) +async def test_async_bridge_preserves_available_history_and_first_turns(previous_response_id: str | None): + provider: Final = _RecordingAnthropicHandler(_CHAT_COMPLETION_PAYLOAD) + database: Final = SimpleNamespace(db=_StoredTurns((_stored_mcp_turn("resp_stored"),))) + client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(provider)) + try: + with patch.object(proxy_server, "prisma_client", database): + response = await litellm.aresponses( + model="hosted_vllm/bridge-test", + api_base="https://provider.example/v1", + api_key="test-key", + input="Double it", + previous_response_id=previous_response_id, + use_chat_completions_api=True, + client=client, + ) + finally: + await client.close() + + assert isinstance(response, ResponsesAPIResponse) + assert response.status == "completed" + assert provider.request_body is not None + assert provider.request_body["messages"] == ( + [ + {"role": "user", "content": "My favorite number is 7."}, + {"role": "assistant", "content": "Noted."}, + {"role": "user", "content": "Double it"}, + ] + if previous_response_id + else [{"role": "user", "content": "Double it"}] + ) diff --git a/tests/unit/responses/litellm_completion_transformation/test_session_handler.py b/tests/unit/responses/litellm_completion_transformation/test_session_handler.py index 0eb1aa26a65..ce4277a5e9c 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_session_handler.py +++ b/tests/unit/responses/litellm_completion_transformation/test_session_handler.py @@ -11,6 +11,7 @@ from fastapi import HTTPException from fastapi.testclient import TestClient import litellm +from litellm.proxy import proxy_server from litellm.proxy.spend_tracking.spend_tracking_utils import ( _get_proxy_server_request_for_spend_logs_payload, ) @@ -170,25 +171,32 @@ async def test_get_chat_completion_message_history_for_previous_response_id(): @pytest.mark.asyncio -async def test_get_chat_completion_message_history_empty_spend_logs(): - """ - Test get_chat_completion_message_history_for_previous_response_id with empty spend logs - """ - with patch.object( - ResponsesSessionHandler, - "get_all_spend_logs_for_previous_response_id", - new_callable=AsyncMock, - ) as mock_get_spend_logs: - mock_get_spend_logs.return_value = [] +async def test_get_chat_completion_message_history_empty_spend_logs(instant_session_lookup_retries: None): + with patch.object(proxy_server, "prisma_client", _FakePrismaClient(results=[])): + with pytest.raises(litellm.BadRequestError, match=r"previous_response_id.*non-existent-id") as error: + await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id( + "non-existent-id" + ) - previous_response_id = "non-existent-id" - result = await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id( - previous_response_id + assert error.value.status_code == 400 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stored_request", [None, "{}", "null", {}, {"model": "bridge-test"}]) +async def test_message_history_rejects_a_stored_turn_without_its_input(stored_request: object): + stored: Final = _StoredSpendLogs( + ( + { + **_spend_log("resp_missing_input", "session-a", "Remember 7", "Noted"), + "proxy_server_request": stored_request, + }, ) - - # Verify empty result structure - assert result.get("messages") == [] - assert result.get("litellm_session_id") is None + ) + with patch.object(proxy_server, "prisma_client", SimpleNamespace(db=stored)): + with pytest.raises(litellm.BadRequestError, match=r"previous_response_id.*resp_missing_input"): + await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id( + "resp_missing_input" + ) @pytest.mark.asyncio