From 2b34af521c28b011bcf54d34b67e9cd856aef1fe Mon Sep 17 00:00:00 2001 From: Dev Chiniwala Date: Mon, 26 Oct 2026 17:40:03 +0530 Subject: [PATCH 1/3] fix(responses)!: reject unavailable continuation history --- .../handler.py | 19 ++- .../transformation.py | 10 +- .../test_handler.py | 119 ++++++++++++++---- .../test_litellm_completion_responses.py | 34 +++-- 4 files changed, 142 insertions(+), 40 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/handler.py b/litellm/responses/litellm_completion_transformation/handler.py index c9d800421f7..2c8c8e0e29b 100644 --- a/litellm/responses/litellm_completion_transformation/handler.py +++ b/litellm/responses/litellm_completion_transformation/handler.py @@ -8,6 +8,7 @@ from typing import Final from pydantic import TypeAdapter import litellm +from litellm.litellm_core_utils.asyncify import run_async_function from litellm.responses.additional_tools import hoist_additional_tools from litellm.responses.litellm_completion_transformation.compaction_items import holds_compaction_summary from litellm.responses.litellm_completion_transformation.streaming_iterator import ( @@ -73,9 +74,25 @@ class LiteLLMCompletionTransformationHandler: **kwargs, ) + previous_response_id: Final = bridged_request.get("previous_response_id") + sync_request: Final = ( + _REQUEST_KWARGS_ADAPTER.validate_python( + run_async_function( + LiteLLMCompletionResponsesConfig.async_responses_api_session_handler, + previous_response_id=previous_response_id, + litellm_completion_request=litellm_completion_request, + instructions=bridged_request.get("instructions"), + request_kwargs=_REQUEST_KWARGS_ADAPTER.validate_python(kwargs), + previous_response_client_ids=previous_response_client_ids, + input_replaces_history=holds_compaction_summary(with_compaction_items_as_params(bridged_input)), + ) + ) + if previous_response_id + else litellm_completion_request + ) completion_args: Final = {} completion_args.update(kwargs) - completion_args.update(litellm_completion_request) + completion_args.update(sync_request) completion_args["_skip_responses_api_bridge"] = True litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = litellm.completion( diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index a9c7818b577..0225be6b8ff 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -1223,6 +1223,14 @@ class LiteLLMCompletionResponsesConfig: _messages: Final = litellm_completion_request.get("messages") or [] route: Final = CompletionRoute(model=model, custom_llm_provider=provider) session_history: Final = chat_completion_session.get("messages") or [] + if previous_response_id and not session_history: + raise litellm.BadRequestError( + message="No stored conversation history is available for previous_response_id. " + "Start a new conversation without previous_response_id and include the full history.", + model=model, + llm_provider=provider or "", + body={"param": "previous_response_id"}, + ) stored_messages: Final = LiteLLMCompletionResponsesConfig._with_compaction_blocks_the_route_takes( _instruction_messages(session_history) if input_replaces_history else session_history, route, @@ -1255,8 +1263,6 @@ class LiteLLMCompletionResponsesConfig: else _messages[:instructions_end] ) - # If session messages are empty (e.g., no database in test environment), - # we still need to process the new input messages # Store original _messages before combining for safety check original_new_messages: Final = _messages.copy() if _messages else [] diff --git a/tests/unit/responses/litellm_completion_transformation/test_handler.py b/tests/unit/responses/litellm_completion_transformation/test_handler.py index d731bf5f41c..9ffd4939903 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_handler.py +++ b/tests/unit/responses/litellm_completion_transformation/test_handler.py @@ -22,7 +22,7 @@ import httpx import pytest import litellm -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.responses.litellm_completion_transformation.handler import ( LiteLLMCompletionTransformationHandler, ) @@ -209,15 +209,17 @@ async def test_bridged_follow_up_turn_keeps_the_addressed_response_id_off_the_pr client: Final = AsyncHTTPHandler() client.client = httpx.AsyncClient(transport=httpx.MockTransport(provider)) - response = await litellm.aresponses( - model="azure_ai/claude-sonnet-4-6", - api_base="https://fake-foundry-resource.services.ai.azure.com", - api_key="fake-api-key", - input="Double it", - previous_response_id="resp_turn_one", - client=client, - **{ADDRESSED_RESPONSE_ID_FIELD: "resp_turn_one"}, - ) + stored_turns: Final = _StoredTurns((_stored_mcp_turn("resp_turn_one"),)) + with patch("litellm.proxy.proxy_server.prisma_client", SimpleNamespace(db=stored_turns)): + response = await litellm.aresponses( + model="azure_ai/claude-sonnet-4-6", + api_base="https://fake-foundry-resource.services.ai.azure.com", + api_key="fake-api-key", + input="Double it", + previous_response_id="resp_turn_one", + client=client, + **{ADDRESSED_RESPONSE_ID_FIELD: "resp_turn_one"}, + ) assert provider.request_body is not None, "the bridged turn never reached the provider" assert ADDRESSED_RESPONSE_ID_FIELD not in provider.request_body, ( @@ -255,6 +257,59 @@ class _StoredTurns: return [dict(row) for row in self.rows if row["session_id"] in sessions] +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize("async_call", [False, True]) +@pytest.mark.asyncio +async def test_unknown_previous_response_fails_before_sending_a_contextless_completion( + async_call: bool, stream: bool +) -> None: + provider: Final = _RecordingAnthropicHandler( + { + "id": "chatcmpl-local", + "object": "chat.completion", + "created": 0, + "model": "test-model", + "choices": [ + {"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "Lost context"}} + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + ) + transport: Final = httpx.MockTransport(provider) + async_client: Final = AsyncHTTPHandler(transport=transport) + with httpx.Client(transport=transport) as sync_client: + try: + with pytest.raises(litellm.BadRequestError, match="previous_response_id") as raised: + if async_call: + await litellm.aresponses( + model="hosted_vllm/test-model", + input="What is my favorite color?", + previous_response_id="resp_does_not_exist", + stream=stream, + use_chat_completions_api=True, + client=async_client, + api_key="test", + api_base="https://provider.invalid/v1", + num_retries=0, + ) + else: + litellm.responses( + model="hosted_vllm/test-model", + input="What is my favorite color?", + previous_response_id="resp_does_not_exist", + stream=stream, + use_chat_completions_api=True, + client=HTTPHandler(client=sync_client), + api_key="test", + api_base="https://provider.invalid/v1", + num_retries=0, + ) + assert raised.value.status_code == 400 + assert provider.request_body is None, "unresolved history must not send the current turn to the provider" + finally: + await async_client.client.aclose() + + def _stored_mcp_turn(request_id: str) -> Mapping[str, object]: return { "request_id": request_id, @@ -279,7 +334,10 @@ def _stored_mcp_turn(request_id: str) -> Mapping[str, object]: @pytest.mark.asyncio @pytest.mark.parametrize("stored_under", ["litellm_encoded_id", "proxy_issued_id"]) -async def test_bridged_next_turn_replays_a_turn_stored_under_the_id_the_client_holds(stored_under: str): +@pytest.mark.parametrize("async_call", [False, True]) +async def test_bridged_next_turn_replays_a_turn_stored_under_the_id_the_client_holds( + stored_under: str, async_call: bool +): litellm_encoded_id: Final = ResponsesAPIRequestUtils._build_responses_api_response_id( custom_llm_provider="azure_ai", model_id="deployment-a", response_id="chatcmpl-turn-one" ) @@ -291,21 +349,34 @@ async def test_bridged_next_turn_replays_a_turn_stored_under_the_id_the_client_h client: Final = AsyncHTTPHandler() client.client = httpx.AsyncClient(transport=httpx.MockTransport(provider)) - with patch("litellm.proxy.proxy_server.prisma_client", SimpleNamespace(db=stored_turns)): - await litellm.aresponses( - model="azure_ai/claude-sonnet-4-6", - api_base="https://fake-foundry-resource.services.ai.azure.com", - api_key="fake-api-key", - input="Double it", - previous_response_id=litellm_encoded_id, - client=client, - **{ADDRESSED_RESPONSE_ID_FIELD: proxy_issued_id}, - ) + with httpx.Client(transport=httpx.MockTransport(provider)) as sync_client: + try: + with patch("litellm.proxy.proxy_server.prisma_client", SimpleNamespace(db=stored_turns)): + if async_call: + await litellm.aresponses( + model="azure_ai/claude-sonnet-4-6", + api_base="https://fake-foundry-resource.services.ai.azure.com", + api_key="fake-api-key", + input="Double it", + previous_response_id=litellm_encoded_id, + client=client, + **{ADDRESSED_RESPONSE_ID_FIELD: proxy_issued_id}, + ) + else: + litellm.responses( + model="azure_ai/claude-sonnet-4-6", + api_base="https://fake-foundry-resource.services.ai.azure.com", + api_key="fake-api-key", + input="Double it", + previous_response_id=litellm_encoded_id, + client=HTTPHandler(client=sync_client), + **{ADDRESSED_RESPONSE_ID_FIELD: proxy_issued_id}, + ) + finally: + await client.client.aclose() assert provider.request_body is not None, "the bridged turn never reached the provider" - assert [ - (message["role"], json.dumps(message["content"])) for message in provider.request_body["messages"] - ] == [ + assert [(message["role"], json.dumps(message["content"])) for message in provider.request_body["messages"]] == [ ("user", json.dumps([{"type": "text", "text": "My favorite number is 7."}])), ("assistant", json.dumps([{"type": "text", "text": "Noted."}])), ("user", json.dumps([{"type": "text", "text": "Double it"}])), diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 023d0ea26fb..ebca649787d 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -6668,7 +6668,6 @@ def test_summary_replay_drops_the_history_its_compaction_item_replaces() -> None ] - @pytest.mark.parametrize( ("call", "output"), [ @@ -6781,7 +6780,7 @@ def test_native_compaction_replay_moves_a_call_its_compaction_item_splits_from_i pytest.param(ResponsesAPIOptionalRequestParams(), id="summary_as_text"), ], ) -def test_compaction_replay_invents_no_call_for_an_output_whose_call_was_never_stored( +def test_compaction_replay_rejects_missing_history_instead_of_inventing_tool_calls( responses_api_request: ResponsesAPIOptionalRequestParams, ) -> None: async def no_stored_turns(previous_response_id: str) -> ChatCompletionSession: @@ -6802,16 +6801,16 @@ def test_compaction_replay_invents_no_call_for_an_output_whose_call_was_never_st ), ) - request: Final = asyncio.run( - LiteLLMCompletionResponsesConfig.async_responses_api_session_handler( - previous_response_id="resp_unknown", - litellm_completion_request=chat_request, - instructions=None, - session_lookup=no_stored_turns, + with pytest.raises(litellm.BadRequestError, match="previous_response_id") as raised: + asyncio.run( + LiteLLMCompletionResponsesConfig.async_responses_api_session_handler( + previous_response_id="resp_unknown", + litellm_completion_request=chat_request, + instructions=None, + session_lookup=no_stored_turns, + ) ) - ) - - assert not any(message.get("tool_calls") for message in request["messages"]) + assert raised.value.status_code == 400 @pytest.mark.parametrize( @@ -6819,7 +6818,14 @@ def test_compaction_replay_invents_no_call_for_an_output_whose_call_was_never_st [ pytest.param( _COMPACTION_REQUESTED, - [("user", ()), ("assistant", ("call_0",)), ("tool", "Sunny."), ("assistant", ()), ("assistant", ("call_0",)), ("tool", "Rain.")], + [ + ("user", ()), + ("assistant", ("call_0",)), + ("tool", "Sunny."), + ("assistant", ()), + ("assistant", ("call_0",)), + ("tool", "Rain."), + ], id="native_compaction_block", ), pytest.param( @@ -6961,7 +6967,9 @@ async def test_bridge_partial_response_on_disconnect_bills_the_relayed_chat_chun assert incomplete_event.type == "response.incomplete" assert incomplete_event.response.status == "incomplete" assert incomplete_event.response.usage is not None - assert incomplete_event.response.usage.output_tokens == litellm.token_counter(model="gpt-4o-mini", text=relayed_text) + assert incomplete_event.response.usage.output_tokens == litellm.token_counter( + model="gpt-4o-mini", text=relayed_text + ) assert incomplete_event.response.usage.input_tokens > 0 From a6ca96710582b40da539653222a77f01481ee2e9 Mon Sep 17 00:00:00 2001 From: Dev Chiniwala Date: Mon, 26 Oct 2026 17:49:29 +0530 Subject: [PATCH 2/3] test(responses): keep exception assertions to one call --- .../litellm_completion_transformation/test_handler.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/unit/responses/litellm_completion_transformation/test_handler.py b/tests/unit/responses/litellm_completion_transformation/test_handler.py index 9ffd4939903..53a0cd68ea1 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_handler.py +++ b/tests/unit/responses/litellm_completion_transformation/test_handler.py @@ -279,8 +279,8 @@ async def test_unknown_previous_response_fails_before_sending_a_contextless_comp async_client: Final = AsyncHTTPHandler(transport=transport) with httpx.Client(transport=transport) as sync_client: try: - with pytest.raises(litellm.BadRequestError, match="previous_response_id") as raised: - if async_call: + if async_call: + with pytest.raises(litellm.BadRequestError, match="previous_response_id") as raised: await litellm.aresponses( model="hosted_vllm/test-model", input="What is my favorite color?", @@ -292,7 +292,8 @@ async def test_unknown_previous_response_fails_before_sending_a_contextless_comp api_base="https://provider.invalid/v1", num_retries=0, ) - else: + else: + with pytest.raises(litellm.BadRequestError, match="previous_response_id") as raised: litellm.responses( model="hosted_vllm/test-model", input="What is my favorite color?", From 1261885972ea5f50938c7cad8a7fd88a5350d048 Mon Sep 17 00:00:00 2001 From: Dev Chiniwala Date: Mon, 26 Oct 2026 18:16:16 +0530 Subject: [PATCH 3/3] fix(responses): validate continuation fields at the sync boundary --- .../handler.py | 40 +++++++++++-------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/handler.py b/litellm/responses/litellm_completion_transformation/handler.py index 2c8c8e0e29b..751ccb755f3 100644 --- a/litellm/responses/litellm_completion_transformation/handler.py +++ b/litellm/responses/litellm_completion_transformation/handler.py @@ -6,6 +6,7 @@ from collections.abc import Coroutine, Mapping from typing import Final from pydantic import TypeAdapter +from typing_extensions import ReadOnly, TypedDict import litellm from litellm.litellm_core_utils.asyncify import run_async_function @@ -29,6 +30,14 @@ from litellm.types.utils import ModelResponse _REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(dict[str, object]) +class _ContinuationFields(TypedDict, total=False): + previous_response_id: ReadOnly[str | None] + instructions: ReadOnly[str | None] + + +_CONTINUATION_FIELDS_ADAPTER: Final = TypeAdapter(_ContinuationFields) + + class LiteLLMCompletionTransformationHandler: def response_api_handler( self, @@ -42,7 +51,7 @@ class LiteLLMCompletionTransformationHandler: _is_async: bool = False, stream: bool | None = None, extra_headers: Mapping[str, object] | None = None, - **kwargs, + **kwargs: object, ) -> ( ResponsesAPIResponse | BaseResponsesAPIStreamingIterator @@ -53,7 +62,7 @@ class LiteLLMCompletionTransformationHandler: bridged_request: Final[ResponsesAPIOptionalRequestParams] = ( {**responses_api_request, "tools": list(hoisted.tools)} if hoisted.hoisted else responses_api_request ) - litellm_completion_request: Final[dict] = ( + litellm_completion_request: Final = _REQUEST_KWARGS_ADAPTER.validate_python( LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( model=model, input=bridged_input, @@ -74,15 +83,16 @@ class LiteLLMCompletionTransformationHandler: **kwargs, ) - previous_response_id: Final = bridged_request.get("previous_response_id") + continuation: Final = _CONTINUATION_FIELDS_ADAPTER.validate_python(bridged_request) + previous_response_id: Final = continuation.get("previous_response_id") sync_request: Final = ( _REQUEST_KWARGS_ADAPTER.validate_python( run_async_function( LiteLLMCompletionResponsesConfig.async_responses_api_session_handler, previous_response_id=previous_response_id, litellm_completion_request=litellm_completion_request, - instructions=bridged_request.get("instructions"), - request_kwargs=_REQUEST_KWARGS_ADAPTER.validate_python(kwargs), + instructions=continuation.get("instructions"), + request_kwargs=kwargs, previous_response_client_ids=previous_response_client_ids, input_replaces_history=holds_compaction_summary(with_compaction_items_as_params(bridged_input)), ) @@ -90,10 +100,7 @@ class LiteLLMCompletionTransformationHandler: if previous_response_id else litellm_completion_request ) - completion_args: Final = {} - completion_args.update(kwargs) - completion_args.update(sync_request) - completion_args["_skip_responses_api_bridge"] = True + completion_args: Final = {**kwargs, **sync_request, "_skip_responses_api_bridge": True} litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = litellm.completion( **completion_args, @@ -117,7 +124,7 @@ class LiteLLMCompletionTransformationHandler: request_input=bridged_input, responses_api_request=bridged_request, custom_llm_provider=custom_llm_provider, - litellm_metadata=kwargs.get("litellm_metadata", {}), + litellm_metadata=_REQUEST_KWARGS_ADAPTER.validate_python(kwargs.get("litellm_metadata") or {}), ) raise ValueError(f"Unexpected response type: {type(litellm_completion_response)}") @@ -129,7 +136,7 @@ class LiteLLMCompletionTransformationHandler: litellm_completion_request: dict, request_input: str | ResponseInputParam, responses_api_request: ResponsesAPIOptionalRequestParams, - **kwargs, + **kwargs: object, ) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator: previous_response_id: Final[str | None] = responses_api_request.get("previous_response_id") if previous_response_id: @@ -142,10 +149,11 @@ class LiteLLMCompletionTransformationHandler: input_replaces_history=holds_compaction_summary(with_compaction_items_as_params(request_input)), ) - acompletion_args: Final = {} - acompletion_args.update(kwargs) - acompletion_args.update(litellm_completion_request) - acompletion_args["_skip_responses_api_bridge"] = True + acompletion_args: Final = { + **kwargs, + **_REQUEST_KWARGS_ADAPTER.validate_python(litellm_completion_request), + "_skip_responses_api_bridge": True, + } litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = await litellm.acompletion( **acompletion_args, @@ -169,6 +177,6 @@ class LiteLLMCompletionTransformationHandler: request_input=request_input, responses_api_request=responses_api_request, custom_llm_provider=litellm_completion_request.get("custom_llm_provider"), - litellm_metadata=kwargs.get("litellm_metadata", {}), + litellm_metadata=_REQUEST_KWARGS_ADAPTER.validate_python(kwargs.get("litellm_metadata") or {}), ) raise ValueError(f"Unexpected response type: {type(litellm_completion_response)}")