scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-12 08:42:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 207

Merge 1261885972 into 9007e0fa70

Browse source
This commit is contained in:
Dev Hiren Chiniwala 2026-10-29 02:13:55 +05:30 • committed by GitHub
commit 58d409ad2d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 163 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
litellm/responses/litellm_completion_transformation/handler.py
View file

@ -6,8 +6,10 @@ from collections.abc import Coroutine, Mapping
from typing import Final
from pydantic import TypeAdapter
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.responses.additional_tools import hoist_additional_tools
from litellm.responses.litellm_completion_transformation.compaction_items import holds_compaction_summary
from litellm.responses.litellm_completion_transformation.streaming_iterator import (
@ -28,6 +30,14 @@ from litellm.types.utils import ModelResponse
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(dict[str, object])
class _ContinuationFields(TypedDict, total=False):
previous_response_id: ReadOnly[str | None]
instructions: ReadOnly[str | None]
_CONTINUATION_FIELDS_ADAPTER: Final = TypeAdapter(_ContinuationFields)
class LiteLLMCompletionTransformationHandler:
def response_api_handler(
self,
@ -41,7 +51,7 @@ class LiteLLMCompletionTransformationHandler:
_is_async: bool = False,
stream: bool | None = None,
extra_headers: Mapping[str, object] | None = None,
**kwargs,
**kwargs: object,
) -> (
ResponsesAPIResponse
| BaseResponsesAPIStreamingIterator
@ -52,7 +62,7 @@ class LiteLLMCompletionTransformationHandler:
bridged_request: Final[ResponsesAPIOptionalRequestParams] = (
{**responses_api_request, "tools": list(hoisted.tools)} if hoisted.hoisted else responses_api_request
)
litellm_completion_request: Final[dict] = (
litellm_completion_request: Final = _REQUEST_KWARGS_ADAPTER.validate_python(
LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
model=model,
input=bridged_input,
@ -73,10 +83,24 @@ class LiteLLMCompletionTransformationHandler:
**kwargs,
)
completion_args: Final = {}
completion_args.update(kwargs)
completion_args.update(litellm_completion_request)
completion_args["_skip_responses_api_bridge"] = True
continuation: Final = _CONTINUATION_FIELDS_ADAPTER.validate_python(bridged_request)
previous_response_id: Final = continuation.get("previous_response_id")
sync_request: Final = (
_REQUEST_KWARGS_ADAPTER.validate_python(
run_async_function(
LiteLLMCompletionResponsesConfig.async_responses_api_session_handler,
previous_response_id=previous_response_id,
litellm_completion_request=litellm_completion_request,
instructions=continuation.get("instructions"),
request_kwargs=kwargs,
previous_response_client_ids=previous_response_client_ids,
input_replaces_history=holds_compaction_summary(with_compaction_items_as_params(bridged_input)),
)
)
if previous_response_id
else litellm_completion_request
)
completion_args: Final = {**kwargs, **sync_request, "_skip_responses_api_bridge": True}
litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = litellm.completion(
**completion_args,
@ -100,7 +124,7 @@ class LiteLLMCompletionTransformationHandler:
request_input=bridged_input,
responses_api_request=bridged_request,
custom_llm_provider=custom_llm_provider,
litellm_metadata=kwargs.get("litellm_metadata", {}),
litellm_metadata=_REQUEST_KWARGS_ADAPTER.validate_python(kwargs.get("litellm_metadata") or {}),
)
raise ValueError(f"Unexpected response type: {type(litellm_completion_response)}")
@ -112,7 +136,7 @@ class LiteLLMCompletionTransformationHandler:
litellm_completion_request: dict,
request_input: str | ResponseInputParam,
responses_api_request: ResponsesAPIOptionalRequestParams,
**kwargs,
**kwargs: object,
) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator:
previous_response_id: Final[str | None] = responses_api_request.get("previous_response_id")
if previous_response_id:
@ -125,10 +149,11 @@ class LiteLLMCompletionTransformationHandler:
input_replaces_history=holds_compaction_summary(with_compaction_items_as_params(request_input)),
)
acompletion_args: Final = {}
acompletion_args.update(kwargs)
acompletion_args.update(litellm_completion_request)
acompletion_args["_skip_responses_api_bridge"] = True
acompletion_args: Final = {
**kwargs,
**_REQUEST_KWARGS_ADAPTER.validate_python(litellm_completion_request),
"_skip_responses_api_bridge": True,
}
litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = await litellm.acompletion(
**acompletion_args,
@ -151,6 +176,6 @@ class LiteLLMCompletionTransformationHandler:
request_input=request_input,
responses_api_request=responses_api_request,
custom_llm_provider=litellm_completion_request.get("custom_llm_provider"),
litellm_metadata=kwargs.get("litellm_metadata", {}),
litellm_metadata=_REQUEST_KWARGS_ADAPTER.validate_python(kwargs.get("litellm_metadata") or {}),
)
raise ValueError(f"Unexpected response type: {type(litellm_completion_response)}")

10
litellm/responses/litellm_completion_transformation/transformation.py
View file

@ -1320,6 +1320,14 @@ class LiteLLMCompletionResponsesConfig:
_messages: Final = litellm_completion_request.get("messages") or []
route: Final = CompletionRoute(model=model, custom_llm_provider=provider)
session_history: Final = chat_completion_session.get("messages") or []
if previous_response_id and not session_history:
raise litellm.BadRequestError(
message="No stored conversation history is available for previous_response_id. "
"Start a new conversation without previous_response_id and include the full history.",
model=model,
llm_provider=provider or "",
body={"param": "previous_response_id"},
)
stored_messages: Final = LiteLLMCompletionResponsesConfig._with_compaction_blocks_the_route_takes(
_instruction_messages(session_history) if input_replaces_history else session_history,
route,
@ -1352,8 +1360,6 @@ class LiteLLMCompletionResponsesConfig:
else _messages[:instructions_end]
)
# If session messages are empty (e.g., no database in test environment),
# we still need to process the new input messages
# Store original _messages before combining for safety check
original_new_messages: Final = _messages.copy() if _messages else []

120
tests/unit/responses/litellm_completion_transformation/test_handler.py
View file

@ -22,7 +22,7 @@ import httpx
import pytest
import litellm
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.responses.litellm_completion_transformation.handler import (
LiteLLMCompletionTransformationHandler,
)
@ -209,15 +209,17 @@ async def test_bridged_follow_up_turn_keeps_the_addressed_response_id_off_the_pr
client: Final = AsyncHTTPHandler()
client.client = httpx.AsyncClient(transport=httpx.MockTransport(provider))
response = await litellm.aresponses(
model="azure_ai/claude-sonnet-4-6",
api_base="https://fake-foundry-resource.services.ai.azure.com",
api_key="fake-api-key",
input="Double it",
previous_response_id="resp_turn_one",
client=client,
**{ADDRESSED_RESPONSE_ID_FIELD: "resp_turn_one"},
)
stored_turns: Final = _StoredTurns((_stored_mcp_turn("resp_turn_one"),))
with patch("litellm.proxy.proxy_server.prisma_client", SimpleNamespace(db=stored_turns)):
response = await litellm.aresponses(
model="azure_ai/claude-sonnet-4-6",
api_base="https://fake-foundry-resource.services.ai.azure.com",
api_key="fake-api-key",
input="Double it",
previous_response_id="resp_turn_one",
client=client,
**{ADDRESSED_RESPONSE_ID_FIELD: "resp_turn_one"},
)
assert provider.request_body is not None, "the bridged turn never reached the provider"
assert ADDRESSED_RESPONSE_ID_FIELD not in provider.request_body, (
@ -255,6 +257,60 @@ class _StoredTurns:
return [dict(row) for row in self.rows if row["session_id"] in sessions]
@pytest.mark.parametrize("stream", [False, True])
@pytest.mark.parametrize("async_call", [False, True])
@pytest.mark.asyncio
async def test_unknown_previous_response_fails_before_sending_a_contextless_completion(
async_call: bool, stream: bool
) -> None:
provider: Final = _RecordingAnthropicHandler(
{
"id": "chatcmpl-local",
"object": "chat.completion",
"created": 0,
"model": "test-model",
"choices": [
{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "Lost context"}}
],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
}
)
transport: Final = httpx.MockTransport(provider)
async_client: Final = AsyncHTTPHandler(transport=transport)
with httpx.Client(transport=transport) as sync_client:
try:
if async_call:
with pytest.raises(litellm.BadRequestError, match="previous_response_id") as raised:
await litellm.aresponses(
model="hosted_vllm/test-model",
input="What is my favorite color?",
previous_response_id="resp_does_not_exist",
stream=stream,
use_chat_completions_api=True,
client=async_client,
api_key="test",
api_base="https://provider.invalid/v1",
num_retries=0,
)
else:
with pytest.raises(litellm.BadRequestError, match="previous_response_id") as raised:
litellm.responses(
model="hosted_vllm/test-model",
input="What is my favorite color?",
previous_response_id="resp_does_not_exist",
stream=stream,
use_chat_completions_api=True,
client=HTTPHandler(client=sync_client),
api_key="test",
api_base="https://provider.invalid/v1",
num_retries=0,
)
assert raised.value.status_code == 400
assert provider.request_body is None, "unresolved history must not send the current turn to the provider"
finally:
await async_client.client.aclose()
def _stored_mcp_turn(request_id: str) -> Mapping[str, object]:
return {
"request_id": request_id,
@ -279,7 +335,10 @@ def _stored_mcp_turn(request_id: str) -> Mapping[str, object]:
@pytest.mark.asyncio
@pytest.mark.parametrize("stored_under", ["litellm_encoded_id", "proxy_issued_id"])
async def test_bridged_next_turn_replays_a_turn_stored_under_the_id_the_client_holds(stored_under: str):
@pytest.mark.parametrize("async_call", [False, True])
async def test_bridged_next_turn_replays_a_turn_stored_under_the_id_the_client_holds(
stored_under: str, async_call: bool
):
litellm_encoded_id: Final = ResponsesAPIRequestUtils._build_responses_api_response_id(
custom_llm_provider="azure_ai", model_id="deployment-a", response_id="chatcmpl-turn-one"
)
@ -291,21 +350,34 @@ async def test_bridged_next_turn_replays_a_turn_stored_under_the_id_the_client_h
client: Final = AsyncHTTPHandler()
client.client = httpx.AsyncClient(transport=httpx.MockTransport(provider))
with patch("litellm.proxy.proxy_server.prisma_client", SimpleNamespace(db=stored_turns)):
await litellm.aresponses(
model="azure_ai/claude-sonnet-4-6",
api_base="https://fake-foundry-resource.services.ai.azure.com",
api_key="fake-api-key",
input="Double it",
previous_response_id=litellm_encoded_id,
client=client,
**{ADDRESSED_RESPONSE_ID_FIELD: proxy_issued_id},
)
with httpx.Client(transport=httpx.MockTransport(provider)) as sync_client:
try:
with patch("litellm.proxy.proxy_server.prisma_client", SimpleNamespace(db=stored_turns)):
if async_call:
await litellm.aresponses(
model="azure_ai/claude-sonnet-4-6",
api_base="https://fake-foundry-resource.services.ai.azure.com",
api_key="fake-api-key",
input="Double it",
previous_response_id=litellm_encoded_id,
client=client,
**{ADDRESSED_RESPONSE_ID_FIELD: proxy_issued_id},
)
else:
litellm.responses(
model="azure_ai/claude-sonnet-4-6",
api_base="https://fake-foundry-resource.services.ai.azure.com",
api_key="fake-api-key",
input="Double it",
previous_response_id=litellm_encoded_id,
client=HTTPHandler(client=sync_client),
**{ADDRESSED_RESPONSE_ID_FIELD: proxy_issued_id},
)
finally:
await client.client.aclose()
assert provider.request_body is not None, "the bridged turn never reached the provider"
assert [
(message["role"], json.dumps(message["content"])) for message in provider.request_body["messages"]
] == [
assert [(message["role"], json.dumps(message["content"])) for message in provider.request_body["messages"]] == [
("user", json.dumps([{"type": "text", "text": "My favorite number is 7."}])),
("assistant", json.dumps([{"type": "text", "text": "Noted."}])),
("user", json.dumps([{"type": "text", "text": "Double it"}])),

34
tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py
View file

@ -6713,7 +6713,6 @@ def test_summary_replay_drops_the_history_its_compaction_item_replaces() -> None
]
@pytest.mark.parametrize(
("call", "output"),
[
@ -6826,7 +6825,7 @@ def test_native_compaction_replay_moves_a_call_its_compaction_item_splits_from_i
pytest.param(ResponsesAPIOptionalRequestParams(), id="summary_as_text"),
],
)
def test_compaction_replay_invents_no_call_for_an_output_whose_call_was_never_stored(
def test_compaction_replay_rejects_missing_history_instead_of_inventing_tool_calls(
responses_api_request: ResponsesAPIOptionalRequestParams,
) -> None:
async def no_stored_turns(previous_response_id: str) -> ChatCompletionSession:
@ -6847,16 +6846,16 @@ def test_compaction_replay_invents_no_call_for_an_output_whose_call_was_never_st
),
)
request: Final = asyncio.run(
LiteLLMCompletionResponsesConfig.async_responses_api_session_handler(
previous_response_id="resp_unknown",
litellm_completion_request=chat_request,
instructions=None,
session_lookup=no_stored_turns,
with pytest.raises(litellm.BadRequestError, match="previous_response_id") as raised:
asyncio.run(
LiteLLMCompletionResponsesConfig.async_responses_api_session_handler(
previous_response_id="resp_unknown",
litellm_completion_request=chat_request,
instructions=None,
session_lookup=no_stored_turns,
)
)
)
assert not any(message.get("tool_calls") for message in request["messages"])
assert raised.value.status_code == 400
@pytest.mark.parametrize(
@ -6864,7 +6863,14 @@ def test_compaction_replay_invents_no_call_for_an_output_whose_call_was_never_st
[
pytest.param(
_COMPACTION_REQUESTED,
[("user", ()), ("assistant", ("call_0",)), ("tool", "Sunny."), ("assistant", ()), ("assistant", ("call_0",)), ("tool", "Rain.")],
[
("user", ()),
("assistant", ("call_0",)),
("tool", "Sunny."),
("assistant", ()),
("assistant", ("call_0",)),
("tool", "Rain."),
],
id="native_compaction_block",
),
pytest.param(
@ -7006,7 +7012,9 @@ async def test_bridge_partial_response_on_disconnect_bills_the_relayed_chat_chun
assert incomplete_event.type == "response.incomplete"
assert incomplete_event.response.status == "incomplete"
assert incomplete_event.response.usage is not None
assert incomplete_event.response.usage.output_tokens == litellm.token_counter(model="gpt-4o-mini", text=relayed_text)
assert incomplete_event.response.usage.output_tokens == litellm.token_counter(
model="gpt-4o-mini", text=relayed_text
)
assert incomplete_event.response.usage.input_tokens > 0