scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-15 09:12:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 311

Merge 69160b8bac into 527d882508

Browse source
This commit is contained in:
yxccai 2026-11-04 05:30:34 +00:00 • committed by GitHub
commit 42d078b27a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 234 additions and 50 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

60
litellm/responses/litellm_completion_transformation/handler.py
View file

@ -10,6 +10,7 @@ from typing import Final
from pydantic import TypeAdapter
import litellm
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id
from litellm.responses.additional_tools import HoistedAdditionalTools, hoist_additional_tools
@ -96,7 +97,9 @@ class LiteLLMCompletionTransformationHandler:
responses_api_request.get("tools"),
drop_params=drops_unsupported_params(_REQUEST_KWARGS_ADAPTER.validate_python(kwargs).get("drop_params")),
)
previous_response_id: Final = responses_api_request.get("previous_response_id")
previous_response_id: Final = (
responses_api_request["previous_response_id"] if "previous_response_id" in responses_api_request else None # noqa: SIM401 # TypedDict indexing preserves the field type
)
if _is_async and previous_response_id and has_client_tool_search(hoisted.tools):
return self._async_response_api_handler_after_earlier_searches(
previous_response_client_ids,
@ -113,20 +116,22 @@ class LiteLLMCompletionTransformationHandler:
bridged: Final = _bridged_request(hoisted, responses_api_request, ())
bridged_input: Final = bridged.input
bridged_request: Final = bridged.responses_api_request
litellm_completion_request: Final[dict] = {
**LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
model=model,
input=bridged_input,
caller_input=input,
names_non_string_fields=True,
responses_api_request=bridged_request,
custom_llm_provider=custom_llm_provider,
stream=stream,
extra_headers=extra_headers,
**_without_bridge_transform_keywords(_REQUEST_KWARGS_ADAPTER.validate_python(kwargs)),
),
**_chat_verbosity(bridged_request),
}
litellm_completion_request: Final = _REQUEST_KWARGS_ADAPTER.validate_python(
{
**LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
model=model,
input=bridged_input,
caller_input=input,
names_non_string_fields=True,
responses_api_request=bridged_request,
custom_llm_provider=custom_llm_provider,
stream=stream,
extra_headers=extra_headers,
**_without_bridge_transform_keywords(_REQUEST_KWARGS_ADAPTER.validate_python(kwargs)),
),
**_chat_verbosity(bridged_request),
}
)
if _is_async:
return self.async_response_api_handler(
@ -137,8 +142,24 @@ class LiteLLMCompletionTransformationHandler:
**kwargs,
)
completion_request: Final = _REQUEST_KWARGS_ADAPTER.validate_python(
run_async_function(
LiteLLMCompletionResponsesConfig.async_responses_api_session_handler,
previous_response_id=previous_response_id,
litellm_completion_request=litellm_completion_request,
instructions=bridged_request["instructions"] if "instructions" in bridged_request else None, # noqa: SIM401 # TypedDict indexing preserves the field type
request_kwargs=_REQUEST_KWARGS_ADAPTER.validate_python(kwargs),
previous_response_client_ids=previous_response_client_ids,
input_replaces_history=holds_compaction_summary(with_compaction_items_as_params(bridged_input)),
)
if previous_response_id
else litellm_completion_request
)
completion_args: Final = _REQUEST_KWARGS_ADAPTER.validate_python(
{**kwargs, **completion_request, "_skip_responses_api_bridge": True}
)
litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = litellm.completion(
**{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True},
**completion_args,
)
if isinstance(litellm_completion_response, ModelResponse):
@ -224,7 +245,7 @@ class LiteLLMCompletionTransformationHandler:
responses_api_request: ResponsesAPIOptionalRequestParams,
**kwargs,
) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator:
previous_response_id: Final[str | None] = responses_api_request.get("previous_response_id")
previous_response_id: Final = responses_api_request.get("previous_response_id")
if previous_response_id:
litellm_completion_request = await LiteLLMCompletionResponsesConfig.async_responses_api_session_handler(
previous_response_id=previous_response_id,
@ -236,8 +257,11 @@ class LiteLLMCompletionTransformationHandler:
input_replaces_history=holds_compaction_summary(with_compaction_items_as_params(request_input)),
)
acompletion_args: Final = _REQUEST_KWARGS_ADAPTER.validate_python(
{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True}
)
litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = await litellm.acompletion(
**{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True},
**acompletion_args,
)
if isinstance(litellm_completion_response, ModelResponse):
responses_api_response: Final[ResponsesAPIResponse] = (

13
litellm/responses/litellm_completion_transformation/session_handler.py
View file

@ -320,6 +320,19 @@ class ResponsesSessionHandler:
replayed_turns: Final = (
replayable_turns if _holds_compaction_point(all_spend_logs, proxy_server_requests) else len(all_spend_logs)
)
if not all_spend_logs or any(
request is None or (request.get("input") is None and request.get("messages") is None)
for index, request in enumerate(proxy_server_requests[start.index : replayed_turns], start=start.index)
if not (start.at_output and index == start.index) and not _is_a_response_read(all_spend_logs[index])
):
raise litellm.BadRequestError(
message=(
f"Cannot reconstruct conversation history for previous_response_id '{previous_response_id}'. "
"Resend the full conversation without previous_response_id."
),
model="",
llm_provider="",
)
earlier_instructions: Final = _instruction_items_before(proxy_server_requests, start)
replayed_history: Final = ResponsesSessionHandler._history_of_turns(
all_spend_logs[start.index : replayed_turns],

164
tests/unit/responses/litellm_completion_transformation/test_handler.py
View file

@ -13,7 +13,7 @@ kwargs lack the flag and these tests fail.
"""
import json
from collections.abc import Mapping, Sequence
from collections.abc import Iterator, Mapping, Sequence
from types import SimpleNamespace
from typing import Final
from unittest.mock import patch
@ -22,7 +22,8 @@ import httpx
import pytest
import litellm
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.proxy import proxy_server
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.responses.litellm_completion_transformation.handler import (
LiteLLMCompletionTransformationHandler,
)
@ -209,15 +210,17 @@ async def test_bridged_follow_up_turn_keeps_the_addressed_response_id_off_the_pr
client: Final = AsyncHTTPHandler()
client.client = httpx.AsyncClient(transport=httpx.MockTransport(provider))
response = await litellm.aresponses(
model="azure_ai/claude-sonnet-4-6",
api_base="https://fake-foundry-resource.services.ai.azure.com",
api_key="fake-api-key",
input="Double it",
previous_response_id="resp_turn_one",
client=client,
**{ADDRESSED_RESPONSE_ID_FIELD: "resp_turn_one"},
)
database: Final = SimpleNamespace(db=_StoredTurns((_stored_mcp_turn("resp_turn_one"),)))
with patch.object(proxy_server, "prisma_client", database):
response = await litellm.aresponses(
model="azure_ai/claude-sonnet-4-6",
api_base="https://fake-foundry-resource.services.ai.azure.com",
api_key="fake-api-key",
input="Double it",
previous_response_id="resp_turn_one",
client=client,
**{ADDRESSED_RESPONSE_ID_FIELD: "resp_turn_one"},
)
assert provider.request_body is not None, "the bridged turn never reached the provider"
assert ADDRESSED_RESPONSE_ID_FIELD not in provider.request_body, (
@ -323,15 +326,148 @@ async def test_bridged_next_turn_replays_a_turn_stored_under_the_id_the_client_h
)
assert provider.request_body is not None, "the bridged turn never reached the provider"
assert [
(message["role"], json.dumps(message["content"])) for message in provider.request_body["messages"]
] == [
assert [(message["role"], json.dumps(message["content"])) for message in provider.request_body["messages"]] == [
("user", json.dumps([{"type": "text", "text": "My favorite number is 7."}])),
("assistant", json.dumps([{"type": "text", "text": "Noted."}])),
("user", json.dumps([{"type": "text", "text": "Double it"}])),
]
_CHAT_COMPLETION_PAYLOAD: Final = {
"id": "chatcmpl-current-turn",
"object": "chat.completion",
"created": 1,
"model": "bridge-test",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "14"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 10, "completion_tokens": 1, "total_tokens": 11},
}
@pytest.fixture(params=["no_database", "unknown_response", "unstored_input"])
def unavailable_response_history(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
rows: Final = (
({**_stored_mcp_turn("resp_missing_history"), "proxy_server_request": {}},)
if request.param == "unstored_input"
else ()
)
database: Final = None if request.param == "no_database" else SimpleNamespace(db=_StoredTurns(rows))
monkeypatch.setattr(litellm.constants, "RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL", 0.0)
with patch.object(proxy_server, "prisma_client", database):
yield
@pytest.mark.parametrize("stream", [False, True])
def test_sync_bridge_rejects_unavailable_previous_response_before_inference(
unavailable_response_history: None, stream: bool
):
provider: Final = _RecordingAnthropicHandler(_CHAT_COMPLETION_PAYLOAD)
with httpx.Client(transport=httpx.MockTransport(provider)) as http_client:
with pytest.raises(litellm.BadRequestError, match=r"previous_response_id.*resp_missing_history") as error:
litellm.responses(
model="hosted_vllm/bridge-test",
api_base="https://provider.example/v1",
api_key="test-key",
input="Double it",
previous_response_id="resp_missing_history",
use_chat_completions_api=True,
stream=stream,
client=HTTPHandler(client=http_client),
)
assert error.value.status_code == 400
assert provider.request_body is None
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True])
async def test_async_bridge_rejects_unavailable_previous_response_before_inference(
unavailable_response_history: None, stream: bool
):
provider: Final = _RecordingAnthropicHandler(_CHAT_COMPLETION_PAYLOAD)
client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(provider))
try:
with pytest.raises(litellm.BadRequestError, match=r"previous_response_id.*resp_missing_history") as error:
await litellm.aresponses(
model="hosted_vllm/bridge-test",
api_base="https://provider.example/v1",
api_key="test-key",
input="Double it",
previous_response_id="resp_missing_history",
use_chat_completions_api=True,
stream=stream,
client=client,
)
finally:
await client.close()
assert error.value.status_code == 400
assert provider.request_body is None
@pytest.mark.parametrize("previous_response_id", [None, "resp_stored"])
def test_sync_bridge_preserves_available_history_and_first_turns(previous_response_id: str | None):
provider: Final = _RecordingAnthropicHandler(_CHAT_COMPLETION_PAYLOAD)
database: Final = SimpleNamespace(db=_StoredTurns((_stored_mcp_turn("resp_stored"),)))
with httpx.Client(transport=httpx.MockTransport(provider)) as http_client:
with patch.object(proxy_server, "prisma_client", database):
response = litellm.responses(
model="hosted_vllm/bridge-test",
api_base="https://provider.example/v1",
api_key="test-key",
input="Double it",
previous_response_id=previous_response_id,
use_chat_completions_api=True,
client=HTTPHandler(client=http_client),
)
assert isinstance(response, ResponsesAPIResponse)
assert response.status == "completed"
assert provider.request_body is not None
assert provider.request_body["messages"] == (
[
{"role": "user", "content": "My favorite number is 7."},
{"role": "assistant", "content": "Noted."},
{"role": "user", "content": "Double it"},
]
if previous_response_id
else [{"role": "user", "content": "Double it"}]
)
@pytest.mark.asyncio
@pytest.mark.parametrize("previous_response_id", [None, "resp_stored"])
async def test_async_bridge_preserves_available_history_and_first_turns(previous_response_id: str | None):
provider: Final = _RecordingAnthropicHandler(_CHAT_COMPLETION_PAYLOAD)
database: Final = SimpleNamespace(db=_StoredTurns((_stored_mcp_turn("resp_stored"),)))
client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(provider))
try:
with patch.object(proxy_server, "prisma_client", database):
response = await litellm.aresponses(
model="hosted_vllm/bridge-test",
api_base="https://provider.example/v1",
api_key="test-key",
input="Double it",
previous_response_id=previous_response_id,
use_chat_completions_api=True,
client=client,
)
finally:
await client.close()
assert isinstance(response, ResponsesAPIResponse)
assert response.status == "completed"
assert provider.request_body is not None
assert provider.request_body["messages"] == (
[
{"role": "user", "content": "My favorite number is 7."},
{"role": "assistant", "content": "Noted."},
{"role": "user", "content": "Double it"},
]
if previous_response_id
else [{"role": "user", "content": "Double it"}]
)
@pytest.mark.asyncio
@pytest.mark.parametrize("client_field", ["caller_input", "names_non_string_fields"])
async def test_bridged_follow_up_after_a_client_tool_search_with_a_client_field_named_like_a_bridge_keyword_reaches_the_provider(

47
tests/unit/responses/litellm_completion_transformation/test_session_handler.py
View file

@ -13,6 +13,7 @@ from fastapi import HTTPException
from fastapi.testclient import TestClient
import litellm
from litellm.proxy import proxy_server
from litellm.proxy.spend_tracking.spend_tracking_utils import (
_get_proxy_server_request_for_spend_logs_payload,
)
@ -172,25 +173,32 @@ async def test_get_chat_completion_message_history_for_previous_response_id():
@pytest.mark.asyncio
async def test_get_chat_completion_message_history_empty_spend_logs():
"""
Test get_chat_completion_message_history_for_previous_response_id with empty spend logs
"""
with patch.object(
ResponsesSessionHandler,
"get_all_spend_logs_for_previous_response_id",
new_callable=AsyncMock,
) as mock_get_spend_logs:
mock_get_spend_logs.return_value = []
async def test_get_chat_completion_message_history_empty_spend_logs(instant_session_lookup_retries: None):
with patch.object(proxy_server, "prisma_client", _FakePrismaClient(results=[])):
with pytest.raises(litellm.BadRequestError, match=r"previous_response_id.*non-existent-id") as error:
await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id(
"non-existent-id"
)
previous_response_id = "non-existent-id"
result = await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id(
previous_response_id
assert error.value.status_code == 400
@pytest.mark.asyncio
@pytest.mark.parametrize("stored_request", [None, "{}", "null", {}, {"model": "bridge-test"}])
async def test_message_history_rejects_a_stored_turn_without_its_input(stored_request: object):
stored: Final = _StoredSpendLogs(
(
{
**_spend_log("resp_missing_input", "session-a", "Remember 7", "Noted"),
"proxy_server_request": stored_request,
},
)
# Verify empty result structure
assert result.get("messages") == []
assert result.get("litellm_session_id") is None
)
with patch.object(proxy_server, "prisma_client", SimpleNamespace(db=stored)):
with pytest.raises(litellm.BadRequestError, match=r"previous_response_id.*resp_missing_input"):
await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id(
"resp_missing_input"
)
@pytest.mark.asyncio
@ -1162,7 +1170,10 @@ async def test_an_item_a_poll_caught_half_written_replays_its_finished_text_once
"resp_poll_done",
"aget_responses",
"completed",
[_identified_output_message("msg_answer", "One, two, three"), _identified_output_message("msg_more", "Done")],
[
_identified_output_message("msg_answer", "One, two, three"),
_identified_output_message("msg_more", "Done"),
],
),
)