scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-05 07:37:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 360

Merge 6bbdb03c5a into 0a1dbc0fb4

Browse source
This commit is contained in:
yxccai 2026-10-27 10:55:01 -07:00 • committed by GitHub
commit 8f18cc9963
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 211 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
litellm/responses/litellm_completion_transformation/handler.py
View file

@ -8,6 +8,7 @@ from typing import Final
from pydantic import TypeAdapter
import litellm
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.responses.additional_tools import hoist_additional_tools
from litellm.responses.litellm_completion_transformation.compaction_items import holds_compaction_summary
from litellm.responses.litellm_completion_transformation.streaming_iterator import (
@ -52,7 +53,7 @@ class LiteLLMCompletionTransformationHandler:
bridged_request: Final[ResponsesAPIOptionalRequestParams] = (
{**responses_api_request, "tools": list(hoisted.tools)} if hoisted.hoisted else responses_api_request
)
litellm_completion_request: Final[dict] = (
litellm_completion_request: Final = _REQUEST_KWARGS_ADAPTER.validate_python(
LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
model=model,
input=bridged_input,
@ -73,10 +74,23 @@ class LiteLLMCompletionTransformationHandler:
**kwargs,
)
completion_args: Final = {}
completion_args.update(kwargs)
completion_args.update(litellm_completion_request)
completion_args["_skip_responses_api_bridge"] = True
previous_response_id: Final = bridged_request.get("previous_response_id")
completion_request: Final = _REQUEST_KWARGS_ADAPTER.validate_python(
run_async_function(
LiteLLMCompletionResponsesConfig.async_responses_api_session_handler,
previous_response_id=previous_response_id,
litellm_completion_request=litellm_completion_request,
instructions=bridged_request.get("instructions"),
request_kwargs=_REQUEST_KWARGS_ADAPTER.validate_python(kwargs),
previous_response_client_ids=previous_response_client_ids,
input_replaces_history=holds_compaction_summary(with_compaction_items_as_params(bridged_input)),
)
if previous_response_id
else litellm_completion_request
)
completion_args: Final = _REQUEST_KWARGS_ADAPTER.validate_python(
{**kwargs, **completion_request, "_skip_responses_api_bridge": True}
)
litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = litellm.completion(
**completion_args,
@ -114,7 +128,7 @@ class LiteLLMCompletionTransformationHandler:
responses_api_request: ResponsesAPIOptionalRequestParams,
**kwargs,
) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator:
previous_response_id: Final[str | None] = responses_api_request.get("previous_response_id")
previous_response_id: Final = responses_api_request.get("previous_response_id")
if previous_response_id:
litellm_completion_request = await LiteLLMCompletionResponsesConfig.async_responses_api_session_handler(
previous_response_id=previous_response_id,
@ -125,10 +139,9 @@ class LiteLLMCompletionTransformationHandler:
input_replaces_history=holds_compaction_summary(with_compaction_items_as_params(request_input)),
)
acompletion_args: Final = {}
acompletion_args.update(kwargs)
acompletion_args.update(litellm_completion_request)
acompletion_args["_skip_responses_api_bridge"] = True
acompletion_args: Final = _REQUEST_KWARGS_ADAPTER.validate_python(
{**kwargs, **litellm_completion_request, "_skip_responses_api_bridge": True}
)
litellm_completion_response: Final[ModelResponse | litellm.CustomStreamWrapper] = await litellm.acompletion(
**acompletion_args,

13
litellm/responses/litellm_completion_transformation/session_handler.py
View file

@ -217,6 +217,19 @@ class ResponsesSessionHandler:
replayed_turns: Final = (
replayable_turns if _holds_compaction_point(all_spend_logs, proxy_server_requests) else len(all_spend_logs)
)
if not all_spend_logs or any(
request is None or (request.get("input") is None and request.get("messages") is None)
for index, request in enumerate(proxy_server_requests[start.index : replayed_turns], start=start.index)
if not (start.at_output and index == start.index)
):
raise litellm.BadRequestError(
message=(
f"Cannot reconstruct conversation history for previous_response_id '{previous_response_id}'. "
"Resend the full conversation without previous_response_id."
),
model="",
llm_provider="",
)
earlier_instructions: Final = _instruction_items_before(proxy_server_requests, start)
replayed_history: Final = ResponsesSessionHandler._history_of_turns(
all_spend_logs[start.index : replayed_turns],

164
tests/unit/responses/litellm_completion_transformation/test_handler.py
View file

@ -13,7 +13,7 @@ kwargs lack the flag and these tests fail.
"""
import json
from collections.abc import Mapping, Sequence
from collections.abc import Iterator, Mapping, Sequence
from types import SimpleNamespace
from typing import Final
from unittest.mock import patch
@ -22,7 +22,8 @@ import httpx
import pytest
import litellm
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.proxy import proxy_server
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.responses.litellm_completion_transformation.handler import (
LiteLLMCompletionTransformationHandler,
)
@ -209,15 +210,17 @@ async def test_bridged_follow_up_turn_keeps_the_addressed_response_id_off_the_pr
client: Final = AsyncHTTPHandler()
client.client = httpx.AsyncClient(transport=httpx.MockTransport(provider))
response = await litellm.aresponses(
model="azure_ai/claude-sonnet-4-6",
api_base="https://fake-foundry-resource.services.ai.azure.com",
api_key="fake-api-key",
input="Double it",
previous_response_id="resp_turn_one",
client=client,
**{ADDRESSED_RESPONSE_ID_FIELD: "resp_turn_one"},
)
database: Final = SimpleNamespace(db=_StoredTurns((_stored_mcp_turn("resp_turn_one"),)))
with patch.object(proxy_server, "prisma_client", database):
response = await litellm.aresponses(
model="azure_ai/claude-sonnet-4-6",
api_base="https://fake-foundry-resource.services.ai.azure.com",
api_key="fake-api-key",
input="Double it",
previous_response_id="resp_turn_one",
client=client,
**{ADDRESSED_RESPONSE_ID_FIELD: "resp_turn_one"},
)
assert provider.request_body is not None, "the bridged turn never reached the provider"
assert ADDRESSED_RESPONSE_ID_FIELD not in provider.request_body, (
@ -303,10 +306,143 @@ async def test_bridged_next_turn_replays_a_turn_stored_under_the_id_the_client_h
)
assert provider.request_body is not None, "the bridged turn never reached the provider"
assert [
(message["role"], json.dumps(message["content"])) for message in provider.request_body["messages"]
] == [
assert [(message["role"], json.dumps(message["content"])) for message in provider.request_body["messages"]] == [
("user", json.dumps([{"type": "text", "text": "My favorite number is 7."}])),
("assistant", json.dumps([{"type": "text", "text": "Noted."}])),
("user", json.dumps([{"type": "text", "text": "Double it"}])),
]
_CHAT_COMPLETION_PAYLOAD: Final = {
"id": "chatcmpl-current-turn",
"object": "chat.completion",
"created": 1,
"model": "bridge-test",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "14"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 10, "completion_tokens": 1, "total_tokens": 11},
}
@pytest.fixture(params=["no_database", "unknown_response", "unstored_input"])
def unavailable_response_history(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
rows: Final = (
({**_stored_mcp_turn("resp_missing_history"), "proxy_server_request": {}},)
if request.param == "unstored_input"
else ()
)
database: Final = None if request.param == "no_database" else SimpleNamespace(db=_StoredTurns(rows))
monkeypatch.setattr(litellm.constants, "RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL", 0.0)
with patch.object(proxy_server, "prisma_client", database):
yield
@pytest.mark.parametrize("stream", [False, True])
def test_sync_bridge_rejects_unavailable_previous_response_before_inference(
unavailable_response_history: None, stream: bool
):
provider: Final = _RecordingAnthropicHandler(_CHAT_COMPLETION_PAYLOAD)
with httpx.Client(transport=httpx.MockTransport(provider)) as http_client:
with pytest.raises(litellm.BadRequestError, match=r"previous_response_id.*resp_missing_history") as error:
litellm.responses(
model="hosted_vllm/bridge-test",
api_base="https://provider.example/v1",
api_key="test-key",
input="Double it",
previous_response_id="resp_missing_history",
use_chat_completions_api=True,
stream=stream,
client=HTTPHandler(client=http_client),
)
assert error.value.status_code == 400
assert provider.request_body is None
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True])
async def test_async_bridge_rejects_unavailable_previous_response_before_inference(
unavailable_response_history: None, stream: bool
):
provider: Final = _RecordingAnthropicHandler(_CHAT_COMPLETION_PAYLOAD)
client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(provider))
try:
with pytest.raises(litellm.BadRequestError, match=r"previous_response_id.*resp_missing_history") as error:
await litellm.aresponses(
model="hosted_vllm/bridge-test",
api_base="https://provider.example/v1",
api_key="test-key",
input="Double it",
previous_response_id="resp_missing_history",
use_chat_completions_api=True,
stream=stream,
client=client,
)
finally:
await client.close()
assert error.value.status_code == 400
assert provider.request_body is None
@pytest.mark.parametrize("previous_response_id", [None, "resp_stored"])
def test_sync_bridge_preserves_available_history_and_first_turns(previous_response_id: str | None):
provider: Final = _RecordingAnthropicHandler(_CHAT_COMPLETION_PAYLOAD)
database: Final = SimpleNamespace(db=_StoredTurns((_stored_mcp_turn("resp_stored"),)))
with httpx.Client(transport=httpx.MockTransport(provider)) as http_client:
with patch.object(proxy_server, "prisma_client", database):
response = litellm.responses(
model="hosted_vllm/bridge-test",
api_base="https://provider.example/v1",
api_key="test-key",
input="Double it",
previous_response_id=previous_response_id,
use_chat_completions_api=True,
client=HTTPHandler(client=http_client),
)
assert isinstance(response, ResponsesAPIResponse)
assert response.status == "completed"
assert provider.request_body is not None
assert provider.request_body["messages"] == (
[
{"role": "user", "content": "My favorite number is 7."},
{"role": "assistant", "content": "Noted."},
{"role": "user", "content": "Double it"},
]
if previous_response_id
else [{"role": "user", "content": "Double it"}]
)
@pytest.mark.asyncio
@pytest.mark.parametrize("previous_response_id", [None, "resp_stored"])
async def test_async_bridge_preserves_available_history_and_first_turns(previous_response_id: str | None):
provider: Final = _RecordingAnthropicHandler(_CHAT_COMPLETION_PAYLOAD)
database: Final = SimpleNamespace(db=_StoredTurns((_stored_mcp_turn("resp_stored"),)))
client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(provider))
try:
with patch.object(proxy_server, "prisma_client", database):
response = await litellm.aresponses(
model="hosted_vllm/bridge-test",
api_base="https://provider.example/v1",
api_key="test-key",
input="Double it",
previous_response_id=previous_response_id,
use_chat_completions_api=True,
client=client,
)
finally:
await client.close()
assert isinstance(response, ResponsesAPIResponse)
assert response.status == "completed"
assert provider.request_body is not None
assert provider.request_body["messages"] == (
[
{"role": "user", "content": "My favorite number is 7."},
{"role": "assistant", "content": "Noted."},
{"role": "user", "content": "Double it"},
]
if previous_response_id
else [{"role": "user", "content": "Double it"}]
)

42
tests/unit/responses/litellm_completion_transformation/test_session_handler.py
View file

@ -11,6 +11,7 @@ from fastapi import HTTPException
from fastapi.testclient import TestClient
import litellm
from litellm.proxy import proxy_server
from litellm.proxy.spend_tracking.spend_tracking_utils import (
_get_proxy_server_request_for_spend_logs_payload,
)
@ -170,25 +171,32 @@ async def test_get_chat_completion_message_history_for_previous_response_id():
@pytest.mark.asyncio
async def test_get_chat_completion_message_history_empty_spend_logs():
"""
Test get_chat_completion_message_history_for_previous_response_id with empty spend logs
"""
with patch.object(
ResponsesSessionHandler,
"get_all_spend_logs_for_previous_response_id",
new_callable=AsyncMock,
) as mock_get_spend_logs:
mock_get_spend_logs.return_value = []
async def test_get_chat_completion_message_history_empty_spend_logs(instant_session_lookup_retries: None):
with patch.object(proxy_server, "prisma_client", _FakePrismaClient(results=[])):
with pytest.raises(litellm.BadRequestError, match=r"previous_response_id.*non-existent-id") as error:
await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id(
"non-existent-id"
)
previous_response_id = "non-existent-id"
result = await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id(
previous_response_id
assert error.value.status_code == 400
@pytest.mark.asyncio
@pytest.mark.parametrize("stored_request", [None, "{}", "null", {}, {"model": "bridge-test"}])
async def test_message_history_rejects_a_stored_turn_without_its_input(stored_request: object):
stored: Final = _StoredSpendLogs(
(
{
**_spend_log("resp_missing_input", "session-a", "Remember 7", "Noted"),
"proxy_server_request": stored_request,
},
)
# Verify empty result structure
assert result.get("messages") == []
assert result.get("litellm_session_id") is None
)
with patch.object(proxy_server, "prisma_client", SimpleNamespace(db=stored)):
with pytest.raises(litellm.BadRequestError, match=r"previous_response_id.*resp_missing_input"):
await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id(
"resp_missing_input"
)
@pytest.mark.asyncio