scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-19 00:01:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 88

fix(responses): preserve list-style input during spend-log history reconstruction

Browse source 
ResponsesSessionHandler.extend_chat_completion_message_with_spend_log_payload
only matched `str` and `dict` for the `input` field of a stored proxy
server request. The standard Responses API ships `input` as a *list* of
input items (EasyInputMessageParam, Message, etc.), so list-style inputs
were silently dropped from history reconstruction. Only the assistant
output from `response` survived, which broke `previous_response_id`
continuity for non-OpenAI providers (e.g. Vertex AI / Gemini): the system
prompt and any prior user turns never reached the model on follow-up
calls, so the model lost its persona and any context introduced via the
system message.

This widens the structured-input branch to `(list, dict)` so list-style
inputs are preserved and replayed as part of the reconstructed chat
completion message history.

Adds a regression test
(test_get_chat_completion_message_history_with_list_style_input) that
seeds a spend log with a list-style input containing a system + user
message and asserts the reconstructed history contains all three turns
(system + user + assistant) with the expected content.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Albert Casademont 2026-05-03 19:53:37 +02:00
parent c011a7e3ba
commit 4842059f46
2 changed files with 126 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
litellm/responses/litellm_completion_transformation/session_handler.py
View file

@ -116,7 +116,7 @@ class ResponsesSessionHandler:
_messages = proxy_server_request_dict.get("messages", None)
if isinstance(_response_input_param, str):
response_input_param = _response_input_param
elif isinstance(_response_input_param, dict):
elif isinstance(_response_input_param, (list, dict)):
response_input_param = cast(ResponseInputParam, _response_input_param)
if response_input_param:

125
tests/test_litellm/responses/litellm_completion_transformation/test_session_handler.py
View file

@ -388,6 +388,131 @@ async def test_should_check_cold_storage_for_full_payload():
), "Should return False when cold storage is not configured, even with truncated content"
@pytest.mark.asyncio
async def test_get_chat_completion_message_history_with_list_style_input():
"""
Regression test: when `proxy_server_request.input` is a list of input
items (the standard Responses API shape, e.g. system + user messages
expressed as ``[{"role": "system", "type": "message", "content": [...]},
{"role": "user", ...}]``), the full input must be replayed in the
reconstructed history.
Previously the handler only matched ``str`` and ``dict`` for ``input``,
so list-style inputs were silently dropped and only the assistant
output from ``response`` survived. That broke ``previous_response_id``
continuity for non-OpenAI providers (e.g. Vertex AI / Gemini): the
system prompt and the prior user turn never reached the model on
follow-up calls, so the model lost its persona and any context
introduced via the system message.
"""
mock_spend_logs = [
{
"request_id": "chatcmpl-list-input-test",
"call_type": "aresponses",
"api_key": "sk-test-mock-api-key-123",
"spend": 0.001,
"total_tokens": 42,
"prompt_tokens": 20,
"completion_tokens": 22,
"startTime": "2025-05-30T03:17:06.703+00:00",
"endTime": "2025-05-30T03:17:11.894+00:00",
"model": "vertex_ai/gemini-2.5-pro",
"session_id": "list-input-session",
"proxy_server_request": {
"model": "gemini-2.5-pro",
"input": [
{
"role": "system",
"type": "message",
"content": [
{
"type": "input_text",
"text": "You are a math tutor. The secret codename is ZEPHYR-7.",
}
],
},
{
"role": "user",
"type": "message",
"content": [
{"type": "input_text", "text": "What is 12 times 7?"}
],
},
],
},
"response": {
"id": "chatcmpl-list-input-test",
"model": "gemini-2.5-pro",
"object": "chat.completion",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "12 times 7 is 84.",
"tool_calls": None,
"function_call": None,
},
"finish_reason": "stop",
}
],
"created": 1748575031,
"usage": {
"total_tokens": 42,
"prompt_tokens": 20,
"completion_tokens": 22,
},
},
"status": "success",
}
]
with patch.object(
ResponsesSessionHandler,
"get_all_spend_logs_for_previous_response_id",
new_callable=AsyncMock,
) as mock_get_spend_logs:
mock_get_spend_logs.return_value = mock_spend_logs
result = await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id(
"chatcmpl-list-input-test"
)
messages = result["messages"]
assert (
len(messages) == 3
), f"expected system + user + assistant, got {len(messages)}: {messages}"
roles = [m.get("role") for m in messages]
assert roles == ["system", "user", "assistant"], (
"list-style Responses API input was not preserved during history "
f"reconstruction; got roles={roles}"
)
system_content = messages[0].get("content", "")
if isinstance(system_content, list):
system_text = "".join(
part.get("text", "")
for part in system_content
if isinstance(part, dict)
)
else:
system_text = system_content
assert (
"ZEPHYR-7" in system_text
), "system prompt content was lost during history reconstruction"
user_content = messages[1].get("content", "")
if isinstance(user_content, list):
user_text = "".join(
part.get("text", "") for part in user_content if isinstance(part, dict)
)
else:
user_text = user_content
assert "12 times 7" in user_text
@pytest.mark.asyncio
async def test_get_chat_completion_message_history_empty_response_dict():
"""