scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

fix(proxy): make /cursor/chat/completions work with Cursor agent mode

Browse source 
- delegate messages-shaped bodies to the standard chat completions handler
- strip chat-only stream_options before the Responses pipeline
- fix cursor_data_generator signature (request kwarg) and duck-type the
  stream gate so router-wrapped streams convert instead of leaking raw
  Responses events
- convert custom_tool_call items and events to chat tool_calls in the
  streaming and non-streaming paths; remap streamed tool_call indices to
  0-based sequential; accumulate raw and pydantic tool calls into one choice
- normalize generic pydantic output items through the raw-dict handler
This commit is contained in:
Tin Chi Lo 2026-07-20 13:48:24 -07:00
parent 2b3070890a
commit 14c97ba8db
6 changed files with 555 additions and 89 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

181
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -100,6 +100,32 @@ def _build_reasoning_item(
}
def _tool_call_dict_from_output_item(item: dict[str, Any]) -> dict[str, Any]:
"""Convert a ``function_call`` or ``custom_tool_call`` output item dict to a chat
completions tool_call dict. Custom (grammar/freeform) tool calls carry their raw
string payload in ``input`` rather than ``arguments``; both map to
``function.arguments`` so chat clients (e.g. Cursor agent mode) receive them like
any other tool call. The single conversion rule shared by the non-streaming
accumulator and the streaming ``output_item.added`` branch."""
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)
is_custom = item.get("type") == "custom_tool_call"
arguments = (item.get("input") if is_custom else item.get("arguments")) or ""
name = item.get("name") or ("custom_tool" if is_custom else "")
tool_call_dict: dict[str, Any] = {
"id": LiteLLMCompletionResponsesConfig._tool_call_id_from_responses_item(item.get("id"), item.get("call_id")),
"function": {"name": name, "arguments": arguments},
"type": "function",
}
provider_specific_fields = item.get("provider_specific_fields")
if isinstance(provider_specific_fields, dict) and provider_specific_fields:
tool_call_dict["provider_specific_fields"] = provider_specific_fields
tool_call_dict["function"]["provider_specific_fields"] = provider_specific_fields
return tool_call_dict
def _reasoning_item_to_response_input(
r_item: Union[ChatCompletionReasoningItem, Dict[str, Any]],
) -> Dict[str, Any]:
@ -176,36 +202,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
choice = Choices(message=msg, finish_reason="stop", index=index)
return choice, index + 1
# Handle function_call items (e.g., from GPT-5 Codex format)
if item_type == "function_call":
# Extract provider_specific_fields if present and pass through as-is
provider_specific_fields = item.get("provider_specific_fields")
if provider_specific_fields and not isinstance(provider_specific_fields, dict):
provider_specific_fields = (
dict(provider_specific_fields) if hasattr(provider_specific_fields, "__dict__") else {}
)
tool_call_dict = {
"id": item.get("call_id") or item.get("id", ""),
"function": {
"name": item.get("name", ""),
"arguments": item.get("arguments", ""),
},
"type": "function",
}
# Pass through provider_specific_fields as-is if present
if provider_specific_fields:
tool_call_dict["provider_specific_fields"] = provider_specific_fields
# Also add to function's provider_specific_fields for consistency
tool_call_dict["function"]["provider_specific_fields"] = provider_specific_fields
msg = Message(
content=None,
tool_calls=[tool_call_dict],
)
choice = Choices(message=msg, finish_reason="tool_calls", index=index)
return choice, index + 1
# function_call / custom_tool_call dicts are intercepted and accumulated by
# _convert_response_output_to_choices before this callback is reached
# Unknown or unsupported type
return None, index
@ -562,11 +560,21 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
accumulated_tool_calls.append(tool_call_dict)
tool_call_index += 1
elif isinstance(item, dict) and handle_raw_dict_callback is not None:
# Handle raw dict responses (e.g., from GPT-5 Codex)
choice, index = handle_raw_dict_callback(item=item, index=index)
if choice is not None:
choices.append(choice)
elif isinstance(item, (dict, BaseModel)):
# Raw dict items (e.g., from GPT-5 Codex) and pydantic items matching no
# openai SDK class above: typed ResponseCustomToolCall and litellm's own
# GenericResponseOutputItem from the completion bridge both land here
raw_item = item if isinstance(item, dict) else item.model_dump()
if raw_item.get("type") in ("function_call", "custom_tool_call"):
# Tool calls accumulate into the single trailing tool_calls choice
# like the typed branches above; a choice per call would hide every
# call after choices[0] from chat clients
accumulated_tool_calls.append(_tool_call_dict_from_output_item(raw_item))
tool_call_index += 1
elif handle_raw_dict_callback is not None:
choice, index = handle_raw_dict_callback(item=raw_item, index=index)
if choice is not None:
choices.append(choice)
else:
pass # don't fail request if item in list is not supported
@ -1078,6 +1086,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
def __init__(self, streaming_response, sync_stream: bool, json_mode: Optional[bool] = False):
super().__init__(streaming_response, sync_stream, json_mode)
self._chat_completion_id: str | None = None
self._tool_call_index_map: dict[int, int] = {}
def _handle_string_chunk(
self, str_line: Union[str, "BaseModel"]
@ -1096,15 +1105,35 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
return self.chunk_parser(json.loads(str_line))
@staticmethod
def _sequential_tool_call_index(
tool_call_index_map: dict[int, int] | None,
output_index: int,
) -> int:
"""Chat-completions tool_call indices must be 0-based and sequential, but
Responses API ``output_index`` counts every output item (reasoning,
message, ...), so the first tool call of a reasoning model arrives at
output_index >= 1 and strict SSE accumulators (e.g. Cursor agent mode)
misplace it. When a per-stream map is provided, remap each distinct
output_index to the next sequential slot; without a map (stateless
callers), fall back to the raw output_index."""
if tool_call_index_map is None:
return output_index
if output_index not in tool_call_index_map:
tool_call_index_map[output_index] = len(tool_call_index_map) # mutable-ok: per-stream accumulator state
return tool_call_index_map[output_index]
@staticmethod
def translate_responses_chunk_to_openai_stream(
parsed_chunk: Union[dict, BaseModel],
tool_call_index_map: dict[int, int] | None = None,
) -> "ModelResponseStream":
"""
Translate a Responses API streaming chunk to OpenAI chat completion streaming format.
Args:
parsed_chunk: Dict containing the Responses API event chunk
tool_call_index_map: Per-stream output_index -> sequential tool_call index map
Returns:
ModelResponseStream: OpenAI-formatted streaming chunk
@ -1165,7 +1194,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
function_chunk = ChatCompletionToolCallFunctionChunk(
name=output_item.get("name", None),
arguments=parsed_chunk.get("arguments", ""),
arguments=output_item.get("arguments") or parsed_chunk.get("arguments") or "",
)
if provider_specific_fields:
@ -1175,7 +1204,9 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
LiteLLMCompletionResponsesConfig,
)
tool_call_index = parsed_chunk.get("output_index", 0)
tool_call_index = OpenAiResponsesToChatCompletionStreamIterator._sequential_tool_call_index(
tool_call_index_map, parsed_chunk.get("output_index", 0)
)
tool_call_chunk = ChatCompletionToolCallChunk(
id=LiteLLMCompletionResponsesConfig._tool_call_id_from_responses_item(
output_item.get("id"), output_item.get("call_id")
@ -1198,10 +1229,41 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
)
]
)
elif event_type == "response.function_call_arguments.delta":
if output_item.get("type") == "custom_tool_call":
tool_call_index = OpenAiResponsesToChatCompletionStreamIterator._sequential_tool_call_index(
tool_call_index_map, parsed_chunk.get("output_index", 0)
)
converted = _tool_call_dict_from_output_item(output_item)
return ModelResponseStream(
choices=[
StreamingChoices(
index=0,
delta=Delta(
tool_calls=[
ChatCompletionToolCallChunk(
id=converted["id"],
index=tool_call_index,
type="function",
function=ChatCompletionToolCallFunctionChunk(
name=converted["function"]["name"],
arguments=converted["function"]["arguments"],
),
)
]
),
finish_reason=None,
)
]
)
elif event_type in (
ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA,
ResponsesAPIStreamEvents.CUSTOM_TOOL_CALL_INPUT_DELTA,
):
content_part: Optional[str] = parsed_chunk.get("delta", None)
if content_part:
tool_call_index = parsed_chunk.get("output_index", 0)
tool_call_index = OpenAiResponsesToChatCompletionStreamIterator._sequential_tool_call_index(
tool_call_index_map, parsed_chunk.get("output_index", 0)
)
return ModelResponseStream(
choices=[
StreamingChoices(
@ -1225,39 +1287,12 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
elif event_type == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE:
# New output item added
output_item = parsed_chunk.get("item", {})
if output_item.get("type") == "function_call":
# Extract provider_specific_fields if present
provider_specific_fields = output_item.get("provider_specific_fields")
if provider_specific_fields and not isinstance(provider_specific_fields, dict):
provider_specific_fields = (
dict(provider_specific_fields) if hasattr(provider_specific_fields, "__dict__") else {}
)
function_chunk = ChatCompletionToolCallFunctionChunk(
name=output_item.get("name", None),
arguments="", # responses API sends everything again, we don't
)
# Add provider_specific_fields to function if present
if provider_specific_fields:
function_chunk["provider_specific_fields"] = provider_specific_fields
tool_call_index = parsed_chunk.get("output_index", 0)
tool_call_chunk = ChatCompletionToolCallChunk(
id=output_item.get("call_id"),
index=tool_call_index,
type="function",
function=function_chunk,
)
# Add provider_specific_fields if present
if provider_specific_fields:
tool_call_chunk.provider_specific_fields = provider_specific_fields # type: ignore
if output_item.get("type") in ("function_call", "custom_tool_call"):
# Do NOT emit finish_reason here — response.completed handles the terminal
# finish_reason. Emitting "tool_calls" here would prematurely terminate
# the stream before subsequent tool calls arrive (same fix as #17246 for
# the message-type branch).
# the message-type branch). The item's fields were already streamed via
# output_item.added and the argument delta events.
return ModelResponseStream(
choices=[
StreamingChoices(
@ -1316,7 +1351,9 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
output_items = response_data.get("output", []) if response_data else []
has_function_calls = any(
item.get("type") == "function_call" for item in output_items if isinstance(item, dict)
item.get("type") in ("function_call", "custom_tool_call")
for item in output_items
if isinstance(item, dict)
)
finish_reason = "tool_calls" if has_function_calls else "stop"
@ -1386,7 +1423,9 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
"""
verbose_logger.debug(f"Chat provider: transform_streaming_response called with chunk: {chunk}")
return self._with_stream_scoped_id(
OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream(chunk)
OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream(
chunk, tool_call_index_map=self._tool_call_index_map
)
)
def _with_stream_scoped_id(self, chunk: "ModelResponseStream") -> "ModelResponseStream":

49
litellm/proxy/response_api_endpoints/endpoints.py
View file

@ -294,11 +294,15 @@ async def cursor_chat_completions(
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
Cursor-specific endpoint that accepts Responses API input format but returns chat completions format.
This endpoint handles requests from Cursor IDE which sends Responses API format (`input` field)
but expects chat completions format response (`choices`, `messages`, etc.).
Cursor BYOK endpoint. Accepts both request shapes Cursor sends to its OpenAI-compatible
base URL and always answers in chat completions format.
Cursor agent mode sends Responses API format bodies (`input`, flat tool defs, `reasoning`,
custom tools) to the chat/completions path while expecting chat completions responses;
those are routed through the Responses API pipeline and converted back. Genuine chat
completions bodies (`messages` present) are routed through the standard chat completions
pipeline untouched.
```bash
curl -X POST http://localhost:4000/cursor/chat/completions \
-H "Content-Type: application/json" \
@ -317,6 +321,7 @@ async def cursor_chat_completions(
from litellm.proxy.proxy_server import (
_read_request_body,
async_data_generator,
chat_completion,
general_settings,
llm_router,
proxy_config,
@ -328,20 +333,28 @@ async def cursor_chat_completions(
user_temperature,
version,
)
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import ModelResponse
data = await _read_request_body(request=request)
# Convert 'messages' to 'input' for Responses API compatibility
# Cursor sends 'messages' but Responses API expects 'input'
if "messages" in data and "input" not in data:
data["input"] = data.pop("messages")
if "messages" in data:
# Genuine chat completions body (Cursor sends these for models whose BYOK it
# already fixed); delegate so behavior matches /chat/completions exactly
return await chat_completion(
request=request,
fastapi_response=fastapi_response,
model=None,
user_api_key_dict=user_api_key_dict,
)
# OpenAI's Responses API rejects chat-completions-only stream_options
# (Cursor sends include_usage); usage arrives via response.completed anyway
data.pop("stream_options", None)
processor = ProxyBaseLLMRequestProcessing(data=data)
def cursor_data_generator(response, user_api_key_dict, request_data):
def cursor_data_generator(response, user_api_key_dict, request_data, request=None):
"""
Custom generator that transforms Responses API streaming chunks to chat completion chunks.
@ -349,17 +362,21 @@ async def cursor_chat_completions(
to chat completion format that Cursor IDE expects.
Args:
response: The streaming response (BaseResponsesAPIStreamingIterator or other)
response: The streaming Responses API event iterator (router-wrapped or not)
user_api_key_dict: User API key authentication dict
request_data: Request data containing model, logging_obj, etc.
request: The originating FastAPI request, forwarded for disconnect handling
Returns:
Async generator that yields SSE-formatted chat completion chunks
"""
# If response is a BaseResponsesAPIStreamingIterator, transform it first
if isinstance(response, BaseResponsesAPIStreamingIterator):
# Any async-iterable here is a Responses API event stream needing conversion.
# Class-identity checks miss router-wrapped streams (e.g.
# HiddenParamsAsyncIteratorWrapper around LiteLLMCompletionStreamingIterator),
# which previously leaked raw Responses events to the client.
if hasattr(response, "__anext__"):
# Transform Responses API iterator to chat completion iterator
# Cast to AsyncIterator[str] since BaseResponsesAPIStreamingIterator implements __aiter__/__anext__
# Cast to AsyncIterator[str] since the stream implements __aiter__/__anext__
completion_stream = responses_api_bridge.transformation_handler.get_model_response_iterator(
streaming_response=cast(AsyncIterator[str], response),
sync_stream=False,
@ -378,12 +395,14 @@ async def cursor_chat_completions(
response=streamwrapper,
user_api_key_dict=user_api_key_dict,
request_data=request_data,
request=request,
)
# Otherwise, use the default generator
return async_data_generator(
response=response,
user_api_key_dict=user_api_key_dict,
request_data=request_data,
request=request,
)
try:

4
litellm/types/llms/openai.py
View file

@ -1405,6 +1405,10 @@ class ResponsesAPIStreamEvents(str, Enum):
FUNCTION_CALL_ARGUMENTS_DELTA = "response.function_call_arguments.delta"
FUNCTION_CALL_ARGUMENTS_DONE = "response.function_call_arguments.done"
# Custom tool call events (grammar/freeform tools, e.g. Cursor agent tools)
CUSTOM_TOOL_CALL_INPUT_DELTA = "response.custom_tool_call_input.delta"
CUSTOM_TOOL_CALL_INPUT_DONE = "response.custom_tool_call_input.done"
# File search events
FILE_SEARCH_CALL_IN_PROGRESS = "response.file_search_call.in_progress"
FILE_SEARCH_CALL_SEARCHING = "response.file_search_call.searching"

253
tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py
View file

@ -2962,3 +2962,256 @@ async def test_acompletion_bridge_normalizes_stream_options_on_the_wire(
assert "stream_options" not in request_body
else:
assert request_body["stream_options"] == expected_wire_stream_options
def test_chunk_parser_custom_tool_call_stream_sequence():
"""Cursor agent mode drives grammar/freeform ``custom_tool_call`` items (e.g. its
ApplyPatch tool). The stream converter must surface them as chat-completions
tool_call deltas: the added event opens the call (id from ``call_id``, name, empty
arguments), each ``custom_tool_call_input.delta`` streams arguments, the done event
must NOT finish the stream, and ``response.completed`` must report
finish_reason="tool_calls". Before the fix every one of these events fell through
to an empty-content chunk and the completed event said "stop", so Cursor never saw
the tool call and agent mode stalled."""
from litellm.completion_extras.litellm_responses_transformation.transformation import (
OpenAiResponsesToChatCompletionStreamIterator,
)
iterator = OpenAiResponsesToChatCompletionStreamIterator(
streaming_response=None, sync_stream=True
)
added = iterator.chunk_parser(
{
"type": "response.output_item.added",
"output_index": 1,
"item": {
"type": "custom_tool_call",
"id": "ctc_1",
"call_id": "call_patch1",
"name": "ApplyPatch",
"input": "",
},
}
)
tool_call = added.choices[0].delta.tool_calls[0]
assert tool_call.id == "call_patch1"
assert tool_call.type == "function"
assert tool_call.function.name == "ApplyPatch"
assert tool_call.function.arguments == ""
assert tool_call.index == 0
assert added.choices[0].finish_reason is None
delta = iterator.chunk_parser(
{
"type": "response.custom_tool_call_input.delta",
"output_index": 1,
"delta": "*** Begin Patch",
}
)
delta_tool_call = delta.choices[0].delta.tool_calls[0]
assert delta_tool_call.function.arguments == "*** Begin Patch"
assert delta_tool_call.index == 0
assert delta.choices[0].finish_reason is None
done = iterator.chunk_parser(
{
"type": "response.output_item.done",
"output_index": 1,
"item": {
"type": "custom_tool_call",
"call_id": "call_patch1",
"name": "ApplyPatch",
"input": "*** Begin Patch",
},
}
)
assert done.choices[0].finish_reason is None
completed = iterator.chunk_parser(
{
"type": "response.completed",
"response": {
"output": [
{"type": "reasoning", "id": "rs_1"},
{"type": "custom_tool_call", "call_id": "call_patch1"},
],
"usage": {"input_tokens": 7, "output_tokens": 3, "total_tokens": 10},
},
}
)
assert completed.choices[0].finish_reason == "tool_calls"
assert completed.usage is not None
assert completed.usage.total_tokens == 10
def test_chunk_parser_remaps_tool_call_indices_sequentially():
"""Responses API output_index counts every output item, so a reasoning model's
first tool call arrives at output_index >= 1. Chat-completions clients accumulate
streamed tool_calls by index and expect the first call at 0; Cursor agent mode
misplaces calls when indices start above 0 (the community BYOK bridge assigns its
own sequential indices for the same reason). The iterator must remap each distinct
output_index to the next sequential slot and route argument deltas to the mapped
slot."""
from litellm.completion_extras.litellm_responses_transformation.transformation import (
OpenAiResponsesToChatCompletionStreamIterator,
)
iterator = OpenAiResponsesToChatCompletionStreamIterator(
streaming_response=None, sync_stream=True
)
first = iterator.chunk_parser(
{
"type": "response.output_item.added",
"output_index": 2,
"item": {
"type": "function_call",
"id": "fc_1",
"call_id": "call_read1",
"name": "read_file",
"arguments": "",
},
}
)
assert first.choices[0].delta.tool_calls[0].index == 0
first_args = iterator.chunk_parser(
{
"type": "response.function_call_arguments.delta",
"output_index": 2,
"delta": '{"path":',
}
)
assert first_args.choices[0].delta.tool_calls[0].index == 0
second = iterator.chunk_parser(
{
"type": "response.output_item.added",
"output_index": 4,
"item": {
"type": "function_call",
"id": "fc_2",
"call_id": "call_grep1",
"name": "grep",
"arguments": "",
},
}
)
assert second.choices[0].delta.tool_calls[0].index == 1
second_args = iterator.chunk_parser(
{
"type": "response.function_call_arguments.delta",
"output_index": 4,
"delta": '{"pattern":',
}
)
assert second_args.choices[0].delta.tool_calls[0].index == 1
def test_convert_response_output_custom_tool_call_to_tool_calls_choice():
"""Non-streaming twin of the custom_tool_call fix: a typed ResponseCustomToolCall
output item must become a chat tool_call (arguments = the raw custom input string,
id = call_id) in a finish_reason="tool_calls" choice instead of being silently
dropped, which left Cursor agent mode with an empty assistant message."""
from openai.types.responses import ResponseCustomToolCall
from litellm.completion_extras.litellm_responses_transformation.transformation import (
LiteLLMResponsesTransformationHandler,
)
item = ResponseCustomToolCall(
type="custom_tool_call",
id="ctc_9",
call_id="call_custom9",
name="ApplyPatch",
input="*** Begin Patch\n*** End Patch",
)
choices = LiteLLMResponsesTransformationHandler._convert_response_output_to_choices([item])
assert len(choices) == 1
choice = choices[0]
assert choice.finish_reason == "tool_calls"
tool_call = choice.message.tool_calls[0]
assert tool_call.id == "call_custom9"
assert tool_call.function.name == "ApplyPatch"
assert tool_call.function.arguments == "*** Begin Patch\n*** End Patch"
def test_convert_response_output_accumulates_raw_tool_calls_into_one_choice():
"""Raw dict and generic-pydantic tool-call items must accumulate into the single
trailing tool_calls choice exactly like typed items. Emitting one choice per tool
call (the old raw-dict behavior) hid every call after choices[0] from chat
clients, which read only the first choice; a multi-tool agent turn through the
completion bridge lost all but one call."""
from litellm.completion_extras.litellm_responses_transformation.transformation import (
LiteLLMResponsesTransformationHandler,
)
handler = LiteLLMResponsesTransformationHandler()
items = [
{
"type": "function_call",
"id": "fc_1",
"call_id": "call_read42",
"name": "read_file",
"arguments": '{"path": "a.py"}',
},
{
"type": "custom_tool_call",
"id": "ctc_1",
"call_id": "call_patch42",
"name": "ApplyPatch",
"input": "*** Begin Patch",
},
]
choices = LiteLLMResponsesTransformationHandler._convert_response_output_to_choices(
items,
handle_raw_dict_callback=handler._handle_raw_dict_response_item,
)
assert len(choices) == 1
choice = choices[0]
assert choice.finish_reason == "tool_calls"
tool_calls = choice.message.tool_calls
assert len(tool_calls) == 2
assert tool_calls[0].id == "call_read42"
assert tool_calls[0].function.name == "read_file"
assert tool_calls[0].function.arguments == '{"path": "a.py"}'
assert tool_calls[1].id == "call_patch42"
assert tool_calls[1].function.name == "ApplyPatch"
assert tool_calls[1].function.arguments == "*** Begin Patch"
def test_convert_response_output_generic_pydantic_message_item():
"""litellm's completion bridge (used for non-Responses-native providers behind the
router) emits GenericResponseOutputItem pydantic models rather than openai SDK
classes. The converter must normalize unrecognized pydantic items through the
raw-dict handler instead of dropping them; dropping them made transform_response
raise 'Unknown items in responses API response' on an otherwise-successful
completion (hit live via /cursor/chat/completions multi-turn tool round trips)."""
from litellm.completion_extras.litellm_responses_transformation.transformation import (
LiteLLMResponsesTransformationHandler,
)
from litellm.types.responses.main import GenericResponseOutputItem, OutputText
handler = LiteLLMResponsesTransformationHandler()
item = GenericResponseOutputItem(
type="message",
id="msg_generic1",
status="completed",
role="assistant",
content=[OutputText(type="output_text", text="42", annotations=[])],
)
choices = LiteLLMResponsesTransformationHandler._convert_response_output_to_choices(
[item],
handle_raw_dict_callback=handler._handle_raw_dict_response_item,
)
assert len(choices) == 1
assert choices[0].message.content == "42"
assert choices[0].finish_reason == "stop"

147
tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py
View file

@ -8,6 +8,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from fastapi.testclient import TestClient
import litellm
from litellm.proxy.proxy_server import app
@ -711,3 +712,149 @@ class TestManagedResponsesSameProvider:
call_kwargs: dict = {}
handler._inject_credentials(call_kwargs, model="vertex_ai/gemini-2.0-flash")
assert "custom_llm_provider" not in call_kwargs
def _auth_override():
from litellm.proxy._types import UserAPIKeyAuth
return UserAPIKeyAuth(api_key="sk-test-cursor", user_id="cursor-user")
def test_cursor_chat_completions_messages_body_uses_chat_pipeline():
"""A genuine chat-completions body (``messages`` present; what Cursor sends for
models whose BYOK it already fixed) must run through the standard chat pipeline
untouched: multi-turn tool history (assistant tool_calls + role="tool" results)
and nested chat-format tool defs are valid there, while blindly renaming
``messages`` to ``input`` (the pre-fix behavior) produced items the Responses API
rejects. Asserts acompletion is called with the exact messages and aresponses is
never touched."""
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
import litellm.proxy.proxy_server as ps
messages = [
{"role": "user", "content": "read a file"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_hist1",
"type": "function",
"function": {"name": "read_file", "arguments": '{"path": "a.py"}'},
}
],
},
{"role": "tool", "tool_call_id": "call_hist1", "content": "file contents"},
{"role": "user", "content": "now summarize"},
]
mock_router = MagicMock()
mock_router.acompletion = AsyncMock(
return_value=litellm.ModelResponse(
id="chatcmpl-cursor-1",
choices=[
{
"index": 0,
"message": {"role": "assistant", "content": "summary"},
"finish_reason": "stop",
}
],
model="gpt-4o",
)
)
mock_router.aresponses = AsyncMock()
mock_router.get_available_deployment = MagicMock(return_value=None)
app.dependency_overrides[user_api_key_auth] = _auth_override
try:
with patch.object(ps, "llm_router", mock_router):
client = TestClient(app)
response = client.post(
"/cursor/chat/completions",
json={
"model": "gpt-4o",
"messages": messages,
"tools": [
{
"type": "function",
"function": {"name": "read_file", "parameters": {"type": "object"}},
}
],
},
headers={"Authorization": "Bearer sk-test-cursor"},
)
finally:
app.dependency_overrides.pop(user_api_key_auth, None)
assert response.status_code == 200, response.text
body = response.json()
assert body["choices"][0]["message"]["content"] == "summary"
assert "output" not in body
mock_router.acompletion.assert_called_once()
called_kwargs = mock_router.acompletion.call_args.kwargs
assert called_kwargs["messages"] == messages
assert "input" not in called_kwargs
mock_router.aresponses.assert_not_called()
def test_cursor_chat_completions_input_body_uses_responses_pipeline_and_strips_stream_options():
"""A Responses-shaped body (``input``, no ``messages``; what Cursor agent mode
sends) must run through the Responses pipeline with chat-completions output, and
``stream_options`` (chat-completions-only; Cursor sends include_usage) must be
stripped before the Responses call since OpenAI's Responses API rejects it."""
from openai.types.responses import ResponseOutputMessage, ResponseOutputText
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.types.llms.openai import ResponsesAPIResponse
import litellm.proxy.proxy_server as ps
mock_router = MagicMock()
mock_router.aresponses = AsyncMock(
return_value=ResponsesAPIResponse(
id="resp_cursor_agent1",
created_at=1234567890,
model="gpt-4o",
object="response",
output=[
ResponseOutputMessage(
id="msg_agent1",
type="message",
role="assistant",
status="completed",
content=[
ResponseOutputText(type="output_text", text="agent reply", annotations=[])
],
)
],
)
)
mock_router.acompletion = AsyncMock()
app.dependency_overrides[user_api_key_auth] = _auth_override
try:
with patch.object(ps, "llm_router", mock_router):
client = TestClient(app)
response = client.post(
"/cursor/chat/completions",
json={
"model": "gpt-4o",
"input": [{"role": "user", "content": "hello"}],
"stream_options": {"include_usage": True},
},
headers={"Authorization": "Bearer sk-test-cursor"},
)
finally:
app.dependency_overrides.pop(user_api_key_auth, None)
assert response.status_code == 200, response.text
body = response.json()
assert body["choices"][0]["message"]["content"] == "agent reply"
assert "output" not in body
mock_router.aresponses.assert_called_once()
called_kwargs = mock_router.aresponses.call_args.kwargs
assert "stream_options" not in called_kwargs
mock_router.acompletion.assert_not_called()

10
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -2621,10 +2621,14 @@ export interface paths {
put?: never;
/**
* Cursor Chat Completions
* @description Cursor-specific endpoint that accepts Responses API input format but returns chat completions format.
* @description Cursor BYOK endpoint. Accepts both request shapes Cursor sends to its OpenAI-compatible
* base URL and always answers in chat completions format.
*
* This endpoint handles requests from Cursor IDE which sends Responses API format (`input` field)
* but expects chat completions format response (`choices`, `messages`, etc.).
* Cursor agent mode sends Responses API format bodies (`input`, flat tool defs, `reasoning`,
* custom tools) to the chat/completions path while expecting chat completions responses;
* those are routed through the Responses API pipeline and converted back. Genuine chat
* completions bodies (`messages` present) are routed through the standard chat completions
* pipeline untouched.
*
* ```bash
* curl -X POST http://localhost:4000/cursor/chat/completions -H "Content-Type: application/json" -H "Authorization: Bearer sk-1234" -d '{