From 14c97ba8db1cb4172e3276ba191c962be8a1cc11 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Mon, 20 Jul 2026 13:48:24 -0700 Subject: [PATCH] fix(proxy): make /cursor/chat/completions work with Cursor agent mode - delegate messages-shaped bodies to the standard chat completions handler - strip chat-only stream_options before the Responses pipeline - fix cursor_data_generator signature (request kwarg) and duck-type the stream gate so router-wrapped streams convert instead of leaking raw Responses events - convert custom_tool_call items and events to chat tool_calls in the streaming and non-streaming paths; remap streamed tool_call indices to 0-based sequential; accumulate raw and pydantic tool calls into one choice - normalize generic pydantic output items through the raw-dict handler --- .../transformation.py | 181 ++++++++----- .../proxy/response_api_endpoints/endpoints.py | 49 ++-- litellm/types/llms/openai.py | 4 + ...responses_transformation_transformation.py | 253 ++++++++++++++++++ .../response_api_endpoints/test_endpoints.py | 147 ++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 10 +- 6 files changed, 555 insertions(+), 89 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 89a44fcdeef..d1df75bde36 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -100,6 +100,32 @@ def _build_reasoning_item( } +def _tool_call_dict_from_output_item(item: dict[str, Any]) -> dict[str, Any]: + """Convert a ``function_call`` or ``custom_tool_call`` output item dict to a chat + completions tool_call dict. Custom (grammar/freeform) tool calls carry their raw + string payload in ``input`` rather than ``arguments``; both map to + ``function.arguments`` so chat clients (e.g. Cursor agent mode) receive them like + any other tool call. The single conversion rule shared by the non-streaming + accumulator and the streaming ``output_item.added`` branch.""" + from litellm.responses.litellm_completion_transformation.transformation import ( + LiteLLMCompletionResponsesConfig, + ) + + is_custom = item.get("type") == "custom_tool_call" + arguments = (item.get("input") if is_custom else item.get("arguments")) or "" + name = item.get("name") or ("custom_tool" if is_custom else "") + tool_call_dict: dict[str, Any] = { + "id": LiteLLMCompletionResponsesConfig._tool_call_id_from_responses_item(item.get("id"), item.get("call_id")), + "function": {"name": name, "arguments": arguments}, + "type": "function", + } + provider_specific_fields = item.get("provider_specific_fields") + if isinstance(provider_specific_fields, dict) and provider_specific_fields: + tool_call_dict["provider_specific_fields"] = provider_specific_fields + tool_call_dict["function"]["provider_specific_fields"] = provider_specific_fields + return tool_call_dict + + def _reasoning_item_to_response_input( r_item: Union[ChatCompletionReasoningItem, Dict[str, Any]], ) -> Dict[str, Any]: @@ -176,36 +202,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): choice = Choices(message=msg, finish_reason="stop", index=index) return choice, index + 1 - # Handle function_call items (e.g., from GPT-5 Codex format) - if item_type == "function_call": - # Extract provider_specific_fields if present and pass through as-is - provider_specific_fields = item.get("provider_specific_fields") - if provider_specific_fields and not isinstance(provider_specific_fields, dict): - provider_specific_fields = ( - dict(provider_specific_fields) if hasattr(provider_specific_fields, "__dict__") else {} - ) - - tool_call_dict = { - "id": item.get("call_id") or item.get("id", ""), - "function": { - "name": item.get("name", ""), - "arguments": item.get("arguments", ""), - }, - "type": "function", - } - - # Pass through provider_specific_fields as-is if present - if provider_specific_fields: - tool_call_dict["provider_specific_fields"] = provider_specific_fields - # Also add to function's provider_specific_fields for consistency - tool_call_dict["function"]["provider_specific_fields"] = provider_specific_fields - - msg = Message( - content=None, - tool_calls=[tool_call_dict], - ) - choice = Choices(message=msg, finish_reason="tool_calls", index=index) - return choice, index + 1 + # function_call / custom_tool_call dicts are intercepted and accumulated by + # _convert_response_output_to_choices before this callback is reached # Unknown or unsupported type return None, index @@ -562,11 +560,21 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): accumulated_tool_calls.append(tool_call_dict) tool_call_index += 1 - elif isinstance(item, dict) and handle_raw_dict_callback is not None: - # Handle raw dict responses (e.g., from GPT-5 Codex) - choice, index = handle_raw_dict_callback(item=item, index=index) - if choice is not None: - choices.append(choice) + elif isinstance(item, (dict, BaseModel)): + # Raw dict items (e.g., from GPT-5 Codex) and pydantic items matching no + # openai SDK class above: typed ResponseCustomToolCall and litellm's own + # GenericResponseOutputItem from the completion bridge both land here + raw_item = item if isinstance(item, dict) else item.model_dump() + if raw_item.get("type") in ("function_call", "custom_tool_call"): + # Tool calls accumulate into the single trailing tool_calls choice + # like the typed branches above; a choice per call would hide every + # call after choices[0] from chat clients + accumulated_tool_calls.append(_tool_call_dict_from_output_item(raw_item)) + tool_call_index += 1 + elif handle_raw_dict_callback is not None: + choice, index = handle_raw_dict_callback(item=raw_item, index=index) + if choice is not None: + choices.append(choice) else: pass # don't fail request if item in list is not supported @@ -1078,6 +1086,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): def __init__(self, streaming_response, sync_stream: bool, json_mode: Optional[bool] = False): super().__init__(streaming_response, sync_stream, json_mode) self._chat_completion_id: str | None = None + self._tool_call_index_map: dict[int, int] = {} def _handle_string_chunk( self, str_line: Union[str, "BaseModel"] @@ -1096,15 +1105,35 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): return self.chunk_parser(json.loads(str_line)) + @staticmethod + def _sequential_tool_call_index( + tool_call_index_map: dict[int, int] | None, + output_index: int, + ) -> int: + """Chat-completions tool_call indices must be 0-based and sequential, but + Responses API ``output_index`` counts every output item (reasoning, + message, ...), so the first tool call of a reasoning model arrives at + output_index >= 1 and strict SSE accumulators (e.g. Cursor agent mode) + misplace it. When a per-stream map is provided, remap each distinct + output_index to the next sequential slot; without a map (stateless + callers), fall back to the raw output_index.""" + if tool_call_index_map is None: + return output_index + if output_index not in tool_call_index_map: + tool_call_index_map[output_index] = len(tool_call_index_map) # mutable-ok: per-stream accumulator state + return tool_call_index_map[output_index] + @staticmethod def translate_responses_chunk_to_openai_stream( parsed_chunk: Union[dict, BaseModel], + tool_call_index_map: dict[int, int] | None = None, ) -> "ModelResponseStream": """ Translate a Responses API streaming chunk to OpenAI chat completion streaming format. Args: parsed_chunk: Dict containing the Responses API event chunk + tool_call_index_map: Per-stream output_index -> sequential tool_call index map Returns: ModelResponseStream: OpenAI-formatted streaming chunk @@ -1165,7 +1194,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): function_chunk = ChatCompletionToolCallFunctionChunk( name=output_item.get("name", None), - arguments=parsed_chunk.get("arguments", ""), + arguments=output_item.get("arguments") or parsed_chunk.get("arguments") or "", ) if provider_specific_fields: @@ -1175,7 +1204,9 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): LiteLLMCompletionResponsesConfig, ) - tool_call_index = parsed_chunk.get("output_index", 0) + tool_call_index = OpenAiResponsesToChatCompletionStreamIterator._sequential_tool_call_index( + tool_call_index_map, parsed_chunk.get("output_index", 0) + ) tool_call_chunk = ChatCompletionToolCallChunk( id=LiteLLMCompletionResponsesConfig._tool_call_id_from_responses_item( output_item.get("id"), output_item.get("call_id") @@ -1198,10 +1229,41 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): ) ] ) - elif event_type == "response.function_call_arguments.delta": + if output_item.get("type") == "custom_tool_call": + tool_call_index = OpenAiResponsesToChatCompletionStreamIterator._sequential_tool_call_index( + tool_call_index_map, parsed_chunk.get("output_index", 0) + ) + converted = _tool_call_dict_from_output_item(output_item) + return ModelResponseStream( + choices=[ + StreamingChoices( + index=0, + delta=Delta( + tool_calls=[ + ChatCompletionToolCallChunk( + id=converted["id"], + index=tool_call_index, + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=converted["function"]["name"], + arguments=converted["function"]["arguments"], + ), + ) + ] + ), + finish_reason=None, + ) + ] + ) + elif event_type in ( + ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA, + ResponsesAPIStreamEvents.CUSTOM_TOOL_CALL_INPUT_DELTA, + ): content_part: Optional[str] = parsed_chunk.get("delta", None) if content_part: - tool_call_index = parsed_chunk.get("output_index", 0) + tool_call_index = OpenAiResponsesToChatCompletionStreamIterator._sequential_tool_call_index( + tool_call_index_map, parsed_chunk.get("output_index", 0) + ) return ModelResponseStream( choices=[ StreamingChoices( @@ -1225,39 +1287,12 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): elif event_type == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE: # New output item added output_item = parsed_chunk.get("item", {}) - if output_item.get("type") == "function_call": - # Extract provider_specific_fields if present - provider_specific_fields = output_item.get("provider_specific_fields") - if provider_specific_fields and not isinstance(provider_specific_fields, dict): - provider_specific_fields = ( - dict(provider_specific_fields) if hasattr(provider_specific_fields, "__dict__") else {} - ) - - function_chunk = ChatCompletionToolCallFunctionChunk( - name=output_item.get("name", None), - arguments="", # responses API sends everything again, we don't - ) - - # Add provider_specific_fields to function if present - if provider_specific_fields: - function_chunk["provider_specific_fields"] = provider_specific_fields - - tool_call_index = parsed_chunk.get("output_index", 0) - tool_call_chunk = ChatCompletionToolCallChunk( - id=output_item.get("call_id"), - index=tool_call_index, - type="function", - function=function_chunk, - ) - - # Add provider_specific_fields if present - if provider_specific_fields: - tool_call_chunk.provider_specific_fields = provider_specific_fields # type: ignore - + if output_item.get("type") in ("function_call", "custom_tool_call"): # Do NOT emit finish_reason here — response.completed handles the terminal # finish_reason. Emitting "tool_calls" here would prematurely terminate # the stream before subsequent tool calls arrive (same fix as #17246 for - # the message-type branch). + # the message-type branch). The item's fields were already streamed via + # output_item.added and the argument delta events. return ModelResponseStream( choices=[ StreamingChoices( @@ -1316,7 +1351,9 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): output_items = response_data.get("output", []) if response_data else [] has_function_calls = any( - item.get("type") == "function_call" for item in output_items if isinstance(item, dict) + item.get("type") in ("function_call", "custom_tool_call") + for item in output_items + if isinstance(item, dict) ) finish_reason = "tool_calls" if has_function_calls else "stop" @@ -1386,7 +1423,9 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): """ verbose_logger.debug(f"Chat provider: transform_streaming_response called with chunk: {chunk}") return self._with_stream_scoped_id( - OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream(chunk) + OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream( + chunk, tool_call_index_map=self._tool_call_index_map + ) ) def _with_stream_scoped_id(self, chunk: "ModelResponseStream") -> "ModelResponseStream": diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 05c36406f36..dcd7ba18e7f 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -294,11 +294,15 @@ async def cursor_chat_completions( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ - Cursor-specific endpoint that accepts Responses API input format but returns chat completions format. - - This endpoint handles requests from Cursor IDE which sends Responses API format (`input` field) - but expects chat completions format response (`choices`, `messages`, etc.). - + Cursor BYOK endpoint. Accepts both request shapes Cursor sends to its OpenAI-compatible + base URL and always answers in chat completions format. + + Cursor agent mode sends Responses API format bodies (`input`, flat tool defs, `reasoning`, + custom tools) to the chat/completions path while expecting chat completions responses; + those are routed through the Responses API pipeline and converted back. Genuine chat + completions bodies (`messages` present) are routed through the standard chat completions + pipeline untouched. + ```bash curl -X POST http://localhost:4000/cursor/chat/completions \ -H "Content-Type: application/json" \ @@ -317,6 +321,7 @@ async def cursor_chat_completions( from litellm.proxy.proxy_server import ( _read_request_body, async_data_generator, + chat_completion, general_settings, llm_router, proxy_config, @@ -328,20 +333,28 @@ async def cursor_chat_completions( user_temperature, version, ) - from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import ModelResponse data = await _read_request_body(request=request) - # Convert 'messages' to 'input' for Responses API compatibility - # Cursor sends 'messages' but Responses API expects 'input' - if "messages" in data and "input" not in data: - data["input"] = data.pop("messages") + if "messages" in data: + # Genuine chat completions body (Cursor sends these for models whose BYOK it + # already fixed); delegate so behavior matches /chat/completions exactly + return await chat_completion( + request=request, + fastapi_response=fastapi_response, + model=None, + user_api_key_dict=user_api_key_dict, + ) + + # OpenAI's Responses API rejects chat-completions-only stream_options + # (Cursor sends include_usage); usage arrives via response.completed anyway + data.pop("stream_options", None) processor = ProxyBaseLLMRequestProcessing(data=data) - def cursor_data_generator(response, user_api_key_dict, request_data): + def cursor_data_generator(response, user_api_key_dict, request_data, request=None): """ Custom generator that transforms Responses API streaming chunks to chat completion chunks. @@ -349,17 +362,21 @@ async def cursor_chat_completions( to chat completion format that Cursor IDE expects. Args: - response: The streaming response (BaseResponsesAPIStreamingIterator or other) + response: The streaming Responses API event iterator (router-wrapped or not) user_api_key_dict: User API key authentication dict request_data: Request data containing model, logging_obj, etc. + request: The originating FastAPI request, forwarded for disconnect handling Returns: Async generator that yields SSE-formatted chat completion chunks """ - # If response is a BaseResponsesAPIStreamingIterator, transform it first - if isinstance(response, BaseResponsesAPIStreamingIterator): + # Any async-iterable here is a Responses API event stream needing conversion. + # Class-identity checks miss router-wrapped streams (e.g. + # HiddenParamsAsyncIteratorWrapper around LiteLLMCompletionStreamingIterator), + # which previously leaked raw Responses events to the client. + if hasattr(response, "__anext__"): # Transform Responses API iterator to chat completion iterator - # Cast to AsyncIterator[str] since BaseResponsesAPIStreamingIterator implements __aiter__/__anext__ + # Cast to AsyncIterator[str] since the stream implements __aiter__/__anext__ completion_stream = responses_api_bridge.transformation_handler.get_model_response_iterator( streaming_response=cast(AsyncIterator[str], response), sync_stream=False, @@ -378,12 +395,14 @@ async def cursor_chat_completions( response=streamwrapper, user_api_key_dict=user_api_key_dict, request_data=request_data, + request=request, ) # Otherwise, use the default generator return async_data_generator( response=response, user_api_key_dict=user_api_key_dict, request_data=request_data, + request=request, ) try: diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 314bb653196..0d064006412 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1405,6 +1405,10 @@ class ResponsesAPIStreamEvents(str, Enum): FUNCTION_CALL_ARGUMENTS_DELTA = "response.function_call_arguments.delta" FUNCTION_CALL_ARGUMENTS_DONE = "response.function_call_arguments.done" + # Custom tool call events (grammar/freeform tools, e.g. Cursor agent tools) + CUSTOM_TOOL_CALL_INPUT_DELTA = "response.custom_tool_call_input.delta" + CUSTOM_TOOL_CALL_INPUT_DONE = "response.custom_tool_call_input.done" + # File search events FILE_SEARCH_CALL_IN_PROGRESS = "response.file_search_call.in_progress" FILE_SEARCH_CALL_SEARCHING = "response.file_search_call.searching" diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index a111b932f2c..36c32d4b3a0 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -2962,3 +2962,256 @@ async def test_acompletion_bridge_normalizes_stream_options_on_the_wire( assert "stream_options" not in request_body else: assert request_body["stream_options"] == expected_wire_stream_options + + +def test_chunk_parser_custom_tool_call_stream_sequence(): + """Cursor agent mode drives grammar/freeform ``custom_tool_call`` items (e.g. its + ApplyPatch tool). The stream converter must surface them as chat-completions + tool_call deltas: the added event opens the call (id from ``call_id``, name, empty + arguments), each ``custom_tool_call_input.delta`` streams arguments, the done event + must NOT finish the stream, and ``response.completed`` must report + finish_reason="tool_calls". Before the fix every one of these events fell through + to an empty-content chunk and the completed event said "stop", so Cursor never saw + the tool call and agent mode stalled.""" + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + OpenAiResponsesToChatCompletionStreamIterator, + ) + + iterator = OpenAiResponsesToChatCompletionStreamIterator( + streaming_response=None, sync_stream=True + ) + + added = iterator.chunk_parser( + { + "type": "response.output_item.added", + "output_index": 1, + "item": { + "type": "custom_tool_call", + "id": "ctc_1", + "call_id": "call_patch1", + "name": "ApplyPatch", + "input": "", + }, + } + ) + tool_call = added.choices[0].delta.tool_calls[0] + assert tool_call.id == "call_patch1" + assert tool_call.type == "function" + assert tool_call.function.name == "ApplyPatch" + assert tool_call.function.arguments == "" + assert tool_call.index == 0 + assert added.choices[0].finish_reason is None + + delta = iterator.chunk_parser( + { + "type": "response.custom_tool_call_input.delta", + "output_index": 1, + "delta": "*** Begin Patch", + } + ) + delta_tool_call = delta.choices[0].delta.tool_calls[0] + assert delta_tool_call.function.arguments == "*** Begin Patch" + assert delta_tool_call.index == 0 + assert delta.choices[0].finish_reason is None + + done = iterator.chunk_parser( + { + "type": "response.output_item.done", + "output_index": 1, + "item": { + "type": "custom_tool_call", + "call_id": "call_patch1", + "name": "ApplyPatch", + "input": "*** Begin Patch", + }, + } + ) + assert done.choices[0].finish_reason is None + + completed = iterator.chunk_parser( + { + "type": "response.completed", + "response": { + "output": [ + {"type": "reasoning", "id": "rs_1"}, + {"type": "custom_tool_call", "call_id": "call_patch1"}, + ], + "usage": {"input_tokens": 7, "output_tokens": 3, "total_tokens": 10}, + }, + } + ) + assert completed.choices[0].finish_reason == "tool_calls" + assert completed.usage is not None + assert completed.usage.total_tokens == 10 + + +def test_chunk_parser_remaps_tool_call_indices_sequentially(): + """Responses API output_index counts every output item, so a reasoning model's + first tool call arrives at output_index >= 1. Chat-completions clients accumulate + streamed tool_calls by index and expect the first call at 0; Cursor agent mode + misplaces calls when indices start above 0 (the community BYOK bridge assigns its + own sequential indices for the same reason). The iterator must remap each distinct + output_index to the next sequential slot and route argument deltas to the mapped + slot.""" + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + OpenAiResponsesToChatCompletionStreamIterator, + ) + + iterator = OpenAiResponsesToChatCompletionStreamIterator( + streaming_response=None, sync_stream=True + ) + + first = iterator.chunk_parser( + { + "type": "response.output_item.added", + "output_index": 2, + "item": { + "type": "function_call", + "id": "fc_1", + "call_id": "call_read1", + "name": "read_file", + "arguments": "", + }, + } + ) + assert first.choices[0].delta.tool_calls[0].index == 0 + + first_args = iterator.chunk_parser( + { + "type": "response.function_call_arguments.delta", + "output_index": 2, + "delta": '{"path":', + } + ) + assert first_args.choices[0].delta.tool_calls[0].index == 0 + + second = iterator.chunk_parser( + { + "type": "response.output_item.added", + "output_index": 4, + "item": { + "type": "function_call", + "id": "fc_2", + "call_id": "call_grep1", + "name": "grep", + "arguments": "", + }, + } + ) + assert second.choices[0].delta.tool_calls[0].index == 1 + + second_args = iterator.chunk_parser( + { + "type": "response.function_call_arguments.delta", + "output_index": 4, + "delta": '{"pattern":', + } + ) + assert second_args.choices[0].delta.tool_calls[0].index == 1 + + +def test_convert_response_output_custom_tool_call_to_tool_calls_choice(): + """Non-streaming twin of the custom_tool_call fix: a typed ResponseCustomToolCall + output item must become a chat tool_call (arguments = the raw custom input string, + id = call_id) in a finish_reason="tool_calls" choice instead of being silently + dropped, which left Cursor agent mode with an empty assistant message.""" + from openai.types.responses import ResponseCustomToolCall + + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + + item = ResponseCustomToolCall( + type="custom_tool_call", + id="ctc_9", + call_id="call_custom9", + name="ApplyPatch", + input="*** Begin Patch\n*** End Patch", + ) + + choices = LiteLLMResponsesTransformationHandler._convert_response_output_to_choices([item]) + + assert len(choices) == 1 + choice = choices[0] + assert choice.finish_reason == "tool_calls" + tool_call = choice.message.tool_calls[0] + assert tool_call.id == "call_custom9" + assert tool_call.function.name == "ApplyPatch" + assert tool_call.function.arguments == "*** Begin Patch\n*** End Patch" + + +def test_convert_response_output_accumulates_raw_tool_calls_into_one_choice(): + """Raw dict and generic-pydantic tool-call items must accumulate into the single + trailing tool_calls choice exactly like typed items. Emitting one choice per tool + call (the old raw-dict behavior) hid every call after choices[0] from chat + clients, which read only the first choice; a multi-tool agent turn through the + completion bridge lost all but one call.""" + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + + handler = LiteLLMResponsesTransformationHandler() + items = [ + { + "type": "function_call", + "id": "fc_1", + "call_id": "call_read42", + "name": "read_file", + "arguments": '{"path": "a.py"}', + }, + { + "type": "custom_tool_call", + "id": "ctc_1", + "call_id": "call_patch42", + "name": "ApplyPatch", + "input": "*** Begin Patch", + }, + ] + + choices = LiteLLMResponsesTransformationHandler._convert_response_output_to_choices( + items, + handle_raw_dict_callback=handler._handle_raw_dict_response_item, + ) + + assert len(choices) == 1 + choice = choices[0] + assert choice.finish_reason == "tool_calls" + tool_calls = choice.message.tool_calls + assert len(tool_calls) == 2 + assert tool_calls[0].id == "call_read42" + assert tool_calls[0].function.name == "read_file" + assert tool_calls[0].function.arguments == '{"path": "a.py"}' + assert tool_calls[1].id == "call_patch42" + assert tool_calls[1].function.name == "ApplyPatch" + assert tool_calls[1].function.arguments == "*** Begin Patch" + + +def test_convert_response_output_generic_pydantic_message_item(): + """litellm's completion bridge (used for non-Responses-native providers behind the + router) emits GenericResponseOutputItem pydantic models rather than openai SDK + classes. The converter must normalize unrecognized pydantic items through the + raw-dict handler instead of dropping them; dropping them made transform_response + raise 'Unknown items in responses API response' on an otherwise-successful + completion (hit live via /cursor/chat/completions multi-turn tool round trips).""" + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + from litellm.types.responses.main import GenericResponseOutputItem, OutputText + + handler = LiteLLMResponsesTransformationHandler() + item = GenericResponseOutputItem( + type="message", + id="msg_generic1", + status="completed", + role="assistant", + content=[OutputText(type="output_text", text="42", annotations=[])], + ) + + choices = LiteLLMResponsesTransformationHandler._convert_response_output_to_choices( + [item], + handle_raw_dict_callback=handler._handle_raw_dict_response_item, + ) + + assert len(choices) == 1 + assert choices[0].message.content == "42" + assert choices[0].finish_reason == "stop" diff --git a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py index 07d1a9d14f9..9af1ef4766d 100644 --- a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py @@ -8,6 +8,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi.testclient import TestClient +import litellm from litellm.proxy.proxy_server import app @@ -711,3 +712,149 @@ class TestManagedResponsesSameProvider: call_kwargs: dict = {} handler._inject_credentials(call_kwargs, model="vertex_ai/gemini-2.0-flash") assert "custom_llm_provider" not in call_kwargs + + +def _auth_override(): + from litellm.proxy._types import UserAPIKeyAuth + + return UserAPIKeyAuth(api_key="sk-test-cursor", user_id="cursor-user") + + +def test_cursor_chat_completions_messages_body_uses_chat_pipeline(): + """A genuine chat-completions body (``messages`` present; what Cursor sends for + models whose BYOK it already fixed) must run through the standard chat pipeline + untouched: multi-turn tool history (assistant tool_calls + role="tool" results) + and nested chat-format tool defs are valid there, while blindly renaming + ``messages`` to ``input`` (the pre-fix behavior) produced items the Responses API + rejects. Asserts acompletion is called with the exact messages and aresponses is + never touched.""" + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + import litellm.proxy.proxy_server as ps + + messages = [ + {"role": "user", "content": "read a file"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_hist1", + "type": "function", + "function": {"name": "read_file", "arguments": '{"path": "a.py"}'}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_hist1", "content": "file contents"}, + {"role": "user", "content": "now summarize"}, + ] + + mock_router = MagicMock() + mock_router.acompletion = AsyncMock( + return_value=litellm.ModelResponse( + id="chatcmpl-cursor-1", + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "summary"}, + "finish_reason": "stop", + } + ], + model="gpt-4o", + ) + ) + mock_router.aresponses = AsyncMock() + mock_router.get_available_deployment = MagicMock(return_value=None) + + app.dependency_overrides[user_api_key_auth] = _auth_override + try: + with patch.object(ps, "llm_router", mock_router): + client = TestClient(app) + response = client.post( + "/cursor/chat/completions", + json={ + "model": "gpt-4o", + "messages": messages, + "tools": [ + { + "type": "function", + "function": {"name": "read_file", "parameters": {"type": "object"}}, + } + ], + }, + headers={"Authorization": "Bearer sk-test-cursor"}, + ) + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + assert response.status_code == 200, response.text + body = response.json() + assert body["choices"][0]["message"]["content"] == "summary" + assert "output" not in body + + mock_router.acompletion.assert_called_once() + called_kwargs = mock_router.acompletion.call_args.kwargs + assert called_kwargs["messages"] == messages + assert "input" not in called_kwargs + mock_router.aresponses.assert_not_called() + + +def test_cursor_chat_completions_input_body_uses_responses_pipeline_and_strips_stream_options(): + """A Responses-shaped body (``input``, no ``messages``; what Cursor agent mode + sends) must run through the Responses pipeline with chat-completions output, and + ``stream_options`` (chat-completions-only; Cursor sends include_usage) must be + stripped before the Responses call since OpenAI's Responses API rejects it.""" + from openai.types.responses import ResponseOutputMessage, ResponseOutputText + + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.types.llms.openai import ResponsesAPIResponse + + import litellm.proxy.proxy_server as ps + + mock_router = MagicMock() + mock_router.aresponses = AsyncMock( + return_value=ResponsesAPIResponse( + id="resp_cursor_agent1", + created_at=1234567890, + model="gpt-4o", + object="response", + output=[ + ResponseOutputMessage( + id="msg_agent1", + type="message", + role="assistant", + status="completed", + content=[ + ResponseOutputText(type="output_text", text="agent reply", annotations=[]) + ], + ) + ], + ) + ) + mock_router.acompletion = AsyncMock() + + app.dependency_overrides[user_api_key_auth] = _auth_override + try: + with patch.object(ps, "llm_router", mock_router): + client = TestClient(app) + response = client.post( + "/cursor/chat/completions", + json={ + "model": "gpt-4o", + "input": [{"role": "user", "content": "hello"}], + "stream_options": {"include_usage": True}, + }, + headers={"Authorization": "Bearer sk-test-cursor"}, + ) + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + assert response.status_code == 200, response.text + body = response.json() + assert body["choices"][0]["message"]["content"] == "agent reply" + assert "output" not in body + + mock_router.aresponses.assert_called_once() + called_kwargs = mock_router.aresponses.call_args.kwargs + assert "stream_options" not in called_kwargs + mock_router.acompletion.assert_not_called() diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 109d638fb9c..4e0b9b88995 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -2621,10 +2621,14 @@ export interface paths { put?: never; /** * Cursor Chat Completions - * @description Cursor-specific endpoint that accepts Responses API input format but returns chat completions format. + * @description Cursor BYOK endpoint. Accepts both request shapes Cursor sends to its OpenAI-compatible + * base URL and always answers in chat completions format. * - * This endpoint handles requests from Cursor IDE which sends Responses API format (`input` field) - * but expects chat completions format response (`choices`, `messages`, etc.). + * Cursor agent mode sends Responses API format bodies (`input`, flat tool defs, `reasoning`, + * custom tools) to the chat/completions path while expecting chat completions responses; + * those are routed through the Responses API pipeline and converted back. Genuine chat + * completions bodies (`messages` present) are routed through the standard chat completions + * pipeline untouched. * * ```bash * curl -X POST http://localhost:4000/cursor/chat/completions -H "Content-Type: application/json" -H "Authorization: Bearer sk-1234" -d '{