scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-10 03:28:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 85

fix(proxy): normalize each tool shape level independently on the Cursor messages arm

Browse source 
Live Cursor Ask-mode captures show the shape dialects mix PER LEVEL: the
tool envelope arrives chat-nested while the grammar format inside it is
still Responses-flat, so a normalizer that pattern-matches whole-tool
templates misses every hybrid. The cursor arm now normalizes the envelope
level and the format level independently and idempotently, making it
total over the envelope x format matrix; a parametrized 8-cell test pins
every combination. The reference BYOK bridge was checked and forwards
chat bodies verbatim, so there is no prior art for these hybrids
This commit is contained in:
Tin Chi Lo 2026-07-21 14:28:03 -07:00
parent b79b01e38a
commit ebe48d67de
3 changed files with 76 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
litellm/proxy/response_api_endpoints/endpoints.py
View file

@ -33,14 +33,21 @@ def _nest_flat_chat_tool(tool: object) -> object:
convert_custom_tool_format_to_chat_shape,
)
if not isinstance(tool, dict) or "name" not in tool:
if not isinstance(tool, dict):
return tool
if tool.get("type") == "custom" and "custom" not in tool:
payload = {k: tool[k] for k in _FLAT_CUSTOM_TOOL_KEYS if k in tool}
if tool.get("type") == "custom":
if isinstance(tool.get("custom"), dict):
envelope = tool
payload = tool["custom"]
elif "name" in tool:
envelope = {"type": "custom"}
payload = {k: tool[k] for k in _FLAT_CUSTOM_TOOL_KEYS if k in tool}
else:
return tool
if isinstance(payload.get("format"), dict):
payload = {**payload, "format": convert_custom_tool_format_to_chat_shape(payload["format"])}
return {"type": "custom", "custom": payload}
if tool.get("type") == "function" and "function" not in tool:
return {**envelope, "custom": payload}
if tool.get("type") == "function" and "function" not in tool and "name" in tool:
return {"type": "function", "function": {k: tool[k] for k in _FLAT_FUNCTION_TOOL_KEYS if k in tool}}
return tool
@ -364,9 +371,13 @@ async def cursor_chat_completions(
custom tools) to the chat/completions path while expecting chat completions responses;
those are routed through the Responses API pipeline and converted back. Genuine chat
completions bodies (`messages` present) are routed through the standard chat completions
pipeline, after nesting any flat Responses-style tool defs Cursor mixes into the chat
`tools` array (e.g. `{"type": "custom", "name": "ApplyPatch", ...}`) into the chat
completions shape OpenAI requires (`{"type": "custom", "custom": {...}}`).
pipeline, after normalizing each level of the `tools` array and `tool_choice` to the chat
completions shapes OpenAI requires. Cursor mixes Responses API shapes into chat bodies
per level, independently: a flat tool def (`{"type": "custom", "name": "ApplyPatch", ...}`)
gets nested under `custom`, and a flat grammar format
(`{"type": "grammar", "definition", "syntax"}`) gets wrapped as
`{"type": "grammar", "grammar": {...}}` wherever it appears, including inside tool defs
Cursor already sent pre-nested.
```bash
curl -X POST http://localhost:4000/cursor/chat/completions \

69
tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py
View file

@ -1052,26 +1052,56 @@ class TestCursorMessagesArmToolNormalization:
assert seen["body"]["messages"] == body["messages"]
class TestNestFlatChatToolGrammarFormat:
def test_flat_grammar_format_is_wrapped_for_chat(self):
class TestNestFlatChatToolShapeMatrix:
"""
Cursor mixes Responses API shapes into chat bodies PER LEVEL, independently
(live-captured: a pre-nested custom envelope carrying a flat grammar format).
Every cell of envelope x format must land on the canonical chat shape.
"""
FLAT_GRAMMAR = {"type": "grammar", "definition": "start: patch", "syntax": "lark"}
NESTED_GRAMMAR = {"type": "grammar", "grammar": {"definition": "start: patch", "syntax": "lark"}}
TEXT = {"type": "text"}
@pytest.mark.parametrize("envelope", ["flat", "nested"])
@pytest.mark.parametrize("format_shape", ["absent", "text", "flat_grammar", "nested_grammar"])
def test_every_envelope_and_format_combination_lands_canonical(self, envelope, format_shape):
from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tools
result = _nest_flat_chat_tools(
[
{
"type": "custom",
"name": "ApplyPatch",
"description": "V4A patch",
"format": {"type": "grammar", "definition": "start: patch", "syntax": "lark"},
}
]
)
assert result == [
format_value = {
"absent": None,
"text": self.TEXT,
"flat_grammar": self.FLAT_GRAMMAR,
"nested_grammar": self.NESTED_GRAMMAR,
}[format_shape]
payload = {"name": "ApplyPatch", "description": "V4A patch"}
if format_value is not None:
payload["format"] = format_value
tool = {"type": "custom", "custom": payload} if envelope == "nested" else {"type": "custom", **payload}
canonical_payload = {"name": "ApplyPatch", "description": "V4A patch"}
if format_shape in ("flat_grammar", "nested_grammar"):
canonical_payload["format"] = self.NESTED_GRAMMAR
elif format_shape == "text":
canonical_payload["format"] = self.TEXT
assert _nest_flat_chat_tools([tool]) == [{"type": "custom", "custom": canonical_payload}]
def test_nested_envelope_with_flat_grammar_matches_live_cursor_capture(self):
from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tools
cursor_tool = {
"type": "custom",
"custom": {
"name": "ApplyPatch",
"format": {"type": "grammar", "definition": "start: patch", "syntax": "lark"},
},
}
assert _nest_flat_chat_tools([cursor_tool]) == [
{
"type": "custom",
"custom": {
"name": "ApplyPatch",
"description": "V4A patch",
"format": {
"type": "grammar",
"grammar": {"definition": "start: patch", "syntax": "lark"},
@ -1080,13 +1110,14 @@ class TestNestFlatChatToolGrammarFormat:
}
]
def test_flat_text_format_is_copied_unchanged(self):
def test_canonical_nested_tool_is_returned_equal(self):
from litellm.proxy.response_api_endpoints.endpoints import _nest_flat_chat_tools
result = _nest_flat_chat_tools(
[{"type": "custom", "name": "A", "format": {"type": "text"}}]
)
assert result == [{"type": "custom", "custom": {"name": "A", "format": {"type": "text"}}}]
canonical = {
"type": "custom",
"custom": {"name": "A", "format": {"type": "grammar", "grammar": {"definition": "d", "syntax": "lark"}}},
}
assert _nest_flat_chat_tools([canonical]) == [canonical]
class TestNestFlatChatToolChoice:

10
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -2628,9 +2628,13 @@ export interface paths {
* custom tools) to the chat/completions path while expecting chat completions responses;
* those are routed through the Responses API pipeline and converted back. Genuine chat
* completions bodies (`messages` present) are routed through the standard chat completions
* pipeline, after nesting any flat Responses-style tool defs Cursor mixes into the chat
* `tools` array (e.g. `{"type": "custom", "name": "ApplyPatch", ...}`) into the chat
* completions shape OpenAI requires (`{"type": "custom", "custom": {...}}`).
* pipeline, after normalizing each level of the `tools` array and `tool_choice` to the chat
* completions shapes OpenAI requires. Cursor mixes Responses API shapes into chat bodies
* per level, independently: a flat tool def (`{"type": "custom", "name": "ApplyPatch", ...}`)
* gets nested under `custom`, and a flat grammar format
* (`{"type": "grammar", "definition", "syntax"}`) gets wrapped as
* `{"type": "grammar", "grammar": {...}}` wherever it appears, including inside tool defs
* Cursor already sent pre-nested.
*
* ```bash
* curl -X POST http://localhost:4000/cursor/chat/completions -H "Content-Type: application/json" -H "Authorization: Bearer sk-1234" -d '{