From efcb0f8b02ec87eec9f1bf9789c98ad4675c5082 Mon Sep 17 00:00:00 2001 From: Philipp Sinitsin Date: Thu, 11 Jun 2026 21:14:30 +0100 Subject: [PATCH 1/3] fix(proxy): restore spend logging for /cursor/chat/completions (#30126) The /cursor/chat/completions route processes requests through the Responses API pipeline, but _get_metadata_variable_name did not treat it as a litellm_metadata route. Proxy auth metadata therefore landed in the user-facing metadata param, so the spend tracking callback saw user_api_key=None and silently skipped the SpendLogs write. Add the route to LITELLM_METADATA_ROUTES and cover it with regression tests. --- litellm/proxy/litellm_pre_call_utils.py | 1 + .../response_api_endpoints/test_endpoints.py | 68 +++++++++++++++++++ .../proxy/test_litellm_pre_call_utils.py | 4 ++ 3 files changed, 73 insertions(+) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 7666b23f2af..0df2cd0da09 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -110,6 +110,7 @@ LITELLM_METADATA_ROUTES = ( "/v1/messages", "responses", "files", + "/cursor/chat/completions", ) _UNTRUSTED_ROOT_CONTROL_FIELDS = ( diff --git a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py index 07d1a9d14f9..dfd6bf4a9ee 100644 --- a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py @@ -198,6 +198,74 @@ class TestResponsesAPIEndpoints(unittest.TestCase): assert response_cost_value == pytest.approx(0.0005, abs=1e-10) +class TestCursorChatCompletionsSpendTracking: + @patch("litellm.proxy.proxy_server.llm_router") + def test_proxy_auth_metadata_forwarded_as_litellm_metadata(self, mock_router): + """ + /cursor/chat/completions runs through the Responses API pipeline, so + proxy auth metadata (user_api_key etc.) must be sent as + `litellm_metadata`. If it lands in the user-facing `metadata` param + instead, the spend tracking callback sees user_api_key=None and + silently skips the SpendLogs write. Regression test for #30126. + """ + from openai.types.responses import ResponseOutputMessage, ResponseOutputText + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.types.llms.openai import ResponsesAPIResponse + + mock_router.aresponses = AsyncMock( + return_value=ResponsesAPIResponse( + id="resp_cursor123", + created_at=1234567890, + model="gpt-4o", + object="response", + output=[ + ResponseOutputMessage( + id="msg_cursor123", + type="message", + role="assistant", + status="completed", + content=[ + ResponseOutputText( + type="output_text", + text="Hello from Cursor!", + annotations=[], + ) + ], + ) + ], + ) + ) + + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="hashed-test-key", + user_id="test-user", + request_route="/cursor/chat/completions", + ) + try: + client = TestClient(app) + response = client.post( + "/cursor/chat/completions", + json={ + "model": "gpt-4o", + "input": [{"role": "user", "content": "Hello"}], + }, + headers={"Authorization": "Bearer sk-1234"}, + ) + finally: + app.dependency_overrides.pop(user_api_key_auth, None) + + assert response.status_code == 200 + mock_router.aresponses.assert_called_once() + request_kwargs = mock_router.aresponses.call_args.kwargs + litellm_metadata = request_kwargs.get("litellm_metadata") or {} + assert litellm_metadata.get("user_api_key") == "hashed-test-key" + assert litellm_metadata.get("user_api_key_user_id") == "test-user" + # proxy internals must not leak into the user-facing metadata param + assert "user_api_key" not in (request_kwargs.get("metadata") or {}) + + import json diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index f336c632546..1a018691142 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -84,6 +84,10 @@ class TestGetMetadataVariableName: request = self._make_request("/v1/files") assert _get_metadata_variable_name(request) == "litellm_metadata" + def test_returns_litellm_metadata_for_cursor_chat_completions(self): + request = self._make_request("/cursor/chat/completions") + assert _get_metadata_variable_name(request) == "litellm_metadata" + def test_returns_metadata_for_chat_completions(self): request = self._make_request("/chat/completions") assert _get_metadata_variable_name(request) == "metadata" From 06957857c2cee460214b25e8e2a6d7ff4b29d986 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 06:08:21 -0700 Subject: [PATCH 2/3] test(proxy): pin provider metadata and key billing on /cursor/chat/completions A Responses-format body on the Cursor route now sends OpenAI only the metadata the client sent and writes a spend row that bills the key. The chat-body branch, which this fix also moves onto litellm_metadata, still bills the key with its team and request tags. The scripted chat reply gets a unique id per test, since a shared id let only the first spend row per database land --- .../providers/test_cursor_responses_wire.py | 100 ++++++++++++++---- 1 file changed, 79 insertions(+), 21 deletions(-) diff --git a/tests/integration/providers/test_cursor_responses_wire.py b/tests/integration/providers/test_cursor_responses_wire.py index d77d35f2ff8..d516e72199a 100644 --- a/tests/integration/providers/test_cursor_responses_wire.py +++ b/tests/integration/providers/test_cursor_responses_wire.py @@ -3,11 +3,13 @@ from __future__ import annotations import json import os import uuid +from collections.abc import Mapping +from hashlib import sha256 from importlib.resources import files from typing import Final import pytest -from integration._support.client import Gateway, Scenario +from integration._support.client import Gateway, Scenario, eventually, list_value, object_value from integration._support.responses_vendor import same_response from integration._support.wire import Reply, Request, Wire, wire_server from integration._support.worker_readiness import wait_for_worker_readiness @@ -38,6 +40,7 @@ _CHAT_SUFFIXES: Final[tuple[tuple[str, str], ...]] = (("", ""),) + tuple( (f"-thinking-{level}", level) for level in _CHAT_THINKING_LEVELS ) _PROMPT: Final = "Inspect the requested change." +_REQUEST_TAG: Final = "cursor-integration" _INPUT: Final = [{"role": "user", "content": [{"type": "input_text", "text": _PROMPT}]}] _CUSTOM_TOOL: Final = { "type": "custom", @@ -107,16 +110,30 @@ def _stream(identity: str) -> tuple[bytes, ...]: ) + (b"data: [DONE]\n\n",) -def _chat_completion(stream: bool) -> bytes | tuple[bytes, ...]: +def _chat_completion(stream: bool, identity: str) -> bytes | tuple[bytes, ...]: if stream: - return ( - b'data: {"id":"chat_cursor_1","object":"chat.completion.chunk","created":1,"model":"gpt-5","choices":[{"index":0,"delta":{"content":"Patch inspected."},"finish_reason":null}]}\n\n', - b'data: {"id":"chat_cursor_1","object":"chat.completion.chunk","created":1,"model":"gpt-5","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n', - b"data: [DONE]\n\n", - ) + return tuple( + f"data: {json.dumps(chunk)}\n\n".encode() + for chunk in ( + { + "id": identity, + "object": "chat.completion.chunk", + "created": 1, + "model": _MODEL, + "choices": [{"index": 0, "delta": {"content": "Patch inspected."}, "finish_reason": None}], + }, + { + "id": identity, + "object": "chat.completion.chunk", + "created": 1, + "model": _MODEL, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + }, + ) + ) + (b"data: [DONE]\n\n",) return json.dumps( { - "id": "chat_cursor_1", + "id": identity, "object": "chat.completion", "created": 1, "model": _MODEL, @@ -167,7 +184,7 @@ def test_cursor_responses_input_uses_responses_wire_and_chat_output( assert request.method == "POST" assert request.headers["authorization"] == f"Bearer {_API_KEY}", dict(request.headers) if request.target == "/v1/chat/completions": - completion: Final = _chat_completion(stream) + completion: Final = _chat_completion(stream, raw_id) return Reply( content_type="text/event-stream" if stream else "application/json", chunks=completion if isinstance(completion, tuple) else None, @@ -175,7 +192,7 @@ def test_cursor_responses_input_uses_responses_wire_and_chat_output( ) assert request.target == "/v1/responses", request.target body: Final = _JSON_OBJECT.validate_json(request.body) - assert {key: value for key, value in body.items() if key != "metadata"} == { + assert body == { "model": _MODEL, "input": _INPUT, **reasoning, @@ -259,41 +276,69 @@ def test_cursor_responses_input_uses_responses_wire_and_chat_output( assert [request.target for request in drain_contract_requests(wire)] == ["/v1/responses"] -def test_cursor_responses_body_carries_no_proxy_metadata(gateway: Gateway) -> None: - pytest.skip( - "BUG: /cursor/chat/completions forwards proxy-internal metadata (key hash, team, user, client ip, user agent) " - "to the provider in the Responses body" - ) +@pytest.mark.parametrize( + "client_metadata", ({}, {"cursor_conversation": "integration"}), ids=("no_client_metadata", "client_metadata") +) +@pytest.mark.parametrize("stream", (False, True)) +def test_cursor_responses_body_sends_only_client_metadata_and_bills_the_key( + gateway: Gateway, + stream: bool, + client_metadata: Mapping[str, JsonValue], +) -> None: raw_id: Final = f"resp_cursor_metadata_{uuid.uuid4().hex}" + alias: Final = f"integration-cursor-{uuid.uuid4().hex}" + sent_metadata: Final = {"metadata": client_metadata} if client_metadata else {} def respond(request: Request) -> Reply: if request.method == "GET" and request.target == "/v1/models": return model_discovery_reply(_MODEL) assert request.method == "POST" and request.target == "/v1/responses", request.target + if stream: + return Reply(content_type="text/event-stream", chunks=_stream(raw_id)) return Reply(body=json.dumps(_response(raw_id)).encode()) with wire_server(respond) as wire, gateway.scenario() as scenario: model: Final = _deployment(scenario, wire) + key: Final = scenario.key(key_alias=alias) + hashed_key: Final = sha256(key.encode()).hexdigest() response: Final = gateway.request( "POST", "/cursor/chat/completions", { - "model": f"{model}-thinking-high", + "model": model, "input": _INPUT, "messages": [], "tools": [_CUSTOM_TOOL], + "stream": stream, + **sent_metadata, }, + key=key, ) assert response.status_code == 200, response.text requests: Final = drain_contract_requests(wire) assert len(requests) == 1, requests - body: Final = _JSON_OBJECT.validate_json(requests[0].body) - assert body == { + assert _JSON_OBJECT.validate_json(requests[0].body) == { "model": _MODEL, "input": _INPUT, - "reasoning": {"effort": "high"}, "tools": [_CUSTOM_TOOL], + "stream": stream, + **sent_metadata, }, requests[0].body + (row,) = eventually( + lambda: gateway.rows( + 'SELECT call_type, spend, metadata FROM "LiteLLM_SpendLogs" WHERE api_key = %s', (hashed_key,) + ), + lambda rows: len(rows) == 1, + seconds=70, + ) + assert row["call_type"] == "aresponses", row + assert object_value(row["metadata"])["user_api_key_alias"] == alias, row + assert isinstance(row["spend"], float) and row["spend"] > 0, row + eventually( + lambda: gateway.rows('SELECT spend FROM "LiteLLM_VerificationToken" WHERE token = %s', (hashed_key,)), + lambda tokens: len(tokens) == 1 and tokens[0]["spend"] == pytest.approx(row["spend"]), + seconds=70, + ) def test_cursor_stream_reports_usage_when_the_client_asks_for_it(gateway: Gateway) -> None: @@ -341,10 +386,12 @@ def test_cursor_messages_body_uses_chat_completions_wire( suffix: str, reasoning_effort: str, ) -> None: + chat_id: Final = f"chat_cursor_{uuid.uuid4().hex}" + def respond(request: Request) -> Reply: if request.method == "GET" and request.target == "/v1/models": return model_discovery_reply(_MODEL) - completion: Final = _chat_completion(stream) + completion: Final = _chat_completion(stream, chat_id) return Reply( content_type="text/event-stream" if stream else "application/json", chunks=completion if isinstance(completion, tuple) else None, @@ -367,6 +414,7 @@ def test_cursor_messages_body_uses_chat_completions_wire( "stream": stream, }, key=key, + headers={"x-litellm-tags": _REQUEST_TAG}, ) assert response.status_code == 200, response.text if stream: @@ -393,7 +441,7 @@ def test_cursor_messages_body_uses_chat_completions_wire( payload: Final = _JSON_OBJECT.validate_json(response.content) identity: Final = payload.get("id") created: Final = payload.get("created") - assert isinstance(identity, str) and same_response(identity, "chat_cursor_1"), response.text + assert isinstance(identity, str) and same_response(identity, chat_id), response.text assert isinstance(created, int) and created > 0, response.text assert payload == { "id": identity, @@ -456,3 +504,13 @@ def test_cursor_messages_body_uses_chat_completions_wire( **({"stream_options": {"include_usage": True}} if stream else {}), **({"reasoning_effort": reasoning_effort} if reasoning_effort else {}), }, requests[0].body + (row,) = eventually( + lambda: gateway.rows( + 'SELECT call_type, team_id, request_tags FROM "LiteLLM_SpendLogs" WHERE api_key = %s', + (sha256(key.encode()).hexdigest(),), + ), + lambda rows: len(rows) == 1, + seconds=70, + ) + assert (row["call_type"], row["team_id"]) == ("acompletion", team), row + assert _REQUEST_TAG in list_value(row["request_tags"]), row From 262d9cbf9f0970cceea8e616f79ed4496c89639a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:44:41 -0700 Subject: [PATCH 3/3] fix(proxy): pick the Cursor route's metadata field from the body shape A Responses-style body on /cursor/chat/completions keeps proxy metadata in litellm_metadata, the same as /v1/responses, while a Chat Completions body keeps metadata, the same as /v1/chat/completions. Adding the whole route to the internal-metadata list let a client's chat-body tags replace the key's tags in spend logs and skip tag budgets. --- litellm/proxy/auth/auth_checks.py | 4 +- litellm/proxy/common_request_processing.py | 16 +++--- .../proxy/hooks/proxy_track_cost_callback.py | 4 +- litellm/proxy/litellm_pre_call_utils.py | 32 +++++++++-- .../proxy/response_api_endpoints/endpoints.py | 12 ++--- .../providers/test_cursor_responses_wire.py | 54 ++++++++++++++++++- tests/unit/proxy/auth/test_proxy_routes.py | 2 +- .../response_api_endpoints/test_endpoints.py | 2 +- .../proxy/test_common_request_processing.py | 34 ++++++++++++ .../unit/proxy/test_litellm_pre_call_utils.py | 37 ++++++++----- 10 files changed, 156 insertions(+), 41 deletions(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 448643ef59b..c968fdfdce6 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -440,7 +440,7 @@ def _request_has_trace_id(request_body: Mapping[str, object], request: Request, from litellm.proxy.litellm_pre_call_utils import ( _trace_id_from_traceparent, # pyright: ignore[reportPrivateUsage] # reuse W3C traceparent parser get_chain_id_from_headers, - metadata_variable_name_for_route, + metadata_variable_name_for_request, ) if get_chain_id_from_headers(dict(request.headers)): @@ -454,7 +454,7 @@ def _request_has_trace_id(request_body: Mapping[str, object], request: Request, traceparent: Final[str | None] = request.headers.get("traceparent") if isinstance(traceparent, str) and _trace_id_from_traceparent(traceparent): return True - metadata_variable_name: Final = metadata_variable_name_for_route(get_request_route(request)) + metadata_variable_name: Final = metadata_variable_name_for_request(get_request_route(request), request_body) selected_metadata: Final[object] = request_body.get(metadata_variable_name) if isinstance(selected_metadata, dict): selected_metadata_dict: Final = _TRACE_ID_METADATA_ADAPTER.validate_python(selected_metadata) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 24a36a7f2dc..e813c246bea 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -2324,9 +2324,10 @@ class ProxyBaseLLMRequestProcessing: version=version, proxy_config=proxy_config, ) + request_fields: Final = _REQUEST_FIELDS.validate_python(self.data) + metadata_variable_name: Final = get_metadata_variable_name(request, request_fields) if logging_auth is not None: - metadata_variable_name: Final = get_metadata_variable_name(request) - request_metadata: Final = self.data.get(metadata_variable_name) + request_metadata: Final = request_fields.get(metadata_variable_name) if isinstance(request_metadata, dict): self.data[metadata_variable_name] = { **TypeAdapter(dict[str, object]).validate_python(request_metadata), @@ -2354,12 +2355,11 @@ class ProxyBaseLLMRequestProcessing: # Store queue time in metadata after add_litellm_data_to_request to ensure it's preserved if queue_time_seconds is not None: - _metadata_variable_name: Final = get_metadata_variable_name(request) - if _metadata_variable_name not in self.data: - self.data[_metadata_variable_name] = {} - if not isinstance(self.data[_metadata_variable_name], dict): - self.data[_metadata_variable_name] = {} - self.data[_metadata_variable_name]["queue_time_seconds"] = queue_time_seconds + if metadata_variable_name not in self.data: + self.data[metadata_variable_name] = {} + if not isinstance(self.data[metadata_variable_name], dict): + self.data[metadata_variable_name] = {} + self.data[metadata_variable_name]["queue_time_seconds"] = queue_time_seconds if isinstance(model, str): reject_url_valued_destination("model", model) diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index c11b59a809a..b07903cc709 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -37,7 +37,7 @@ from litellm.proxy.db.db_spend_update_writer import ( get_llm_router, ) from litellm.proxy.hooks.search_free_tier import apply_search_free_tier -from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, metadata_variable_name_for_route +from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, metadata_variable_name_for_request from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.proxy.spend_tracking.spend_counter_batch import post_call_counter_keys, spend_counter_batch_scope from litellm.proxy.spend_tracking.spend_event import ( @@ -143,7 +143,7 @@ def _proxy_stamped_used_client_oauth_token( proxy_bucket: Final = ( get_metadata_variable_name_from_kwargs(request_data) if request_route is None - else metadata_variable_name_for_route(request_route) + else metadata_variable_name_for_request(request_route, request_data) ) proxy_metadata: Final = request_data.get(proxy_bucket) stamped: Final = proxy_metadata.get("used_client_oauth_token") if isinstance(proxy_metadata, dict) else None diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 5f0be6f78cd..9fbc05abe17 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -340,9 +340,10 @@ LITELLM_METADATA_ROUTES: Final = ( "responses", "files", "interactions", - "/cursor/chat/completions", ) +CURSOR_CHAT_COMPLETIONS_ROUTE: Final = "/cursor/chat/completions" + LITELLM_TRACE_CONTROL_METADATA_FIELDS: Final = frozenset( { "mask_input", @@ -827,7 +828,7 @@ def _strip_router_reserved_metadata( ) -def get_metadata_variable_name(request: Request) -> str: +def get_metadata_variable_name(request: Request, data: Mapping[str, object]) -> str: """ Helper to return what the "metadata" field should be called in the request data @@ -838,7 +839,7 @@ def get_metadata_variable_name(request: Request) -> str: # Inline imports — auth_utils/route_checks participate in a proxy import cycle. from litellm.proxy.auth.auth_utils import get_request_route # noqa: PLC0415 - return metadata_variable_name_for_route(get_request_route(request)) + return metadata_variable_name_for_request(get_request_route(request), data) _get_metadata_variable_name: Final = get_metadata_variable_name @@ -854,6 +855,25 @@ def metadata_variable_name_for_route(route: str) -> Literal["metadata", "litellm return "metadata" +def is_chat_completions_body(data: Mapping[str, object]) -> bool: + messages: Final = data.get("messages") + if isinstance(messages, list) and messages: + return True + return "messages" in data and "input" not in data + + +def is_cursor_responses_request(route: str, body: Mapping[str, object]) -> bool: + return CURSOR_CHAT_COMPLETIONS_ROUTE in route and not is_chat_completions_body(body) + + +def metadata_variable_name_for_request( + route: str, body: Mapping[str, object] +) -> Literal["metadata", "litellm_metadata"]: + if is_cursor_responses_request(route, body): + return "litellm_metadata" + return metadata_variable_name_for_route(route) + + def _promoted_trace_control_fields( requester_metadata: Mapping[str, object], litellm_metadata: Mapping[str, object], @@ -2263,7 +2283,9 @@ class LiteLLMProxyRequestSetup: ``_tag_max_budget_check`` resolve to ``litellm_metadata``, leaving header tags invisible to per-tag budget enforcement. """ - if any(metadata_route in route for metadata_route in LITELLM_METADATA_ROUTES): + if any(metadata_route in route for metadata_route in LITELLM_METADATA_ROUTES) or is_cursor_responses_request( + route, request_data + ): request_data.setdefault("litellm_metadata", {}) @staticmethod @@ -2590,7 +2612,7 @@ async def add_litellm_data_to_request( } safe_add_api_version_from_query_params(data, request) - _metadata_variable_name: Final = get_metadata_variable_name(request) + _metadata_variable_name: Final = get_metadata_variable_name(request, _REQUEST_FIELDS.validate_python(data)) if data.get(_metadata_variable_name, None) is None: data[_metadata_variable_name] = {} diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 97bbf893e3e..e0cf704852c 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -75,6 +75,7 @@ from litellm.proxy.hooks.responses_id_security import ResponsesIDSecurity, decry from litellm.proxy.litellm_pre_call_utils import ( _CLIENT_PRICING_METADATA_FIELDS, # pyright: ignore[reportPrivateUsage] # the HTTP path's own strip list _UNTRUSTED_METADATA_CONTROL_FIELDS, # pyright: ignore[reportPrivateUsage] # the HTTP path's own strip list + is_chat_completions_body, ) from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution import ( request_tags_from_request_data, @@ -175,13 +176,6 @@ def _normalize_tool_dialect( return {**data, **{key: value for key, value in replaceable if key in data}} -def _is_chat_completions_body(data: Mapping[str, object]) -> bool: - messages: Final = data.get("messages") - if isinstance(messages, list) and messages: - return True - return "messages" in data and "input" not in data - - _CURSOR_THINKING_SEPARATOR: Final = "-thinking-" _CURSOR_FAST_SUFFIX: Final = "-fast" _CURSOR_THINKING_LEVELS: Final[frozenset[str]] = frozenset(get_args(REASONING_EFFORT)) @@ -222,7 +216,7 @@ def _resolve_cursor_model_variant( resolved: Final = {**data, "model": variant.base_model} if variant.reasoning_effort is None: return resolved - if _is_chat_completions_body(data): + if is_chat_completions_body(data): if "reasoning_effort" in data: return resolved return {**resolved, "reasoning_effort": variant.reasoning_effort} @@ -604,7 +598,7 @@ async def cursor_chat_completions( raw_body: Final = await read_request_body(request=request) - if _is_chat_completions_body(raw_body): + if is_chat_completions_body(raw_body): # Genuine chat completions body (Cursor sends these for models whose BYOK it # already fixed); delegate so behavior matches /chat/completions exactly. # Keyed on messages CONTENT, not key presence: Cursor can send a null or diff --git a/tests/integration/providers/test_cursor_responses_wire.py b/tests/integration/providers/test_cursor_responses_wire.py index d516e72199a..2b39b0e312c 100644 --- a/tests/integration/providers/test_cursor_responses_wire.py +++ b/tests/integration/providers/test_cursor_responses_wire.py @@ -8,6 +8,7 @@ from hashlib import sha256 from importlib.resources import files from typing import Final +import httpx import pytest from integration._support.client import Gateway, Scenario, eventually, list_value, object_value from integration._support.responses_vendor import same_response @@ -150,11 +151,12 @@ def _chat_completion(stream: bool, identity: str) -> bytes | tuple[bytes, ...]: ).encode() -def _deployment(scenario: Scenario, wire: Wire) -> str: +def _deployment(scenario: Scenario, wire: Wire, **pricing: JsonValue) -> str: model: Final = scenario.model( model=f"openai/{_MODEL}", api_key=_API_KEY, api_base=f"{wire.url}/v1", + **pricing, ) wait_for_worker_readiness(scenario.gateway, model, 1, _PROXY_WORKERS) return model @@ -514,3 +516,53 @@ def test_cursor_messages_body_uses_chat_completions_wire( ) assert (row["call_type"], row["team_id"]) == ("acompletion", team), row assert _REQUEST_TAG in list_value(row["request_tags"]), row + + +def test_cursor_messages_body_keeps_client_and_key_tags_and_enforces_the_client_tag_budget(gateway: Gateway) -> None: + chat_id: Final = f"chat_cursor_{uuid.uuid4().hex}" + budget_tag: Final = f"cursor-tag-budget-{uuid.uuid4().hex}" + key_tag: Final = f"cursor-key-tag-{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return model_discovery_reply(_MODEL) + completion: Final = _chat_completion(False, chat_id) + return Reply(body=completion if isinstance(completion, bytes) else b"") + + def delete_tag() -> None: + gateway.post("/tag/delete", {"name": budget_tag}) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, input_cost_per_token=0.01, output_cost_per_token=0.01) + key: Final = scenario.key(models=[model], metadata={"tags": [key_tag]}) + gateway.post("/tag/new", {"name": budget_tag, "max_budget": 0.0001}) + scenario.cleanups.callback(delete_tag) + + def tagged_chat(text: str) -> httpx.Response: + return gateway.request( + "POST", + "/cursor/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": text}], + "metadata": {"tags": [budget_tag]}, + }, + key=key, + ) + + first: Final = tagged_chat("first tagged chat") + assert first.status_code == 200, first.text + (row,) = eventually( + lambda: gateway.rows( + 'SELECT request_tags FROM "LiteLLM_SpendLogs" WHERE api_key = %s', + (sha256(key.encode()).hexdigest(),), + ), + lambda rows: len(rows) == 1, + seconds=70, + ) + assert {budget_tag, key_tag} <= set(list_value(row["request_tags"])), row + blocked: Final = eventually( + lambda: tagged_chat("tag budget probe"), lambda response: response.status_code != 200, seconds=70 + ) + assert blocked.status_code == 422, blocked.text + assert "budget" in blocked.text.lower(), blocked.text diff --git a/tests/unit/proxy/auth/test_proxy_routes.py b/tests/unit/proxy/auth/test_proxy_routes.py index 24ef2a0e150..5616f0a4e3d 100644 --- a/tests/unit/proxy/auth/test_proxy_routes.py +++ b/tests/unit/proxy/auth/test_proxy_routes.py @@ -248,7 +248,7 @@ def _is_assistants(req): def _metadata_var_name(req): from litellm.proxy.litellm_pre_call_utils import get_metadata_variable_name - return get_metadata_variable_name(req) + return get_metadata_variable_name(req, {}) def _vector_store_id_in_path(req): diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index 985aee061bc..0b9c8535fef 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -3688,7 +3688,7 @@ class TestCursorInputArmFlattening: class TestChatCompletionsBodyDetection: def test_routing_matrix(self): - from litellm.proxy.response_api_endpoints.endpoints import _is_chat_completions_body + from litellm.proxy.litellm_pre_call_utils import is_chat_completions_body as _is_chat_completions_body assert _is_chat_completions_body({"messages": [{"role": "user", "content": "hi"}]}) is True assert _is_chat_completions_body({"messages": [{"role": "user", "content": "hi"}], "input": []}) is True diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 52816dc50a8..7e6de52324f 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -9537,6 +9537,40 @@ async def test_saved_weights_override_caller_input_and_preserve_key_precedence( assert logging.pre_call_hook.call_args.kwargs["data"].get("_router_weights") == expected +def _hook_data(**kwargs: object) -> object: + return kwargs["data"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("prompt, proxy_field, client_field", [ + ({"input": "hi"}, "litellm_metadata", "metadata"), + ({"messages": [{"role": "user", "content": "hi"}]}, "metadata", "litellm_metadata"), +]) +async def test_cursor_route_writes_proxy_fields_only_to_the_field_its_body_shape_picks( + prompt: dict[str, str | list[dict[str, str]]], + proxy_field: str, + client_field: str, +) -> None: + from litellm.proxy import proxy_server + + logging: Final = MagicMock(spec=ProxyLogging) + logging.pre_call_hook = AsyncMock(side_effect=_hook_data) + processor: Final = ProxyBaseLLMRequestProcessing(data={"model": "gpt-x", "metadata": {"client": "cursor"}, **prompt}) + data, _ = await processor.common_processing_pre_call_logic( + request=Request({"type": "http", "method": "POST", "path": "/cursor/chat/completions", "headers": []}), + general_settings={}, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="hash"), + proxy_logging_obj=logging, + proxy_config=proxy_server.ProxyConfig(), + route_type="acompletion", + logging_auth=ProxyUserAPIKeyAuth(api_key="logging-hash", key_alias="logging-alias"), + ) + assert data[proxy_field]["user_api_key_alias"] == "logging-alias" + assert "queue_time_seconds" in data[proxy_field] + assert "user_api_key_alias" not in data.get(client_field, {}) + assert "queue_time_seconds" not in data.get(client_field, {}) + assert data["metadata"]["client"] == "cursor" + class TestPerRequestModelGroupAlias: """``router_settings.model_group_alias`` on a key or team has to be resolved by the proxy: the Router resolves aliases from its own shared instance diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index a32c40e5a43..2e3bb026509 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -90,52 +90,65 @@ class TestGetMetadataVariableName: def test_returns_litellm_metadata_for_thread_routes(self): request = self._make_request("/v1/threads/thread_123/messages") - assert get_metadata_variable_name(request) == "litellm_metadata" + assert get_metadata_variable_name(request, {}) == "litellm_metadata" def test_returns_litellm_metadata_for_assistant_routes(self): request = self._make_request("/v1/assistants/asst_123") - assert get_metadata_variable_name(request) == "litellm_metadata" + assert get_metadata_variable_name(request, {}) == "litellm_metadata" def test_returns_litellm_metadata_for_batches_route(self): request = self._make_request("/v1/batches") - assert get_metadata_variable_name(request) == "litellm_metadata" + assert get_metadata_variable_name(request, {}) == "litellm_metadata" def test_returns_litellm_metadata_for_messages_route(self): request = self._make_request("/v1/messages") - assert get_metadata_variable_name(request) == "litellm_metadata" + assert get_metadata_variable_name(request, {}) == "litellm_metadata" def test_returns_litellm_metadata_for_files_route(self): request = self._make_request("/v1/files") - assert get_metadata_variable_name(request) == "litellm_metadata" + assert get_metadata_variable_name(request, {}) == "litellm_metadata" @pytest.mark.parametrize( "path", ["/v1beta/interactions", "/interactions", "/v1beta/interactions/v1_abc123", "/interactions/v1_abc123/cancel"] ) def test_returns_litellm_metadata_for_interactions_routes(self, path): request = self._make_request(path) - assert get_metadata_variable_name(request) == "litellm_metadata" + assert get_metadata_variable_name(request, {}) == "litellm_metadata" def test_returns_metadata_for_chat_completions(self): request = self._make_request("/chat/completions") - assert get_metadata_variable_name(request) == "metadata" + assert get_metadata_variable_name(request, {}) == "metadata" + + @pytest.mark.parametrize( + "body, expected", + [ + ({"messages": [{"role": "user", "content": "hi"}]}, "metadata"), + ({"messages": []}, "metadata"), + ({"input": [{"role": "user", "content": "hi"}]}, "litellm_metadata"), + ({"input": "hi", "messages": []}, "litellm_metadata"), + ], + ) + def test_cursor_route_picks_the_metadata_field_by_body_shape(self, body, expected): + request = self._make_request("/cursor/chat/completions") + assert get_metadata_variable_name(request, body) == expected def test_returns_metadata_for_completions(self): request = self._make_request("/v1/completions") - assert get_metadata_variable_name(request) == "metadata" + assert get_metadata_variable_name(request, {}) == "metadata" def test_returns_metadata_for_embeddings(self): request = self._make_request("/v1/embeddings") - assert get_metadata_variable_name(request) == "metadata" + assert get_metadata_variable_name(request, {}) == "metadata" def test_returns_litellm_metadata_for_bedrock_invoke(self): # GH#30629: bedrock passthrough must use litellm_metadata # to prevent key-level tags from leaking into provider body request = self._make_request("/bedrock/model/us.anthropic.claude-sonnet-4-6/invoke") - assert get_metadata_variable_name(request) == "litellm_metadata" + assert get_metadata_variable_name(request, {}) == "litellm_metadata" def test_returns_litellm_metadata_for_bedrock_converse(self): request = self._make_request("/bedrock/model/us.anthropic.claude-sonnet-4-6/converse") - assert get_metadata_variable_name(request) == "litellm_metadata" + assert get_metadata_variable_name(request, {}) == "litellm_metadata" def test_get_enforced_params_for_service_account_settings(): @@ -7844,7 +7857,7 @@ class TestPromotedTraceControlFields: ) def test_returns_litellm_metadata_for_responses_route(self): - assert get_metadata_variable_name(self._make_request("/v1/responses")) == "litellm_metadata" + assert get_metadata_variable_name(self._make_request("/v1/responses"), {}) == "litellm_metadata" def test_promotes_trace_prefixed_and_allow_listed_fields(self): requester_metadata = {