mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(proxy): pick the Cursor route's metadata field from the body shape
A Responses-style body on /cursor/chat/completions keeps proxy metadata in litellm_metadata, the same as /v1/responses, while a Chat Completions body keeps metadata, the same as /v1/chat/completions. Adding the whole route to the internal-metadata list let a client's chat-body tags replace the key's tags in spend logs and skip tag budgets.
This commit is contained in:
parent
06957857c2
commit
262d9cbf9f
10 changed files with 156 additions and 41 deletions
|
|
@ -440,7 +440,7 @@ def _request_has_trace_id(request_body: Mapping[str, object], request: Request,
|
|||
from litellm.proxy.litellm_pre_call_utils import (
|
||||
_trace_id_from_traceparent, # pyright: ignore[reportPrivateUsage] # reuse W3C traceparent parser
|
||||
get_chain_id_from_headers,
|
||||
metadata_variable_name_for_route,
|
||||
metadata_variable_name_for_request,
|
||||
)
|
||||
|
||||
if get_chain_id_from_headers(dict(request.headers)):
|
||||
|
|
@ -454,7 +454,7 @@ def _request_has_trace_id(request_body: Mapping[str, object], request: Request,
|
|||
traceparent: Final[str | None] = request.headers.get("traceparent")
|
||||
if isinstance(traceparent, str) and _trace_id_from_traceparent(traceparent):
|
||||
return True
|
||||
metadata_variable_name: Final = metadata_variable_name_for_route(get_request_route(request))
|
||||
metadata_variable_name: Final = metadata_variable_name_for_request(get_request_route(request), request_body)
|
||||
selected_metadata: Final[object] = request_body.get(metadata_variable_name)
|
||||
if isinstance(selected_metadata, dict):
|
||||
selected_metadata_dict: Final = _TRACE_ID_METADATA_ADAPTER.validate_python(selected_metadata)
|
||||
|
|
|
|||
|
|
@ -2324,9 +2324,10 @@ class ProxyBaseLLMRequestProcessing:
|
|||
version=version,
|
||||
proxy_config=proxy_config,
|
||||
)
|
||||
request_fields: Final = _REQUEST_FIELDS.validate_python(self.data)
|
||||
metadata_variable_name: Final = get_metadata_variable_name(request, request_fields)
|
||||
if logging_auth is not None:
|
||||
metadata_variable_name: Final = get_metadata_variable_name(request)
|
||||
request_metadata: Final = self.data.get(metadata_variable_name)
|
||||
request_metadata: Final = request_fields.get(metadata_variable_name)
|
||||
if isinstance(request_metadata, dict):
|
||||
self.data[metadata_variable_name] = {
|
||||
**TypeAdapter(dict[str, object]).validate_python(request_metadata),
|
||||
|
|
@ -2354,12 +2355,11 @@ class ProxyBaseLLMRequestProcessing:
|
|||
|
||||
# Store queue time in metadata after add_litellm_data_to_request to ensure it's preserved
|
||||
if queue_time_seconds is not None:
|
||||
_metadata_variable_name: Final = get_metadata_variable_name(request)
|
||||
if _metadata_variable_name not in self.data:
|
||||
self.data[_metadata_variable_name] = {}
|
||||
if not isinstance(self.data[_metadata_variable_name], dict):
|
||||
self.data[_metadata_variable_name] = {}
|
||||
self.data[_metadata_variable_name]["queue_time_seconds"] = queue_time_seconds
|
||||
if metadata_variable_name not in self.data:
|
||||
self.data[metadata_variable_name] = {}
|
||||
if not isinstance(self.data[metadata_variable_name], dict):
|
||||
self.data[metadata_variable_name] = {}
|
||||
self.data[metadata_variable_name]["queue_time_seconds"] = queue_time_seconds
|
||||
|
||||
if isinstance(model, str):
|
||||
reject_url_valued_destination("model", model)
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ from litellm.proxy.db.db_spend_update_writer import (
|
|||
get_llm_router,
|
||||
)
|
||||
from litellm.proxy.hooks.search_free_tier import apply_search_free_tier
|
||||
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, metadata_variable_name_for_route
|
||||
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, metadata_variable_name_for_request
|
||||
from litellm.proxy.route_llm_request import ProxyModelNotFoundError
|
||||
from litellm.proxy.spend_tracking.spend_counter_batch import post_call_counter_keys, spend_counter_batch_scope
|
||||
from litellm.proxy.spend_tracking.spend_event import (
|
||||
|
|
@ -143,7 +143,7 @@ def _proxy_stamped_used_client_oauth_token(
|
|||
proxy_bucket: Final = (
|
||||
get_metadata_variable_name_from_kwargs(request_data)
|
||||
if request_route is None
|
||||
else metadata_variable_name_for_route(request_route)
|
||||
else metadata_variable_name_for_request(request_route, request_data)
|
||||
)
|
||||
proxy_metadata: Final = request_data.get(proxy_bucket)
|
||||
stamped: Final = proxy_metadata.get("used_client_oauth_token") if isinstance(proxy_metadata, dict) else None
|
||||
|
|
|
|||
|
|
@ -340,9 +340,10 @@ LITELLM_METADATA_ROUTES: Final = (
|
|||
"responses",
|
||||
"files",
|
||||
"interactions",
|
||||
"/cursor/chat/completions",
|
||||
)
|
||||
|
||||
CURSOR_CHAT_COMPLETIONS_ROUTE: Final = "/cursor/chat/completions"
|
||||
|
||||
LITELLM_TRACE_CONTROL_METADATA_FIELDS: Final = frozenset(
|
||||
{
|
||||
"mask_input",
|
||||
|
|
@ -827,7 +828,7 @@ def _strip_router_reserved_metadata(
|
|||
)
|
||||
|
||||
|
||||
def get_metadata_variable_name(request: Request) -> str:
|
||||
def get_metadata_variable_name(request: Request, data: Mapping[str, object]) -> str:
|
||||
"""
|
||||
Helper to return what the "metadata" field should be called in the request data
|
||||
|
||||
|
|
@ -838,7 +839,7 @@ def get_metadata_variable_name(request: Request) -> str:
|
|||
# Inline imports — auth_utils/route_checks participate in a proxy import cycle.
|
||||
from litellm.proxy.auth.auth_utils import get_request_route # noqa: PLC0415
|
||||
|
||||
return metadata_variable_name_for_route(get_request_route(request))
|
||||
return metadata_variable_name_for_request(get_request_route(request), data)
|
||||
|
||||
|
||||
_get_metadata_variable_name: Final = get_metadata_variable_name
|
||||
|
|
@ -854,6 +855,25 @@ def metadata_variable_name_for_route(route: str) -> Literal["metadata", "litellm
|
|||
return "metadata"
|
||||
|
||||
|
||||
def is_chat_completions_body(data: Mapping[str, object]) -> bool:
|
||||
messages: Final = data.get("messages")
|
||||
if isinstance(messages, list) and messages:
|
||||
return True
|
||||
return "messages" in data and "input" not in data
|
||||
|
||||
|
||||
def is_cursor_responses_request(route: str, body: Mapping[str, object]) -> bool:
|
||||
return CURSOR_CHAT_COMPLETIONS_ROUTE in route and not is_chat_completions_body(body)
|
||||
|
||||
|
||||
def metadata_variable_name_for_request(
|
||||
route: str, body: Mapping[str, object]
|
||||
) -> Literal["metadata", "litellm_metadata"]:
|
||||
if is_cursor_responses_request(route, body):
|
||||
return "litellm_metadata"
|
||||
return metadata_variable_name_for_route(route)
|
||||
|
||||
|
||||
def _promoted_trace_control_fields(
|
||||
requester_metadata: Mapping[str, object],
|
||||
litellm_metadata: Mapping[str, object],
|
||||
|
|
@ -2263,7 +2283,9 @@ class LiteLLMProxyRequestSetup:
|
|||
``_tag_max_budget_check`` resolve to ``litellm_metadata``, leaving
|
||||
header tags invisible to per-tag budget enforcement.
|
||||
"""
|
||||
if any(metadata_route in route for metadata_route in LITELLM_METADATA_ROUTES):
|
||||
if any(metadata_route in route for metadata_route in LITELLM_METADATA_ROUTES) or is_cursor_responses_request(
|
||||
route, request_data
|
||||
):
|
||||
request_data.setdefault("litellm_metadata", {})
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -2590,7 +2612,7 @@ async def add_litellm_data_to_request(
|
|||
}
|
||||
|
||||
safe_add_api_version_from_query_params(data, request)
|
||||
_metadata_variable_name: Final = get_metadata_variable_name(request)
|
||||
_metadata_variable_name: Final = get_metadata_variable_name(request, _REQUEST_FIELDS.validate_python(data))
|
||||
if data.get(_metadata_variable_name, None) is None:
|
||||
data[_metadata_variable_name] = {}
|
||||
|
||||
|
|
|
|||
|
|
@ -75,6 +75,7 @@ from litellm.proxy.hooks.responses_id_security import ResponsesIDSecurity, decry
|
|||
from litellm.proxy.litellm_pre_call_utils import (
|
||||
_CLIENT_PRICING_METADATA_FIELDS, # pyright: ignore[reportPrivateUsage] # the HTTP path's own strip list
|
||||
_UNTRUSTED_METADATA_CONTROL_FIELDS, # pyright: ignore[reportPrivateUsage] # the HTTP path's own strip list
|
||||
is_chat_completions_body,
|
||||
)
|
||||
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution import (
|
||||
request_tags_from_request_data,
|
||||
|
|
@ -175,13 +176,6 @@ def _normalize_tool_dialect(
|
|||
return {**data, **{key: value for key, value in replaceable if key in data}}
|
||||
|
||||
|
||||
def _is_chat_completions_body(data: Mapping[str, object]) -> bool:
|
||||
messages: Final = data.get("messages")
|
||||
if isinstance(messages, list) and messages:
|
||||
return True
|
||||
return "messages" in data and "input" not in data
|
||||
|
||||
|
||||
_CURSOR_THINKING_SEPARATOR: Final = "-thinking-"
|
||||
_CURSOR_FAST_SUFFIX: Final = "-fast"
|
||||
_CURSOR_THINKING_LEVELS: Final[frozenset[str]] = frozenset(get_args(REASONING_EFFORT))
|
||||
|
|
@ -222,7 +216,7 @@ def _resolve_cursor_model_variant(
|
|||
resolved: Final = {**data, "model": variant.base_model}
|
||||
if variant.reasoning_effort is None:
|
||||
return resolved
|
||||
if _is_chat_completions_body(data):
|
||||
if is_chat_completions_body(data):
|
||||
if "reasoning_effort" in data:
|
||||
return resolved
|
||||
return {**resolved, "reasoning_effort": variant.reasoning_effort}
|
||||
|
|
@ -604,7 +598,7 @@ async def cursor_chat_completions(
|
|||
|
||||
raw_body: Final = await read_request_body(request=request)
|
||||
|
||||
if _is_chat_completions_body(raw_body):
|
||||
if is_chat_completions_body(raw_body):
|
||||
# Genuine chat completions body (Cursor sends these for models whose BYOK it
|
||||
# already fixed); delegate so behavior matches /chat/completions exactly.
|
||||
# Keyed on messages CONTENT, not key presence: Cursor can send a null or
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from hashlib import sha256
|
|||
from importlib.resources import files
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from integration._support.client import Gateway, Scenario, eventually, list_value, object_value
|
||||
from integration._support.responses_vendor import same_response
|
||||
|
|
@ -150,11 +151,12 @@ def _chat_completion(stream: bool, identity: str) -> bytes | tuple[bytes, ...]:
|
|||
).encode()
|
||||
|
||||
|
||||
def _deployment(scenario: Scenario, wire: Wire) -> str:
|
||||
def _deployment(scenario: Scenario, wire: Wire, **pricing: JsonValue) -> str:
|
||||
model: Final = scenario.model(
|
||||
model=f"openai/{_MODEL}",
|
||||
api_key=_API_KEY,
|
||||
api_base=f"{wire.url}/v1",
|
||||
**pricing,
|
||||
)
|
||||
wait_for_worker_readiness(scenario.gateway, model, 1, _PROXY_WORKERS)
|
||||
return model
|
||||
|
|
@ -514,3 +516,53 @@ def test_cursor_messages_body_uses_chat_completions_wire(
|
|||
)
|
||||
assert (row["call_type"], row["team_id"]) == ("acompletion", team), row
|
||||
assert _REQUEST_TAG in list_value(row["request_tags"]), row
|
||||
|
||||
|
||||
def test_cursor_messages_body_keeps_client_and_key_tags_and_enforces_the_client_tag_budget(gateway: Gateway) -> None:
|
||||
chat_id: Final = f"chat_cursor_{uuid.uuid4().hex}"
|
||||
budget_tag: Final = f"cursor-tag-budget-{uuid.uuid4().hex}"
|
||||
key_tag: Final = f"cursor-key-tag-{uuid.uuid4().hex}"
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
if request.method == "GET" and request.target == "/v1/models":
|
||||
return model_discovery_reply(_MODEL)
|
||||
completion: Final = _chat_completion(False, chat_id)
|
||||
return Reply(body=completion if isinstance(completion, bytes) else b"")
|
||||
|
||||
def delete_tag() -> None:
|
||||
gateway.post("/tag/delete", {"name": budget_tag})
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _deployment(scenario, wire, input_cost_per_token=0.01, output_cost_per_token=0.01)
|
||||
key: Final = scenario.key(models=[model], metadata={"tags": [key_tag]})
|
||||
gateway.post("/tag/new", {"name": budget_tag, "max_budget": 0.0001})
|
||||
scenario.cleanups.callback(delete_tag)
|
||||
|
||||
def tagged_chat(text: str) -> httpx.Response:
|
||||
return gateway.request(
|
||||
"POST",
|
||||
"/cursor/chat/completions",
|
||||
{
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": text}],
|
||||
"metadata": {"tags": [budget_tag]},
|
||||
},
|
||||
key=key,
|
||||
)
|
||||
|
||||
first: Final = tagged_chat("first tagged chat")
|
||||
assert first.status_code == 200, first.text
|
||||
(row,) = eventually(
|
||||
lambda: gateway.rows(
|
||||
'SELECT request_tags FROM "LiteLLM_SpendLogs" WHERE api_key = %s',
|
||||
(sha256(key.encode()).hexdigest(),),
|
||||
),
|
||||
lambda rows: len(rows) == 1,
|
||||
seconds=70,
|
||||
)
|
||||
assert {budget_tag, key_tag} <= set(list_value(row["request_tags"])), row
|
||||
blocked: Final = eventually(
|
||||
lambda: tagged_chat("tag budget probe"), lambda response: response.status_code != 200, seconds=70
|
||||
)
|
||||
assert blocked.status_code == 422, blocked.text
|
||||
assert "budget" in blocked.text.lower(), blocked.text
|
||||
|
|
|
|||
|
|
@ -248,7 +248,7 @@ def _is_assistants(req):
|
|||
def _metadata_var_name(req):
|
||||
from litellm.proxy.litellm_pre_call_utils import get_metadata_variable_name
|
||||
|
||||
return get_metadata_variable_name(req)
|
||||
return get_metadata_variable_name(req, {})
|
||||
|
||||
|
||||
def _vector_store_id_in_path(req):
|
||||
|
|
|
|||
|
|
@ -3688,7 +3688,7 @@ class TestCursorInputArmFlattening:
|
|||
|
||||
class TestChatCompletionsBodyDetection:
|
||||
def test_routing_matrix(self):
|
||||
from litellm.proxy.response_api_endpoints.endpoints import _is_chat_completions_body
|
||||
from litellm.proxy.litellm_pre_call_utils import is_chat_completions_body as _is_chat_completions_body
|
||||
|
||||
assert _is_chat_completions_body({"messages": [{"role": "user", "content": "hi"}]}) is True
|
||||
assert _is_chat_completions_body({"messages": [{"role": "user", "content": "hi"}], "input": []}) is True
|
||||
|
|
|
|||
|
|
@ -9537,6 +9537,40 @@ async def test_saved_weights_override_caller_input_and_preserve_key_precedence(
|
|||
assert logging.pre_call_hook.call_args.kwargs["data"].get("_router_weights") == expected
|
||||
|
||||
|
||||
def _hook_data(**kwargs: object) -> object:
|
||||
return kwargs["data"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("prompt, proxy_field, client_field", [
|
||||
({"input": "hi"}, "litellm_metadata", "metadata"),
|
||||
({"messages": [{"role": "user", "content": "hi"}]}, "metadata", "litellm_metadata"),
|
||||
])
|
||||
async def test_cursor_route_writes_proxy_fields_only_to_the_field_its_body_shape_picks(
|
||||
prompt: dict[str, str | list[dict[str, str]]],
|
||||
proxy_field: str,
|
||||
client_field: str,
|
||||
) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
logging: Final = MagicMock(spec=ProxyLogging)
|
||||
logging.pre_call_hook = AsyncMock(side_effect=_hook_data)
|
||||
processor: Final = ProxyBaseLLMRequestProcessing(data={"model": "gpt-x", "metadata": {"client": "cursor"}, **prompt})
|
||||
data, _ = await processor.common_processing_pre_call_logic(
|
||||
request=Request({"type": "http", "method": "POST", "path": "/cursor/chat/completions", "headers": []}),
|
||||
general_settings={},
|
||||
user_api_key_dict=ProxyUserAPIKeyAuth(api_key="hash"),
|
||||
proxy_logging_obj=logging,
|
||||
proxy_config=proxy_server.ProxyConfig(),
|
||||
route_type="acompletion",
|
||||
logging_auth=ProxyUserAPIKeyAuth(api_key="logging-hash", key_alias="logging-alias"),
|
||||
)
|
||||
assert data[proxy_field]["user_api_key_alias"] == "logging-alias"
|
||||
assert "queue_time_seconds" in data[proxy_field]
|
||||
assert "user_api_key_alias" not in data.get(client_field, {})
|
||||
assert "queue_time_seconds" not in data.get(client_field, {})
|
||||
assert data["metadata"]["client"] == "cursor"
|
||||
|
||||
class TestPerRequestModelGroupAlias:
|
||||
"""``router_settings.model_group_alias`` on a key or team has to be resolved
|
||||
by the proxy: the Router resolves aliases from its own shared instance
|
||||
|
|
|
|||
|
|
@ -90,52 +90,65 @@ class TestGetMetadataVariableName:
|
|||
|
||||
def test_returns_litellm_metadata_for_thread_routes(self):
|
||||
request = self._make_request("/v1/threads/thread_123/messages")
|
||||
assert get_metadata_variable_name(request) == "litellm_metadata"
|
||||
assert get_metadata_variable_name(request, {}) == "litellm_metadata"
|
||||
|
||||
def test_returns_litellm_metadata_for_assistant_routes(self):
|
||||
request = self._make_request("/v1/assistants/asst_123")
|
||||
assert get_metadata_variable_name(request) == "litellm_metadata"
|
||||
assert get_metadata_variable_name(request, {}) == "litellm_metadata"
|
||||
|
||||
def test_returns_litellm_metadata_for_batches_route(self):
|
||||
request = self._make_request("/v1/batches")
|
||||
assert get_metadata_variable_name(request) == "litellm_metadata"
|
||||
assert get_metadata_variable_name(request, {}) == "litellm_metadata"
|
||||
|
||||
def test_returns_litellm_metadata_for_messages_route(self):
|
||||
request = self._make_request("/v1/messages")
|
||||
assert get_metadata_variable_name(request) == "litellm_metadata"
|
||||
assert get_metadata_variable_name(request, {}) == "litellm_metadata"
|
||||
|
||||
def test_returns_litellm_metadata_for_files_route(self):
|
||||
request = self._make_request("/v1/files")
|
||||
assert get_metadata_variable_name(request) == "litellm_metadata"
|
||||
assert get_metadata_variable_name(request, {}) == "litellm_metadata"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path", ["/v1beta/interactions", "/interactions", "/v1beta/interactions/v1_abc123", "/interactions/v1_abc123/cancel"]
|
||||
)
|
||||
def test_returns_litellm_metadata_for_interactions_routes(self, path):
|
||||
request = self._make_request(path)
|
||||
assert get_metadata_variable_name(request) == "litellm_metadata"
|
||||
assert get_metadata_variable_name(request, {}) == "litellm_metadata"
|
||||
|
||||
def test_returns_metadata_for_chat_completions(self):
|
||||
request = self._make_request("/chat/completions")
|
||||
assert get_metadata_variable_name(request) == "metadata"
|
||||
assert get_metadata_variable_name(request, {}) == "metadata"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"body, expected",
|
||||
[
|
||||
({"messages": [{"role": "user", "content": "hi"}]}, "metadata"),
|
||||
({"messages": []}, "metadata"),
|
||||
({"input": [{"role": "user", "content": "hi"}]}, "litellm_metadata"),
|
||||
({"input": "hi", "messages": []}, "litellm_metadata"),
|
||||
],
|
||||
)
|
||||
def test_cursor_route_picks_the_metadata_field_by_body_shape(self, body, expected):
|
||||
request = self._make_request("/cursor/chat/completions")
|
||||
assert get_metadata_variable_name(request, body) == expected
|
||||
|
||||
def test_returns_metadata_for_completions(self):
|
||||
request = self._make_request("/v1/completions")
|
||||
assert get_metadata_variable_name(request) == "metadata"
|
||||
assert get_metadata_variable_name(request, {}) == "metadata"
|
||||
|
||||
def test_returns_metadata_for_embeddings(self):
|
||||
request = self._make_request("/v1/embeddings")
|
||||
assert get_metadata_variable_name(request) == "metadata"
|
||||
assert get_metadata_variable_name(request, {}) == "metadata"
|
||||
|
||||
def test_returns_litellm_metadata_for_bedrock_invoke(self):
|
||||
# GH#30629: bedrock passthrough must use litellm_metadata
|
||||
# to prevent key-level tags from leaking into provider body
|
||||
request = self._make_request("/bedrock/model/us.anthropic.claude-sonnet-4-6/invoke")
|
||||
assert get_metadata_variable_name(request) == "litellm_metadata"
|
||||
assert get_metadata_variable_name(request, {}) == "litellm_metadata"
|
||||
|
||||
def test_returns_litellm_metadata_for_bedrock_converse(self):
|
||||
request = self._make_request("/bedrock/model/us.anthropic.claude-sonnet-4-6/converse")
|
||||
assert get_metadata_variable_name(request) == "litellm_metadata"
|
||||
assert get_metadata_variable_name(request, {}) == "litellm_metadata"
|
||||
|
||||
|
||||
def test_get_enforced_params_for_service_account_settings():
|
||||
|
|
@ -7844,7 +7857,7 @@ class TestPromotedTraceControlFields:
|
|||
)
|
||||
|
||||
def test_returns_litellm_metadata_for_responses_route(self):
|
||||
assert get_metadata_variable_name(self._make_request("/v1/responses")) == "litellm_metadata"
|
||||
assert get_metadata_variable_name(self._make_request("/v1/responses"), {}) == "litellm_metadata"
|
||||
|
||||
def test_promotes_trace_prefixed_and_allow_listed_fields(self):
|
||||
requester_metadata = {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue