fix(proxy): apply x-litellm-priority only where the router schedules the request

The header now maps onto priority only for chat and text completions that a
router serves, so the Cursor route's responses call and proxies started without
a config never forward it to the provider. /queue/chat/completions now honors
the header too, with the header winning over a body priority.
This commit is contained in:
mateo-berri 2026-10-20 15:06:19 -07:00
parent e272786a87
commit 1da2efd71f
6 changed files with 148 additions and 63 deletions

View file

@ -253,6 +253,7 @@ from litellm.proxy.anthropic_endpoints.streaming_model_restamp import (
AnthropicStreamModelRestamper,
)
from litellm.proxy.litellm_pre_call_utils import (
ROUTER_SCHEDULED_ROUTE_TYPES,
LiteLLMProxyRequestSetup,
add_litellm_data_to_request,
get_metadata_variable_name,
@ -2265,6 +2266,7 @@ class ProxyBaseLLMRequestProcessing:
user_api_key_dict=user_api_key_dict,
version=version,
proxy_config=proxy_config,
router_schedules_request=llm_router is not None and route_type in ROUTER_SCHEDULED_ROUTE_TYPES,
)
if logging_auth is not None:
metadata_variable_name: Final = get_metadata_variable_name(request)

View file

@ -114,15 +114,14 @@ _SESSION_ID_VALUE_RE: Final = re.compile(r"^[a-zA-Z0-9_\-]{8,}$")
_SHA256_HEX_RE: Final = re.compile(r"^[0-9a-f]{64}$")
SCHEDULER_PRIORITY_HEADER: Final = "x-litellm-priority"
ROUTER_SCHEDULED_ROUTE_TYPES: Final = frozenset({"acompletion", "atext_completion"})
# W3C Trace Context traceparent header: https://www.w3.org/TR/trace-context/
# e.g. "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"
_TRACEPARENT_RE: Final = re.compile(r"^[0-9a-f]{2}-([0-9a-f]{32})-[0-9a-f]{16}-[0-9a-f]{2}$", re.IGNORECASE)
def _is_scheduler_queued_route(path: str) -> bool:
return path.rstrip("/").endswith("/completions")
def _trace_id_from_traceparent(traceparent: str) -> str | None:
"""Extract the trace-id from a W3C Trace Context traceparent header, e.g.
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01" -> the 32-hex
@ -2367,6 +2366,7 @@ async def add_litellm_data_to_request(
proxy_config: ProxyConfig,
general_settings: dict[str, Any] | None = None,
version: str | None = None,
router_schedules_request: bool = False,
):
"""
Adds LiteLLM-specific data to the request.
@ -2501,10 +2501,9 @@ async def add_litellm_data_to_request(
)
)
if _is_scheduler_queued_route(request.url.path):
request_priority: Final = _headers.get("x-litellm-priority")
if request_priority is not None:
data["priority"] = request_priority # rebind-ok: data is an out-param
header_priority: Final = _headers.get(SCHEDULER_PRIORITY_HEADER) if router_schedules_request else None
if header_priority is not None:
data["priority"] = header_priority # rebind-ok: data is an out-param
LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers(
headers=_headers,

View file

@ -657,7 +657,11 @@ from litellm.proxy.list_api.common import (
problem_response,
request_validation_problem,
)
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, add_litellm_data_to_request
from litellm.proxy.litellm_pre_call_utils import (
SCHEDULER_PRIORITY_HEADER,
LiteLLMProxyRequestSetup,
add_litellm_data_to_request,
)
from litellm.proxy.logging_endpoints.callback_logs_endpoints import (
rust_control_plane_router,
)
@ -18267,6 +18271,8 @@ async def async_queue_request(
raise HTTPException(status_code=500, detail={"error": CommonProxyErrors.no_llm_router.value})
router: Final = llm_router
header_priority: Final = request.headers.get(SCHEDULER_PRIORITY_HEADER)
data.update({} if header_priority is None else {"priority": header_priority})
queued_priority, _ = router.split_request_priority(model=data["model"], kwargs=data)
if "stream" in data and data["stream"] is True: # use generate_responses to stream responses

View file

@ -169,6 +169,47 @@ def test_queue_chat_completions_rejects_a_request_it_cannot_queue_as_an_invalid_
assert add_request.await_count == 0
@pytest.mark.parametrize(
("priority_fields", "expected_priority"),
[({"priority": 5}, 0), ({}, 0)],
ids=("header-beats-body", "header-only"),
)
def test_queue_chat_completions_queues_at_the_priority_header(
client,
auth_as,
monkeypatch: pytest.MonkeyPatch,
priority_fields: dict[str, object],
expected_priority: int,
) -> None:
router: Final = _queue_router(monkeypatch, default_priority=5)
payload: Final = {"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}], **priority_fields}
with patch.object(router.scheduler, "add_request", wraps=router.scheduler.add_request) as add_request:
with auth_as():
response: Final = client.post(
"/queue/chat/completions", json=payload, headers={"x-litellm-priority": "0"}
)
assert response.status_code == 200, response.text
assert [call.kwargs["request"].priority for call in add_request.await_args_list] == [expected_priority]
assert response.headers.get("x-litellm-priority") == str(expected_priority)
def test_queue_chat_completions_rejects_a_malformed_priority_header_like_a_malformed_body_value(
client, auth_as, monkeypatch: pytest.MonkeyPatch
) -> None:
router: Final = _queue_router(monkeypatch, default_priority=5)
payload: Final = {"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}], "priority": 1}
with patch.object(router.scheduler, "add_request", wraps=router.scheduler.add_request) as add_request:
with auth_as():
response: Final = client.post(
"/queue/chat/completions", json=payload, headers={"x-litellm-priority": "high"}
)
assert response.status_code == 400, response.text
error: Final = response.json()["error"]
assert (error["type"], error["param"]) == ("invalid_request_error", "priority")
assert "priority must be an integer" in error["message"]
assert add_request.await_count == 0
def test_streamed_queue_responses_report_the_priority_the_router_queued_at(
client, auth_as, monkeypatch: pytest.MonkeyPatch
) -> None:

View file

@ -512,6 +512,97 @@ class TestProxyBaseLLMRequestProcessing:
assert exc_info.value.param == "model"
add_litellm_data_to_request.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.parametrize(
("path", "route_type", "has_router", "priority_header", "body_priority", "expected_priority"),
[
("/v1/chat/completions", "acompletion", True, "0", 5, "0"),
("/v1/completions", "atext_completion", True, "0", 5, "0"),
("/v1/chat/completions", "acompletion", True, "1", None, "1"),
("/v1/completions", "atext_completion", True, "1", None, "1"),
("/v1/chat/completions", "acompletion", True, None, 5, 5),
("/cursor/chat/completions", "aresponses", True, "0", None, None),
("/v1/responses", "aresponses", True, "0", 5, 5),
("/v1/embeddings", "aembedding", True, "0", None, None),
("/v1/chat/completions", "acompletion", False, "0", None, None),
("/v1/chat/completions", "acompletion", False, "0", 5, 5),
],
ids=(
"chat-header-beats-body",
"text-completion-header-beats-body",
"chat-header-only",
"text-completion-header-only",
"chat-body-only",
"cursor-responses-route-ignores-header",
"responses-keeps-body",
"embeddings-ignores-header",
"routerless-ignores-header",
"routerless-keeps-body",
),
)
async def test_common_processing_pre_call_logic_applies_priority_header_only_where_the_router_schedules(
self,
path: str,
route_type: Literal["acompletion", "atext_completion", "aresponses", "aembedding"],
has_router: bool,
priority_header: str | None,
body_priority: int | None,
expected_priority: str | int | None,
) -> None:
from litellm.proxy.proxy_server import ProxyConfig as RealProxyConfig
header_entries: Final[tuple[tuple[bytes, bytes], ...]] = (
() if priority_header is None else ((b"x-litellm-priority", priority_header.encode()),)
)
request: Final = Request(
{
"type": "http",
"method": "POST",
"path": path,
"scheme": "http",
"server": ("localhost", 4000),
"client": ("127.0.0.1", 12345),
"query_string": b"",
"headers": ((b"content-type", b"application/json"), *header_entries),
}
)
router: Final = (
Router(
model_list=[
{
"model_name": "qa-model",
"litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-test", "mock_response": "hi"},
}
]
)
if has_router
else None
)
processing_obj: Final = ProxyBaseLLMRequestProcessing(
data={
"model": "qa-model",
"messages": [{"role": "user", "content": "hi"}],
**({"priority": body_priority} if body_priority is not None else {}),
}
)
proxy_logging_obj: Final = MagicMock(spec=ProxyLogging)
proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"])
returned_data, _ = await processing_obj.common_processing_pre_call_logic(
request=request,
general_settings={},
user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"),
proxy_logging_obj=proxy_logging_obj,
proxy_config=RealProxyConfig(),
route_type=route_type,
llm_router=router,
)
assert ("priority" in returned_data, returned_data.get("priority")) == (
expected_priority is not None,
expected_priority,
)
@pytest.mark.asyncio
@pytest.mark.parametrize("safe_memory_mode", [False, True])
@pytest.mark.parametrize(

View file

@ -2684,60 +2684,6 @@ def test_add_litellm_data_for_backend_llm_call_merges_keepalive_seconds_header()
assert result.get("keepalive_seconds") == 20.0
@pytest.mark.asyncio
@pytest.mark.parametrize(
("path", "priority_header", "body_priority", "expected_priority", "expected_priority_present"),
[
("/v1/chat/completions", "0", 5, "0", True),
("/v1/completions", "0", 5, "0", True),
("/v1/embeddings", "0", None, None, False),
("/v1/responses", "0", None, None, False),
("/v1/embeddings", "0", 5, 5, True),
("/v1/responses", "0", 5, 5, True),
("/v1/chat/completions", None, 5, 5, True),
],
)
async def test_add_litellm_data_to_request_applies_priority_header_only_to_completion_routes(
path: str,
priority_header: str | None,
body_priority: int | None,
expected_priority: int | str | None,
expected_priority_present: bool,
) -> None:
from litellm.proxy.proxy_server import ProxyConfig
header_entries: Final[tuple[tuple[bytes, bytes], ...]] = (
() if priority_header is None else ((b"x-litellm-priority", priority_header.encode()),)
)
request_headers: Final = ((b"content-type", b"application/json"), *header_entries)
request: Final = Request(
{
"type": "http",
"method": "POST",
"path": path,
"scheme": "http",
"server": ("localhost", 4000),
"client": ("127.0.0.1", 12345),
"query_string": b"",
"headers": request_headers,
}
)
request_data: Final[dict[str, object]] = {
"model": "qa-model",
**({"priority": body_priority} if body_priority is not None else {}),
}
updated_data: Final = await add_litellm_data_to_request(
data=request_data,
request=request,
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
proxy_config=ProxyConfig(),
general_settings={},
)
assert ("priority" in updated_data) is expected_priority_present
if expected_priority_present:
assert updated_data["priority"] == expected_priority
def test_add_user_api_key_auth_to_request_metadata():
"""
Test that add_user_api_key_auth_to_request_metadata properly adds user API key authentication data to request metadata