From 1da2efd71f6d19ca1206a553ec570d316111d69c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:06:19 -0700 Subject: [PATCH] fix(proxy): apply x-litellm-priority only where the router schedules the request The header now maps onto priority only for chat and text completions that a router serves, so the Cursor route's responses call and proxies started without a config never forward it to the provider. /queue/chat/completions now honors the header too, with the header winning over a body priority. --- litellm/proxy/common_request_processing.py | 2 + litellm/proxy/litellm_pre_call_utils.py | 15 ++- litellm/proxy/proxy_server.py | 8 +- .../proxy/proxy_server/test_routes_queue.py | 41 +++++++++ .../proxy/test_common_request_processing.py | 91 +++++++++++++++++++ .../unit/proxy/test_litellm_pre_call_utils.py | 54 ----------- 6 files changed, 148 insertions(+), 63 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 24a7b46da4b..4be324cae6c 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -253,6 +253,7 @@ from litellm.proxy.anthropic_endpoints.streaming_model_restamp import ( AnthropicStreamModelRestamper, ) from litellm.proxy.litellm_pre_call_utils import ( + ROUTER_SCHEDULED_ROUTE_TYPES, LiteLLMProxyRequestSetup, add_litellm_data_to_request, get_metadata_variable_name, @@ -2265,6 +2266,7 @@ class ProxyBaseLLMRequestProcessing: user_api_key_dict=user_api_key_dict, version=version, proxy_config=proxy_config, + router_schedules_request=llm_router is not None and route_type in ROUTER_SCHEDULED_ROUTE_TYPES, ) if logging_auth is not None: metadata_variable_name: Final = get_metadata_variable_name(request) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 0ef91c580f6..242c7e412d4 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -114,15 +114,14 @@ _SESSION_ID_VALUE_RE: Final = re.compile(r"^[a-zA-Z0-9_\-]{8,}$") _SHA256_HEX_RE: Final = re.compile(r"^[0-9a-f]{64}$") +SCHEDULER_PRIORITY_HEADER: Final = "x-litellm-priority" +ROUTER_SCHEDULED_ROUTE_TYPES: Final = frozenset({"acompletion", "atext_completion"}) + # W3C Trace Context traceparent header: https://www.w3.org/TR/trace-context/ # e.g. "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01" _TRACEPARENT_RE: Final = re.compile(r"^[0-9a-f]{2}-([0-9a-f]{32})-[0-9a-f]{16}-[0-9a-f]{2}$", re.IGNORECASE) -def _is_scheduler_queued_route(path: str) -> bool: - return path.rstrip("/").endswith("/completions") - - def _trace_id_from_traceparent(traceparent: str) -> str | None: """Extract the trace-id from a W3C Trace Context traceparent header, e.g. "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01" -> the 32-hex @@ -2367,6 +2366,7 @@ async def add_litellm_data_to_request( proxy_config: ProxyConfig, general_settings: dict[str, Any] | None = None, version: str | None = None, + router_schedules_request: bool = False, ): """ Adds LiteLLM-specific data to the request. @@ -2501,10 +2501,9 @@ async def add_litellm_data_to_request( ) ) - if _is_scheduler_queued_route(request.url.path): - request_priority: Final = _headers.get("x-litellm-priority") - if request_priority is not None: - data["priority"] = request_priority # rebind-ok: data is an out-param + header_priority: Final = _headers.get(SCHEDULER_PRIORITY_HEADER) if router_schedules_request else None + if header_priority is not None: + data["priority"] = header_priority # rebind-ok: data is an out-param LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( headers=_headers, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c6d3e12a378..8a8d726e562 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -657,7 +657,11 @@ from litellm.proxy.list_api.common import ( problem_response, request_validation_problem, ) -from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, add_litellm_data_to_request +from litellm.proxy.litellm_pre_call_utils import ( + SCHEDULER_PRIORITY_HEADER, + LiteLLMProxyRequestSetup, + add_litellm_data_to_request, +) from litellm.proxy.logging_endpoints.callback_logs_endpoints import ( rust_control_plane_router, ) @@ -18267,6 +18271,8 @@ async def async_queue_request( raise HTTPException(status_code=500, detail={"error": CommonProxyErrors.no_llm_router.value}) router: Final = llm_router + header_priority: Final = request.headers.get(SCHEDULER_PRIORITY_HEADER) + data.update({} if header_priority is None else {"priority": header_priority}) queued_priority, _ = router.split_request_priority(model=data["model"], kwargs=data) if "stream" in data and data["stream"] is True: # use generate_responses to stream responses diff --git a/tests/unit/proxy/proxy_server/test_routes_queue.py b/tests/unit/proxy/proxy_server/test_routes_queue.py index 07a61c8587b..e2c83aaa252 100644 --- a/tests/unit/proxy/proxy_server/test_routes_queue.py +++ b/tests/unit/proxy/proxy_server/test_routes_queue.py @@ -169,6 +169,47 @@ def test_queue_chat_completions_rejects_a_request_it_cannot_queue_as_an_invalid_ assert add_request.await_count == 0 +@pytest.mark.parametrize( + ("priority_fields", "expected_priority"), + [({"priority": 5}, 0), ({}, 0)], + ids=("header-beats-body", "header-only"), +) +def test_queue_chat_completions_queues_at_the_priority_header( + client, + auth_as, + monkeypatch: pytest.MonkeyPatch, + priority_fields: dict[str, object], + expected_priority: int, +) -> None: + router: Final = _queue_router(monkeypatch, default_priority=5) + payload: Final = {"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}], **priority_fields} + with patch.object(router.scheduler, "add_request", wraps=router.scheduler.add_request) as add_request: + with auth_as(): + response: Final = client.post( + "/queue/chat/completions", json=payload, headers={"x-litellm-priority": "0"} + ) + assert response.status_code == 200, response.text + assert [call.kwargs["request"].priority for call in add_request.await_args_list] == [expected_priority] + assert response.headers.get("x-litellm-priority") == str(expected_priority) + + +def test_queue_chat_completions_rejects_a_malformed_priority_header_like_a_malformed_body_value( + client, auth_as, monkeypatch: pytest.MonkeyPatch +) -> None: + router: Final = _queue_router(monkeypatch, default_priority=5) + payload: Final = {"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}], "priority": 1} + with patch.object(router.scheduler, "add_request", wraps=router.scheduler.add_request) as add_request: + with auth_as(): + response: Final = client.post( + "/queue/chat/completions", json=payload, headers={"x-litellm-priority": "high"} + ) + assert response.status_code == 400, response.text + error: Final = response.json()["error"] + assert (error["type"], error["param"]) == ("invalid_request_error", "priority") + assert "priority must be an integer" in error["message"] + assert add_request.await_count == 0 + + def test_streamed_queue_responses_report_the_priority_the_router_queued_at( client, auth_as, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index f837dc3fb5f..be6af2ce278 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -512,6 +512,97 @@ class TestProxyBaseLLMRequestProcessing: assert exc_info.value.param == "model" add_litellm_data_to_request.assert_not_awaited() + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("path", "route_type", "has_router", "priority_header", "body_priority", "expected_priority"), + [ + ("/v1/chat/completions", "acompletion", True, "0", 5, "0"), + ("/v1/completions", "atext_completion", True, "0", 5, "0"), + ("/v1/chat/completions", "acompletion", True, "1", None, "1"), + ("/v1/completions", "atext_completion", True, "1", None, "1"), + ("/v1/chat/completions", "acompletion", True, None, 5, 5), + ("/cursor/chat/completions", "aresponses", True, "0", None, None), + ("/v1/responses", "aresponses", True, "0", 5, 5), + ("/v1/embeddings", "aembedding", True, "0", None, None), + ("/v1/chat/completions", "acompletion", False, "0", None, None), + ("/v1/chat/completions", "acompletion", False, "0", 5, 5), + ], + ids=( + "chat-header-beats-body", + "text-completion-header-beats-body", + "chat-header-only", + "text-completion-header-only", + "chat-body-only", + "cursor-responses-route-ignores-header", + "responses-keeps-body", + "embeddings-ignores-header", + "routerless-ignores-header", + "routerless-keeps-body", + ), + ) + async def test_common_processing_pre_call_logic_applies_priority_header_only_where_the_router_schedules( + self, + path: str, + route_type: Literal["acompletion", "atext_completion", "aresponses", "aembedding"], + has_router: bool, + priority_header: str | None, + body_priority: int | None, + expected_priority: str | int | None, + ) -> None: + from litellm.proxy.proxy_server import ProxyConfig as RealProxyConfig + + header_entries: Final[tuple[tuple[bytes, bytes], ...]] = ( + () if priority_header is None else ((b"x-litellm-priority", priority_header.encode()),) + ) + request: Final = Request( + { + "type": "http", + "method": "POST", + "path": path, + "scheme": "http", + "server": ("localhost", 4000), + "client": ("127.0.0.1", 12345), + "query_string": b"", + "headers": ((b"content-type", b"application/json"), *header_entries), + } + ) + router: Final = ( + Router( + model_list=[ + { + "model_name": "qa-model", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-test", "mock_response": "hi"}, + } + ] + ) + if has_router + else None + ) + processing_obj: Final = ProxyBaseLLMRequestProcessing( + data={ + "model": "qa-model", + "messages": [{"role": "user", "content": "hi"}], + **({"priority": body_priority} if body_priority is not None else {}), + } + ) + proxy_logging_obj: Final = MagicMock(spec=ProxyLogging) + proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"]) + + returned_data, _ = await processing_obj.common_processing_pre_call_logic( + request=request, + general_settings={}, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + proxy_logging_obj=proxy_logging_obj, + proxy_config=RealProxyConfig(), + route_type=route_type, + llm_router=router, + ) + + assert ("priority" in returned_data, returned_data.get("priority")) == ( + expected_priority is not None, + expected_priority, + ) + @pytest.mark.asyncio @pytest.mark.parametrize("safe_memory_mode", [False, True]) @pytest.mark.parametrize( diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 18de1cd9ecc..246496ec48f 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -2684,60 +2684,6 @@ def test_add_litellm_data_for_backend_llm_call_merges_keepalive_seconds_header() assert result.get("keepalive_seconds") == 20.0 -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("path", "priority_header", "body_priority", "expected_priority", "expected_priority_present"), - [ - ("/v1/chat/completions", "0", 5, "0", True), - ("/v1/completions", "0", 5, "0", True), - ("/v1/embeddings", "0", None, None, False), - ("/v1/responses", "0", None, None, False), - ("/v1/embeddings", "0", 5, 5, True), - ("/v1/responses", "0", 5, 5, True), - ("/v1/chat/completions", None, 5, 5, True), - ], -) -async def test_add_litellm_data_to_request_applies_priority_header_only_to_completion_routes( - path: str, - priority_header: str | None, - body_priority: int | None, - expected_priority: int | str | None, - expected_priority_present: bool, -) -> None: - from litellm.proxy.proxy_server import ProxyConfig - - header_entries: Final[tuple[tuple[bytes, bytes], ...]] = ( - () if priority_header is None else ((b"x-litellm-priority", priority_header.encode()),) - ) - request_headers: Final = ((b"content-type", b"application/json"), *header_entries) - request: Final = Request( - { - "type": "http", - "method": "POST", - "path": path, - "scheme": "http", - "server": ("localhost", 4000), - "client": ("127.0.0.1", 12345), - "query_string": b"", - "headers": request_headers, - } - ) - request_data: Final[dict[str, object]] = { - "model": "qa-model", - **({"priority": body_priority} if body_priority is not None else {}), - } - updated_data: Final = await add_litellm_data_to_request( - data=request_data, - request=request, - user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), - proxy_config=ProxyConfig(), - general_settings={}, - ) - assert ("priority" in updated_data) is expected_priority_present - if expected_priority_present: - assert updated_data["priority"] == expected_priority - - def test_add_user_api_key_auth_to_request_metadata(): """ Test that add_user_api_key_auth_to_request_metadata properly adds user API key authentication data to request metadata