From 4ba2cc27dc6340a015821e4127d632e0d97ae0ef Mon Sep 17 00:00:00 2001 From: Mubashir Osmani Date: Tue, 20 Oct 2026 19:03:01 +0000 Subject: [PATCH 1/6] feat(proxy): accept request priority from the x-litellm-priority header Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_types.py | 1 + litellm/proxy/litellm_pre_call_utils.py | 7 +++++ .../unit/proxy/test_litellm_pre_call_utils.py | 31 +++++++++++++++++++ 3 files changed, 39 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 315533d6d0f..a651d735399 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -4468,6 +4468,7 @@ class JWTKeyMappingResponse(LiteLLMPydanticObjectBase): class LitellmDataForBackendLLMCall(TypedDict, total=False): headers: dict organization: str + priority: str # writable-ok: request setup adds caller priority before routing timeout: float | None stream_timeout: float | None user: str | None diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index efc8575feb3..9fe4aed6d59 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1724,6 +1724,13 @@ class LiteLLMProxyRequestSetup: if num_retries is not None: data["num_retries"] = num_retries + setup: Final = LiteLLMProxyRequestSetup + priority: Final = setup._get_case_insensitive_header( # pyright: ignore[reportUnknownMemberType] # legacy dict + headers, "x-litellm-priority" + ) + if priority is not None: + data["priority"] = priority + keepalive_seconds: Final = LiteLLMProxyRequestSetup._get_keepalive_seconds_from_request(headers) if keepalive_seconds is not None: data["keepalive_seconds"] = keepalive_seconds diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 246496ec48f..5edb453b7b1 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -2684,6 +2684,37 @@ def test_add_litellm_data_for_backend_llm_call_merges_keepalive_seconds_header() assert result.get("keepalive_seconds") == 20.0 +def test_add_litellm_data_for_backend_llm_call_priority_header_precedence() -> None: + header_only: Final = LiteLLMProxyRequestSetup.add_litellm_data_for_backend_llm_call( + headers={"x-litellm-priority": "0"}, + request_data={}, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + ) + assert header_only.get("priority") == "0" + + body_priority: Final = {"priority": 5} + header_overrides_body: Final = { + **body_priority, + **LiteLLMProxyRequestSetup.add_litellm_data_for_backend_llm_call( + headers={"x-litellm-priority": "0"}, + request_data=body_priority, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + ), + } + assert header_overrides_body["priority"] == "0" + + body_without_header: Final = {"priority": 5} + body_priority_untouched: Final = { + **body_without_header, + **LiteLLMProxyRequestSetup.add_litellm_data_for_backend_llm_call( + headers={}, + request_data=body_without_header, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + ), + } + assert body_priority_untouched["priority"] == 5 + + def test_add_user_api_key_auth_to_request_metadata(): """ Test that add_user_api_key_auth_to_request_metadata properly adds user API key authentication data to request metadata From e272786a875225ab5cc3afbc40544ce0ce8c3e1b Mon Sep 17 00:00:00 2001 From: Mubashir Osmani Date: Tue, 20 Oct 2026 20:01:12 +0000 Subject: [PATCH 2/6] fix(proxy): only apply x-litellm-priority on scheduler-queued routes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_types.py | 1 - litellm/proxy/litellm_pre_call_utils.py | 16 ++-- .../unit/proxy/test_litellm_pre_call_utils.py | 77 ++++++++++++------- 3 files changed, 59 insertions(+), 35 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index a651d735399..315533d6d0f 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -4468,7 +4468,6 @@ class JWTKeyMappingResponse(LiteLLMPydanticObjectBase): class LitellmDataForBackendLLMCall(TypedDict, total=False): headers: dict organization: str - priority: str # writable-ok: request setup adds caller priority before routing timeout: float | None stream_timeout: float | None user: str | None diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 9fe4aed6d59..0ef91c580f6 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -119,6 +119,10 @@ _SHA256_HEX_RE: Final = re.compile(r"^[0-9a-f]{64}$") _TRACEPARENT_RE: Final = re.compile(r"^[0-9a-f]{2}-([0-9a-f]{32})-[0-9a-f]{16}-[0-9a-f]{2}$", re.IGNORECASE) +def _is_scheduler_queued_route(path: str) -> bool: + return path.rstrip("/").endswith("/completions") + + def _trace_id_from_traceparent(traceparent: str) -> str | None: """Extract the trace-id from a W3C Trace Context traceparent header, e.g. "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01" -> the 32-hex @@ -1724,13 +1728,6 @@ class LiteLLMProxyRequestSetup: if num_retries is not None: data["num_retries"] = num_retries - setup: Final = LiteLLMProxyRequestSetup - priority: Final = setup._get_case_insensitive_header( # pyright: ignore[reportUnknownMemberType] # legacy dict - headers, "x-litellm-priority" - ) - if priority is not None: - data["priority"] = priority - keepalive_seconds: Final = LiteLLMProxyRequestSetup._get_keepalive_seconds_from_request(headers) if keepalive_seconds is not None: data["keepalive_seconds"] = keepalive_seconds @@ -2504,6 +2501,11 @@ async def add_litellm_data_to_request( ) ) + if _is_scheduler_queued_route(request.url.path): + request_priority: Final = _headers.get("x-litellm-priority") + if request_priority is not None: + data["priority"] = request_priority # rebind-ok: data is an out-param + LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( headers=_headers, data=data, diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 5edb453b7b1..18de1cd9ecc 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -2684,35 +2684,58 @@ def test_add_litellm_data_for_backend_llm_call_merges_keepalive_seconds_header() assert result.get("keepalive_seconds") == 20.0 -def test_add_litellm_data_for_backend_llm_call_priority_header_precedence() -> None: - header_only: Final = LiteLLMProxyRequestSetup.add_litellm_data_for_backend_llm_call( - headers={"x-litellm-priority": "0"}, - request_data={}, - user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("path", "priority_header", "body_priority", "expected_priority", "expected_priority_present"), + [ + ("/v1/chat/completions", "0", 5, "0", True), + ("/v1/completions", "0", 5, "0", True), + ("/v1/embeddings", "0", None, None, False), + ("/v1/responses", "0", None, None, False), + ("/v1/embeddings", "0", 5, 5, True), + ("/v1/responses", "0", 5, 5, True), + ("/v1/chat/completions", None, 5, 5, True), + ], +) +async def test_add_litellm_data_to_request_applies_priority_header_only_to_completion_routes( + path: str, + priority_header: str | None, + body_priority: int | None, + expected_priority: int | str | None, + expected_priority_present: bool, +) -> None: + from litellm.proxy.proxy_server import ProxyConfig + + header_entries: Final[tuple[tuple[bytes, bytes], ...]] = ( + () if priority_header is None else ((b"x-litellm-priority", priority_header.encode()),) ) - assert header_only.get("priority") == "0" - - body_priority: Final = {"priority": 5} - header_overrides_body: Final = { - **body_priority, - **LiteLLMProxyRequestSetup.add_litellm_data_for_backend_llm_call( - headers={"x-litellm-priority": "0"}, - request_data=body_priority, - user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), - ), + request_headers: Final = ((b"content-type", b"application/json"), *header_entries) + request: Final = Request( + { + "type": "http", + "method": "POST", + "path": path, + "scheme": "http", + "server": ("localhost", 4000), + "client": ("127.0.0.1", 12345), + "query_string": b"", + "headers": request_headers, + } + ) + request_data: Final[dict[str, object]] = { + "model": "qa-model", + **({"priority": body_priority} if body_priority is not None else {}), } - assert header_overrides_body["priority"] == "0" - - body_without_header: Final = {"priority": 5} - body_priority_untouched: Final = { - **body_without_header, - **LiteLLMProxyRequestSetup.add_litellm_data_for_backend_llm_call( - headers={}, - request_data=body_without_header, - user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), - ), - } - assert body_priority_untouched["priority"] == 5 + updated_data: Final = await add_litellm_data_to_request( + data=request_data, + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + proxy_config=ProxyConfig(), + general_settings={}, + ) + assert ("priority" in updated_data) is expected_priority_present + if expected_priority_present: + assert updated_data["priority"] == expected_priority def test_add_user_api_key_auth_to_request_metadata(): From 1da2efd71f6d19ca1206a553ec570d316111d69c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:06:19 -0700 Subject: [PATCH 3/6] fix(proxy): apply x-litellm-priority only where the router schedules the request The header now maps onto priority only for chat and text completions that a router serves, so the Cursor route's responses call and proxies started without a config never forward it to the provider. /queue/chat/completions now honors the header too, with the header winning over a body priority. --- litellm/proxy/common_request_processing.py | 2 + litellm/proxy/litellm_pre_call_utils.py | 15 ++- litellm/proxy/proxy_server.py | 8 +- .../proxy/proxy_server/test_routes_queue.py | 41 +++++++++ .../proxy/test_common_request_processing.py | 91 +++++++++++++++++++ .../unit/proxy/test_litellm_pre_call_utils.py | 54 ----------- 6 files changed, 148 insertions(+), 63 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 24a7b46da4b..4be324cae6c 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -253,6 +253,7 @@ from litellm.proxy.anthropic_endpoints.streaming_model_restamp import ( AnthropicStreamModelRestamper, ) from litellm.proxy.litellm_pre_call_utils import ( + ROUTER_SCHEDULED_ROUTE_TYPES, LiteLLMProxyRequestSetup, add_litellm_data_to_request, get_metadata_variable_name, @@ -2265,6 +2266,7 @@ class ProxyBaseLLMRequestProcessing: user_api_key_dict=user_api_key_dict, version=version, proxy_config=proxy_config, + router_schedules_request=llm_router is not None and route_type in ROUTER_SCHEDULED_ROUTE_TYPES, ) if logging_auth is not None: metadata_variable_name: Final = get_metadata_variable_name(request) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 0ef91c580f6..242c7e412d4 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -114,15 +114,14 @@ _SESSION_ID_VALUE_RE: Final = re.compile(r"^[a-zA-Z0-9_\-]{8,}$") _SHA256_HEX_RE: Final = re.compile(r"^[0-9a-f]{64}$") +SCHEDULER_PRIORITY_HEADER: Final = "x-litellm-priority" +ROUTER_SCHEDULED_ROUTE_TYPES: Final = frozenset({"acompletion", "atext_completion"}) + # W3C Trace Context traceparent header: https://www.w3.org/TR/trace-context/ # e.g. "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01" _TRACEPARENT_RE: Final = re.compile(r"^[0-9a-f]{2}-([0-9a-f]{32})-[0-9a-f]{16}-[0-9a-f]{2}$", re.IGNORECASE) -def _is_scheduler_queued_route(path: str) -> bool: - return path.rstrip("/").endswith("/completions") - - def _trace_id_from_traceparent(traceparent: str) -> str | None: """Extract the trace-id from a W3C Trace Context traceparent header, e.g. "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01" -> the 32-hex @@ -2367,6 +2366,7 @@ async def add_litellm_data_to_request( proxy_config: ProxyConfig, general_settings: dict[str, Any] | None = None, version: str | None = None, + router_schedules_request: bool = False, ): """ Adds LiteLLM-specific data to the request. @@ -2501,10 +2501,9 @@ async def add_litellm_data_to_request( ) ) - if _is_scheduler_queued_route(request.url.path): - request_priority: Final = _headers.get("x-litellm-priority") - if request_priority is not None: - data["priority"] = request_priority # rebind-ok: data is an out-param + header_priority: Final = _headers.get(SCHEDULER_PRIORITY_HEADER) if router_schedules_request else None + if header_priority is not None: + data["priority"] = header_priority # rebind-ok: data is an out-param LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( headers=_headers, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c6d3e12a378..8a8d726e562 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -657,7 +657,11 @@ from litellm.proxy.list_api.common import ( problem_response, request_validation_problem, ) -from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, add_litellm_data_to_request +from litellm.proxy.litellm_pre_call_utils import ( + SCHEDULER_PRIORITY_HEADER, + LiteLLMProxyRequestSetup, + add_litellm_data_to_request, +) from litellm.proxy.logging_endpoints.callback_logs_endpoints import ( rust_control_plane_router, ) @@ -18267,6 +18271,8 @@ async def async_queue_request( raise HTTPException(status_code=500, detail={"error": CommonProxyErrors.no_llm_router.value}) router: Final = llm_router + header_priority: Final = request.headers.get(SCHEDULER_PRIORITY_HEADER) + data.update({} if header_priority is None else {"priority": header_priority}) queued_priority, _ = router.split_request_priority(model=data["model"], kwargs=data) if "stream" in data and data["stream"] is True: # use generate_responses to stream responses diff --git a/tests/unit/proxy/proxy_server/test_routes_queue.py b/tests/unit/proxy/proxy_server/test_routes_queue.py index 07a61c8587b..e2c83aaa252 100644 --- a/tests/unit/proxy/proxy_server/test_routes_queue.py +++ b/tests/unit/proxy/proxy_server/test_routes_queue.py @@ -169,6 +169,47 @@ def test_queue_chat_completions_rejects_a_request_it_cannot_queue_as_an_invalid_ assert add_request.await_count == 0 +@pytest.mark.parametrize( + ("priority_fields", "expected_priority"), + [({"priority": 5}, 0), ({}, 0)], + ids=("header-beats-body", "header-only"), +) +def test_queue_chat_completions_queues_at_the_priority_header( + client, + auth_as, + monkeypatch: pytest.MonkeyPatch, + priority_fields: dict[str, object], + expected_priority: int, +) -> None: + router: Final = _queue_router(monkeypatch, default_priority=5) + payload: Final = {"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}], **priority_fields} + with patch.object(router.scheduler, "add_request", wraps=router.scheduler.add_request) as add_request: + with auth_as(): + response: Final = client.post( + "/queue/chat/completions", json=payload, headers={"x-litellm-priority": "0"} + ) + assert response.status_code == 200, response.text + assert [call.kwargs["request"].priority for call in add_request.await_args_list] == [expected_priority] + assert response.headers.get("x-litellm-priority") == str(expected_priority) + + +def test_queue_chat_completions_rejects_a_malformed_priority_header_like_a_malformed_body_value( + client, auth_as, monkeypatch: pytest.MonkeyPatch +) -> None: + router: Final = _queue_router(monkeypatch, default_priority=5) + payload: Final = {"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}], "priority": 1} + with patch.object(router.scheduler, "add_request", wraps=router.scheduler.add_request) as add_request: + with auth_as(): + response: Final = client.post( + "/queue/chat/completions", json=payload, headers={"x-litellm-priority": "high"} + ) + assert response.status_code == 400, response.text + error: Final = response.json()["error"] + assert (error["type"], error["param"]) == ("invalid_request_error", "priority") + assert "priority must be an integer" in error["message"] + assert add_request.await_count == 0 + + def test_streamed_queue_responses_report_the_priority_the_router_queued_at( client, auth_as, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index f837dc3fb5f..be6af2ce278 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -512,6 +512,97 @@ class TestProxyBaseLLMRequestProcessing: assert exc_info.value.param == "model" add_litellm_data_to_request.assert_not_awaited() + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("path", "route_type", "has_router", "priority_header", "body_priority", "expected_priority"), + [ + ("/v1/chat/completions", "acompletion", True, "0", 5, "0"), + ("/v1/completions", "atext_completion", True, "0", 5, "0"), + ("/v1/chat/completions", "acompletion", True, "1", None, "1"), + ("/v1/completions", "atext_completion", True, "1", None, "1"), + ("/v1/chat/completions", "acompletion", True, None, 5, 5), + ("/cursor/chat/completions", "aresponses", True, "0", None, None), + ("/v1/responses", "aresponses", True, "0", 5, 5), + ("/v1/embeddings", "aembedding", True, "0", None, None), + ("/v1/chat/completions", "acompletion", False, "0", None, None), + ("/v1/chat/completions", "acompletion", False, "0", 5, 5), + ], + ids=( + "chat-header-beats-body", + "text-completion-header-beats-body", + "chat-header-only", + "text-completion-header-only", + "chat-body-only", + "cursor-responses-route-ignores-header", + "responses-keeps-body", + "embeddings-ignores-header", + "routerless-ignores-header", + "routerless-keeps-body", + ), + ) + async def test_common_processing_pre_call_logic_applies_priority_header_only_where_the_router_schedules( + self, + path: str, + route_type: Literal["acompletion", "atext_completion", "aresponses", "aembedding"], + has_router: bool, + priority_header: str | None, + body_priority: int | None, + expected_priority: str | int | None, + ) -> None: + from litellm.proxy.proxy_server import ProxyConfig as RealProxyConfig + + header_entries: Final[tuple[tuple[bytes, bytes], ...]] = ( + () if priority_header is None else ((b"x-litellm-priority", priority_header.encode()),) + ) + request: Final = Request( + { + "type": "http", + "method": "POST", + "path": path, + "scheme": "http", + "server": ("localhost", 4000), + "client": ("127.0.0.1", 12345), + "query_string": b"", + "headers": ((b"content-type", b"application/json"), *header_entries), + } + ) + router: Final = ( + Router( + model_list=[ + { + "model_name": "qa-model", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-test", "mock_response": "hi"}, + } + ] + ) + if has_router + else None + ) + processing_obj: Final = ProxyBaseLLMRequestProcessing( + data={ + "model": "qa-model", + "messages": [{"role": "user", "content": "hi"}], + **({"priority": body_priority} if body_priority is not None else {}), + } + ) + proxy_logging_obj: Final = MagicMock(spec=ProxyLogging) + proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"]) + + returned_data, _ = await processing_obj.common_processing_pre_call_logic( + request=request, + general_settings={}, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + proxy_logging_obj=proxy_logging_obj, + proxy_config=RealProxyConfig(), + route_type=route_type, + llm_router=router, + ) + + assert ("priority" in returned_data, returned_data.get("priority")) == ( + expected_priority is not None, + expected_priority, + ) + @pytest.mark.asyncio @pytest.mark.parametrize("safe_memory_mode", [False, True]) @pytest.mark.parametrize( diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 18de1cd9ecc..246496ec48f 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -2684,60 +2684,6 @@ def test_add_litellm_data_for_backend_llm_call_merges_keepalive_seconds_header() assert result.get("keepalive_seconds") == 20.0 -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("path", "priority_header", "body_priority", "expected_priority", "expected_priority_present"), - [ - ("/v1/chat/completions", "0", 5, "0", True), - ("/v1/completions", "0", 5, "0", True), - ("/v1/embeddings", "0", None, None, False), - ("/v1/responses", "0", None, None, False), - ("/v1/embeddings", "0", 5, 5, True), - ("/v1/responses", "0", 5, 5, True), - ("/v1/chat/completions", None, 5, 5, True), - ], -) -async def test_add_litellm_data_to_request_applies_priority_header_only_to_completion_routes( - path: str, - priority_header: str | None, - body_priority: int | None, - expected_priority: int | str | None, - expected_priority_present: bool, -) -> None: - from litellm.proxy.proxy_server import ProxyConfig - - header_entries: Final[tuple[tuple[bytes, bytes], ...]] = ( - () if priority_header is None else ((b"x-litellm-priority", priority_header.encode()),) - ) - request_headers: Final = ((b"content-type", b"application/json"), *header_entries) - request: Final = Request( - { - "type": "http", - "method": "POST", - "path": path, - "scheme": "http", - "server": ("localhost", 4000), - "client": ("127.0.0.1", 12345), - "query_string": b"", - "headers": request_headers, - } - ) - request_data: Final[dict[str, object]] = { - "model": "qa-model", - **({"priority": body_priority} if body_priority is not None else {}), - } - updated_data: Final = await add_litellm_data_to_request( - data=request_data, - request=request, - user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), - proxy_config=ProxyConfig(), - general_settings={}, - ) - assert ("priority" in updated_data) is expected_priority_present - if expected_priority_present: - assert updated_data["priority"] == expected_priority - - def test_add_user_api_key_auth_to_request_metadata(): """ Test that add_user_api_key_auth_to_request_metadata properly adds user API key authentication data to request metadata From 6e2fe77d46f231d5ab304a2a4d62d8b39482e3be Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:50:45 -0700 Subject: [PATCH 4/6] test(proxy): use a real ProxyLogging in the priority header routing test --- tests/unit/proxy/test_common_request_processing.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index be6af2ce278..80adf349aa1 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -34,6 +34,7 @@ from litellm.constants import ( ) from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.opentelemetry import UserAPIKeyAuth +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.common_request_processing import ( ProxyBaseLLMRequestProcessing, ProxyConfig, @@ -585,14 +586,11 @@ class TestProxyBaseLLMRequestProcessing: **({"priority": body_priority} if body_priority is not None else {}), } ) - proxy_logging_obj: Final = MagicMock(spec=ProxyLogging) - proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"]) - returned_data, _ = await processing_obj.common_processing_pre_call_logic( request=request, general_settings={}, user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), - proxy_logging_obj=proxy_logging_obj, + proxy_logging_obj=ProxyLogging(user_api_key_cache=UserApiKeyCache()), proxy_config=RealProxyConfig(), route_type=route_type, llm_router=router, From 15e63b4436dba8289aa20c97b0c984904bce188f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:38:47 -0700 Subject: [PATCH 5/6] fix(proxy): ignore a malformed priority header under drop_params instead of dropping the body priority --- litellm/proxy/common_request_processing.py | 12 +++- litellm/proxy/litellm_pre_call_utils.py | 5 -- litellm/proxy/proxy_server.py | 7 +- litellm/router.py | 13 +++- litellm/router_utils/request_priority.py | 6 ++ .../proxy/proxy_server/test_routes_queue.py | 16 +++-- .../proxy/test_common_request_processing.py | 72 +++++++++++++++++++ 7 files changed, 113 insertions(+), 18 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 4be324cae6c..c9622108788 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -254,6 +254,7 @@ from litellm.proxy.anthropic_endpoints.streaming_model_restamp import ( ) from litellm.proxy.litellm_pre_call_utils import ( ROUTER_SCHEDULED_ROUTE_TYPES, + SCHEDULER_PRIORITY_HEADER, LiteLLMProxyRequestSetup, add_litellm_data_to_request, get_metadata_variable_name, @@ -2266,7 +2267,6 @@ class ProxyBaseLLMRequestProcessing: user_api_key_dict=user_api_key_dict, version=version, proxy_config=proxy_config, - router_schedules_request=llm_router is not None and route_type in ROUTER_SCHEDULED_ROUTE_TYPES, ) if logging_auth is not None: metadata_variable_name: Final = get_metadata_variable_name(request) @@ -2378,14 +2378,20 @@ class ProxyBaseLLMRequestProcessing: self.data["model"] = alias_target aliased_model: Final = self.data.get("model") + if llm_router is not None and route_type in ROUTER_SCHEDULED_ROUTE_TYPES and isinstance(aliased_model, str): + self.data.update( + llm_router.priority_header_update( + model=aliased_model, kwargs=self.data, header=request.headers.get(SCHEDULER_PRIORITY_HEADER) + ) + ) litellm_call_id: Final = resolve_litellm_call_id(request.headers.get("x-litellm-call-id")) self.data["litellm_call_id"] = litellm_call_id bind_telemetry_call_id(litellm_call_id) - DDSpanTagger.tag_call_id(self.data.get("litellm_call_id")) + DDSpanTagger.tag_call_id(litellm_call_id) DDSpanTagger.tag_request( user_api_key_dict=user_api_key_dict, - requested_model=self.data.get("model"), + requested_model=aliased_model, ) ### AUTO STREAM USAGE TRACKING ### diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 242c7e412d4..113478a3838 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -2366,7 +2366,6 @@ async def add_litellm_data_to_request( proxy_config: ProxyConfig, general_settings: dict[str, Any] | None = None, version: str | None = None, - router_schedules_request: bool = False, ): """ Adds LiteLLM-specific data to the request. @@ -2501,10 +2500,6 @@ async def add_litellm_data_to_request( ) ) - header_priority: Final = _headers.get(SCHEDULER_PRIORITY_HEADER) if router_schedules_request else None - if header_priority is not None: - data["priority"] = header_priority # rebind-ok: data is an out-param - LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( headers=_headers, data=data, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 8a8d726e562..585e5e11271 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -18271,8 +18271,11 @@ async def async_queue_request( raise HTTPException(status_code=500, detail={"error": CommonProxyErrors.no_llm_router.value}) router: Final = llm_router - header_priority: Final = request.headers.get(SCHEDULER_PRIORITY_HEADER) - data.update({} if header_priority is None else {"priority": header_priority}) + data.update( + router.priority_header_update( + model=data["model"], kwargs=data, header=request.headers.get(SCHEDULER_PRIORITY_HEADER) + ) + ) queued_priority, _ = router.split_request_priority(model=data["model"], kwargs=data) if "stream" in data and data["stream"] is True: # use generate_responses to stream responses diff --git a/litellm/router.py b/litellm/router.py index ac0933e158b..8cabfd02c3c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -344,6 +344,7 @@ from litellm.router_utils.request_priority import ( InvalidPriority, parse_default_priority, priority_error, + priority_header_fields, request_drops_params, resolve_request_priority, unqueueable_priority_message, @@ -5117,14 +5118,20 @@ class Router: request_priority: Final = resolve_request_priority( requested=kwargs.get("priority"), default_priority=self.default_priority, - drops_params=lambda: request_drops_params( - kwargs, self.default_litellm_params, self._request_deployment_params(model=model, kwargs=kwargs) - ), + drops_params=lambda: self._drops_params(model=model, kwargs=kwargs), ) if isinstance(request_priority, InvalidPriority): raise priority_error(model=model, message=request_priority.message) return request_priority, {key: value for key, value in kwargs.items() if key != "priority"} + def priority_header_update(self, model: str, kwargs: Mapping[str, object], header: str | None) -> Mapping[str, str]: + return priority_header_fields(header, drops_params=lambda: self._drops_params(model=model, kwargs=kwargs)) + + def _drops_params(self, model: str, kwargs: Mapping[str, object]) -> bool: + return request_drops_params( + kwargs, self.default_litellm_params, self._request_deployment_params(model=model, kwargs=kwargs) + ) + async def _schedule_factory( self, model: str, diff --git a/litellm/router_utils/request_priority.py b/litellm/router_utils/request_priority.py index 14608cc1bcf..1471534342a 100644 --- a/litellm/router_utils/request_priority.py +++ b/litellm/router_utils/request_priority.py @@ -47,6 +47,12 @@ def resolve_request_priority( return priority +def priority_header_fields(header: str | None, drops_params: Callable[[], bool]) -> Mapping[str, str]: + if header is None or (isinstance(_as_priority(header), InvalidPriority) and drops_params()): + return {} + return {"priority": header} + + def parse_default_priority(value: object) -> int | None | InvalidPriority: if value is None: return None diff --git a/tests/unit/proxy/proxy_server/test_routes_queue.py b/tests/unit/proxy/proxy_server/test_routes_queue.py index e2c83aaa252..38b60797aa3 100644 --- a/tests/unit/proxy/proxy_server/test_routes_queue.py +++ b/tests/unit/proxy/proxy_server/test_routes_queue.py @@ -170,23 +170,29 @@ def test_queue_chat_completions_rejects_a_request_it_cannot_queue_as_an_invalid_ @pytest.mark.parametrize( - ("priority_fields", "expected_priority"), - [({"priority": 5}, 0), ({}, 0)], - ids=("header-beats-body", "header-only"), + ("default_priority", "priority_header", "priority_fields", "expected_priority"), + [ + (5, "0", {"priority": 5}, 0), + (5, "0", {}, 0), + (None, "high", {"priority": 1, "drop_params": True}, 1), + ], + ids=("header-beats-body", "header-only", "drop-params-keeps-body"), ) def test_queue_chat_completions_queues_at_the_priority_header( client, auth_as, monkeypatch: pytest.MonkeyPatch, + default_priority: int | None, + priority_header: str, priority_fields: dict[str, object], expected_priority: int, ) -> None: - router: Final = _queue_router(monkeypatch, default_priority=5) + router: Final = _queue_router(monkeypatch, default_priority=default_priority) payload: Final = {"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}], **priority_fields} with patch.object(router.scheduler, "add_request", wraps=router.scheduler.add_request) as add_request: with auth_as(): response: Final = client.post( - "/queue/chat/completions", json=payload, headers={"x-litellm-priority": "0"} + "/queue/chat/completions", json=payload, headers={"x-litellm-priority": priority_header} ) assert response.status_code == 200, response.text assert [call.kwargs["request"].priority for call in add_request.await_args_list] == [expected_priority] diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 80adf349aa1..cfb75f0ddc1 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -601,6 +601,78 @@ class TestProxyBaseLLMRequestProcessing: expected_priority, ) + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("requested_model", "deployment_drops_params", "body_priority", "expected_priority"), + [ + ("qa-model", True, 5, 5), + ("qa-model", True, None, None), + ("key-alias", True, 5, 5), + ("qa-model", False, 5, "high"), + ], + ids=( + "drop-params-keeps-body", + "drop-params-without-body-leaves-default", + "drop-params-read-from-the-aliased-deployment", + "no-drop-params-passes-header-on-to-400", + ), + ) + async def test_common_processing_pre_call_logic_drops_only_a_malformed_priority_header_under_drop_params( + self, + requested_model: str, + deployment_drops_params: bool, + body_priority: int | None, + expected_priority: str | int | None, + ) -> None: + from litellm.proxy.proxy_server import ProxyConfig as RealProxyConfig + + request: Final = Request( + { + "type": "http", + "method": "POST", + "path": "/v1/chat/completions", + "scheme": "http", + "server": ("localhost", 4000), + "client": ("127.0.0.1", 12345), + "query_string": b"", + "headers": ((b"content-type", b"application/json"), (b"x-litellm-priority", b"high")), + } + ) + router: Final = Router( + model_list=[ + { + "model_name": "qa-model", + "litellm_params": { + "model": "openai/gpt-4o", + "api_key": "sk-test", + "mock_response": "hi", + "drop_params": deployment_drops_params, + }, + } + ] + ) + processing_obj: Final = ProxyBaseLLMRequestProcessing( + data={ + "model": requested_model, + "messages": [{"role": "user", "content": "hi"}], + **({"priority": body_priority} if body_priority is not None else {}), + } + ) + returned_data, _ = await processing_obj.common_processing_pre_call_logic( + request=request, + general_settings={}, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test", aliases={"key-alias": "qa-model"}), + proxy_logging_obj=ProxyLogging(user_api_key_cache=UserApiKeyCache()), + proxy_config=RealProxyConfig(), + route_type="acompletion", + llm_router=router, + ) + + assert ("priority" in returned_data, returned_data.get("priority")) == ( + expected_priority is not None, + expected_priority, + ) + @pytest.mark.asyncio @pytest.mark.parametrize("safe_memory_mode", [False, True]) @pytest.mark.parametrize( From c095bb60a0e95488eb86d150eed2ff3da4bf06b5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:38:46 -0700 Subject: [PATCH 6/6] fix(proxy): treat an empty priority header as unset and pass a valid one on as an integer --- litellm/router.py | 4 +++- litellm/router_utils/request_priority.py | 9 ++++++--- tests/code_coverage_tests/router_code_coverage.py | 2 ++ tests/unit/proxy/proxy_server/test_routes_queue.py | 3 ++- tests/unit/proxy/test_common_request_processing.py | 10 ++++++---- 5 files changed, 19 insertions(+), 9 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 8cabfd02c3c..2b4d9f215f3 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -5124,7 +5124,9 @@ class Router: raise priority_error(model=model, message=request_priority.message) return request_priority, {key: value for key, value in kwargs.items() if key != "priority"} - def priority_header_update(self, model: str, kwargs: Mapping[str, object], header: str | None) -> Mapping[str, str]: + def priority_header_update( + self, model: str, kwargs: Mapping[str, object], header: str | None + ) -> Mapping[str, int | str]: return priority_header_fields(header, drops_params=lambda: self._drops_params(model=model, kwargs=kwargs)) def _drops_params(self, model: str, kwargs: Mapping[str, object]) -> bool: diff --git a/litellm/router_utils/request_priority.py b/litellm/router_utils/request_priority.py index 1471534342a..61627d4b0cf 100644 --- a/litellm/router_utils/request_priority.py +++ b/litellm/router_utils/request_priority.py @@ -47,10 +47,13 @@ def resolve_request_priority( return priority -def priority_header_fields(header: str | None, drops_params: Callable[[], bool]) -> Mapping[str, str]: - if header is None or (isinstance(_as_priority(header), InvalidPriority) and drops_params()): +def priority_header_fields(header: str | None, drops_params: Callable[[], bool]) -> Mapping[str, int | str]: + if not header: return {} - return {"priority": header} + priority: Final = _as_priority(header) + if isinstance(priority, InvalidPriority): + return {} if drops_params() else {"priority": header} + return {"priority": priority} def parse_default_priority(value: object) -> int | None | InvalidPriority: diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 7809a23e955..dbec8028400 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -155,6 +155,8 @@ ignored_function_names = [ "_deployment_field", # Read by is_batch_capable, tested through the mixed-group tests in test_router.py "_deployment_provider", # Read by without_deployments_that_cannot_batch, tested through the mixed-group tests in test_router.py "_cannot_run_batches", # Tested through the Azure and OpenAI mixed-group acreate_batch tests in test_router.py + "priority_header_update", # Tested through the priority header tests in test_common_request_processing.py + "_drops_params", # Read by split_request_priority and priority_header_update, tested through both ] diff --git a/tests/unit/proxy/proxy_server/test_routes_queue.py b/tests/unit/proxy/proxy_server/test_routes_queue.py index 38b60797aa3..f8791525ba1 100644 --- a/tests/unit/proxy/proxy_server/test_routes_queue.py +++ b/tests/unit/proxy/proxy_server/test_routes_queue.py @@ -175,8 +175,9 @@ def test_queue_chat_completions_rejects_a_request_it_cannot_queue_as_an_invalid_ (5, "0", {"priority": 5}, 0), (5, "0", {}, 0), (None, "high", {"priority": 1, "drop_params": True}, 1), + (None, "", {"priority": 1}, 1), ], - ids=("header-beats-body", "header-only", "drop-params-keeps-body"), + ids=("header-beats-body", "header-only", "drop-params-keeps-body", "empty-header-keeps-body"), ) def test_queue_chat_completions_queues_at_the_priority_header( client, diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index cfb75f0ddc1..110ee9d32fd 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -517,11 +517,12 @@ class TestProxyBaseLLMRequestProcessing: @pytest.mark.parametrize( ("path", "route_type", "has_router", "priority_header", "body_priority", "expected_priority"), [ - ("/v1/chat/completions", "acompletion", True, "0", 5, "0"), - ("/v1/completions", "atext_completion", True, "0", 5, "0"), - ("/v1/chat/completions", "acompletion", True, "1", None, "1"), - ("/v1/completions", "atext_completion", True, "1", None, "1"), + ("/v1/chat/completions", "acompletion", True, "0", 5, 0), + ("/v1/completions", "atext_completion", True, "0", 5, 0), + ("/v1/chat/completions", "acompletion", True, "1", None, 1), + ("/v1/completions", "atext_completion", True, "1", None, 1), ("/v1/chat/completions", "acompletion", True, None, 5, 5), + ("/v1/chat/completions", "acompletion", True, "", 5, 5), ("/cursor/chat/completions", "aresponses", True, "0", None, None), ("/v1/responses", "aresponses", True, "0", 5, 5), ("/v1/embeddings", "aembedding", True, "0", None, None), @@ -534,6 +535,7 @@ class TestProxyBaseLLMRequestProcessing: "chat-header-only", "text-completion-header-only", "chat-body-only", + "chat-empty-header-keeps-body", "cursor-responses-route-ignores-header", "responses-keeps-body", "embeddings-ignores-header",