From e272786a875225ab5cc3afbc40544ce0ce8c3e1b Mon Sep 17 00:00:00 2001 From: Mubashir Osmani Date: Tue, 20 Oct 2026 20:01:12 +0000 Subject: [PATCH] fix(proxy): only apply x-litellm-priority on scheduler-queued routes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_types.py | 1 - litellm/proxy/litellm_pre_call_utils.py | 16 ++-- .../unit/proxy/test_litellm_pre_call_utils.py | 77 ++++++++++++------- 3 files changed, 59 insertions(+), 35 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index a651d735399..315533d6d0f 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -4468,7 +4468,6 @@ class JWTKeyMappingResponse(LiteLLMPydanticObjectBase): class LitellmDataForBackendLLMCall(TypedDict, total=False): headers: dict organization: str - priority: str # writable-ok: request setup adds caller priority before routing timeout: float | None stream_timeout: float | None user: str | None diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 9fe4aed6d59..0ef91c580f6 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -119,6 +119,10 @@ _SHA256_HEX_RE: Final = re.compile(r"^[0-9a-f]{64}$") _TRACEPARENT_RE: Final = re.compile(r"^[0-9a-f]{2}-([0-9a-f]{32})-[0-9a-f]{16}-[0-9a-f]{2}$", re.IGNORECASE) +def _is_scheduler_queued_route(path: str) -> bool: + return path.rstrip("/").endswith("/completions") + + def _trace_id_from_traceparent(traceparent: str) -> str | None: """Extract the trace-id from a W3C Trace Context traceparent header, e.g. "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01" -> the 32-hex @@ -1724,13 +1728,6 @@ class LiteLLMProxyRequestSetup: if num_retries is not None: data["num_retries"] = num_retries - setup: Final = LiteLLMProxyRequestSetup - priority: Final = setup._get_case_insensitive_header( # pyright: ignore[reportUnknownMemberType] # legacy dict - headers, "x-litellm-priority" - ) - if priority is not None: - data["priority"] = priority - keepalive_seconds: Final = LiteLLMProxyRequestSetup._get_keepalive_seconds_from_request(headers) if keepalive_seconds is not None: data["keepalive_seconds"] = keepalive_seconds @@ -2504,6 +2501,11 @@ async def add_litellm_data_to_request( ) ) + if _is_scheduler_queued_route(request.url.path): + request_priority: Final = _headers.get("x-litellm-priority") + if request_priority is not None: + data["priority"] = request_priority # rebind-ok: data is an out-param + LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( headers=_headers, data=data, diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 5edb453b7b1..18de1cd9ecc 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -2684,35 +2684,58 @@ def test_add_litellm_data_for_backend_llm_call_merges_keepalive_seconds_header() assert result.get("keepalive_seconds") == 20.0 -def test_add_litellm_data_for_backend_llm_call_priority_header_precedence() -> None: - header_only: Final = LiteLLMProxyRequestSetup.add_litellm_data_for_backend_llm_call( - headers={"x-litellm-priority": "0"}, - request_data={}, - user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("path", "priority_header", "body_priority", "expected_priority", "expected_priority_present"), + [ + ("/v1/chat/completions", "0", 5, "0", True), + ("/v1/completions", "0", 5, "0", True), + ("/v1/embeddings", "0", None, None, False), + ("/v1/responses", "0", None, None, False), + ("/v1/embeddings", "0", 5, 5, True), + ("/v1/responses", "0", 5, 5, True), + ("/v1/chat/completions", None, 5, 5, True), + ], +) +async def test_add_litellm_data_to_request_applies_priority_header_only_to_completion_routes( + path: str, + priority_header: str | None, + body_priority: int | None, + expected_priority: int | str | None, + expected_priority_present: bool, +) -> None: + from litellm.proxy.proxy_server import ProxyConfig + + header_entries: Final[tuple[tuple[bytes, bytes], ...]] = ( + () if priority_header is None else ((b"x-litellm-priority", priority_header.encode()),) ) - assert header_only.get("priority") == "0" - - body_priority: Final = {"priority": 5} - header_overrides_body: Final = { - **body_priority, - **LiteLLMProxyRequestSetup.add_litellm_data_for_backend_llm_call( - headers={"x-litellm-priority": "0"}, - request_data=body_priority, - user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), - ), + request_headers: Final = ((b"content-type", b"application/json"), *header_entries) + request: Final = Request( + { + "type": "http", + "method": "POST", + "path": path, + "scheme": "http", + "server": ("localhost", 4000), + "client": ("127.0.0.1", 12345), + "query_string": b"", + "headers": request_headers, + } + ) + request_data: Final[dict[str, object]] = { + "model": "qa-model", + **({"priority": body_priority} if body_priority is not None else {}), } - assert header_overrides_body["priority"] == "0" - - body_without_header: Final = {"priority": 5} - body_priority_untouched: Final = { - **body_without_header, - **LiteLLMProxyRequestSetup.add_litellm_data_for_backend_llm_call( - headers={}, - request_data=body_without_header, - user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), - ), - } - assert body_priority_untouched["priority"] == 5 + updated_data: Final = await add_litellm_data_to_request( + data=request_data, + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + proxy_config=ProxyConfig(), + general_settings={}, + ) + assert ("priority" in updated_data) is expected_priority_present + if expected_priority_present: + assert updated_data["priority"] == expected_priority def test_add_user_api_key_auth_to_request_metadata():