fix(proxy): only apply x-litellm-priority on scheduler-queued routes

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Mubashir Osmani 2026-10-20 20:01:12 +00:00
parent 4ba2cc27dc
commit e272786a87
3 changed files with 59 additions and 35 deletions

View file

@ -4468,7 +4468,6 @@ class JWTKeyMappingResponse(LiteLLMPydanticObjectBase):
class LitellmDataForBackendLLMCall(TypedDict, total=False):
headers: dict
organization: str
priority: str # writable-ok: request setup adds caller priority before routing
timeout: float | None
stream_timeout: float | None
user: str | None

View file

@ -119,6 +119,10 @@ _SHA256_HEX_RE: Final = re.compile(r"^[0-9a-f]{64}$")
_TRACEPARENT_RE: Final = re.compile(r"^[0-9a-f]{2}-([0-9a-f]{32})-[0-9a-f]{16}-[0-9a-f]{2}$", re.IGNORECASE)
def _is_scheduler_queued_route(path: str) -> bool:
return path.rstrip("/").endswith("/completions")
def _trace_id_from_traceparent(traceparent: str) -> str | None:
"""Extract the trace-id from a W3C Trace Context traceparent header, e.g.
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01" -> the 32-hex
@ -1724,13 +1728,6 @@ class LiteLLMProxyRequestSetup:
if num_retries is not None:
data["num_retries"] = num_retries
setup: Final = LiteLLMProxyRequestSetup
priority: Final = setup._get_case_insensitive_header( # pyright: ignore[reportUnknownMemberType] # legacy dict
headers, "x-litellm-priority"
)
if priority is not None:
data["priority"] = priority
keepalive_seconds: Final = LiteLLMProxyRequestSetup._get_keepalive_seconds_from_request(headers)
if keepalive_seconds is not None:
data["keepalive_seconds"] = keepalive_seconds
@ -2504,6 +2501,11 @@ async def add_litellm_data_to_request(
)
)
if _is_scheduler_queued_route(request.url.path):
request_priority: Final = _headers.get("x-litellm-priority")
if request_priority is not None:
data["priority"] = request_priority # rebind-ok: data is an out-param
LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers(
headers=_headers,
data=data,

View file

@ -2684,35 +2684,58 @@ def test_add_litellm_data_for_backend_llm_call_merges_keepalive_seconds_header()
assert result.get("keepalive_seconds") == 20.0
def test_add_litellm_data_for_backend_llm_call_priority_header_precedence() -> None:
header_only: Final = LiteLLMProxyRequestSetup.add_litellm_data_for_backend_llm_call(
headers={"x-litellm-priority": "0"},
request_data={},
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
@pytest.mark.asyncio
@pytest.mark.parametrize(
("path", "priority_header", "body_priority", "expected_priority", "expected_priority_present"),
[
("/v1/chat/completions", "0", 5, "0", True),
("/v1/completions", "0", 5, "0", True),
("/v1/embeddings", "0", None, None, False),
("/v1/responses", "0", None, None, False),
("/v1/embeddings", "0", 5, 5, True),
("/v1/responses", "0", 5, 5, True),
("/v1/chat/completions", None, 5, 5, True),
],
)
async def test_add_litellm_data_to_request_applies_priority_header_only_to_completion_routes(
path: str,
priority_header: str | None,
body_priority: int | None,
expected_priority: int | str | None,
expected_priority_present: bool,
) -> None:
from litellm.proxy.proxy_server import ProxyConfig
header_entries: Final[tuple[tuple[bytes, bytes], ...]] = (
() if priority_header is None else ((b"x-litellm-priority", priority_header.encode()),)
)
assert header_only.get("priority") == "0"
body_priority: Final = {"priority": 5}
header_overrides_body: Final = {
**body_priority,
**LiteLLMProxyRequestSetup.add_litellm_data_for_backend_llm_call(
headers={"x-litellm-priority": "0"},
request_data=body_priority,
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
),
request_headers: Final = ((b"content-type", b"application/json"), *header_entries)
request: Final = Request(
{
"type": "http",
"method": "POST",
"path": path,
"scheme": "http",
"server": ("localhost", 4000),
"client": ("127.0.0.1", 12345),
"query_string": b"",
"headers": request_headers,
}
)
request_data: Final[dict[str, object]] = {
"model": "qa-model",
**({"priority": body_priority} if body_priority is not None else {}),
}
assert header_overrides_body["priority"] == "0"
body_without_header: Final = {"priority": 5}
body_priority_untouched: Final = {
**body_without_header,
**LiteLLMProxyRequestSetup.add_litellm_data_for_backend_llm_call(
headers={},
request_data=body_without_header,
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
),
}
assert body_priority_untouched["priority"] == 5
updated_data: Final = await add_litellm_data_to_request(
data=request_data,
request=request,
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
proxy_config=ProxyConfig(),
general_settings={},
)
assert ("priority" in updated_data) is expected_priority_present
if expected_priority_present:
assert updated_data["priority"] == expected_priority
def test_add_user_api_key_auth_to_request_metadata():