mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-21 10:15:47 +00:00
fix(proxy): apply x-litellm-priority only where the router schedules the request
The header now maps onto priority only for chat and text completions that a router serves, so the Cursor route's responses call and proxies started without a config never forward it to the provider. /queue/chat/completions now honors the header too, with the header winning over a body priority.
This commit is contained in:
parent
e272786a87
commit
1da2efd71f
6 changed files with 148 additions and 63 deletions
|
|
@ -253,6 +253,7 @@ from litellm.proxy.anthropic_endpoints.streaming_model_restamp import (
|
|||
AnthropicStreamModelRestamper,
|
||||
)
|
||||
from litellm.proxy.litellm_pre_call_utils import (
|
||||
ROUTER_SCHEDULED_ROUTE_TYPES,
|
||||
LiteLLMProxyRequestSetup,
|
||||
add_litellm_data_to_request,
|
||||
get_metadata_variable_name,
|
||||
|
|
@ -2265,6 +2266,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
user_api_key_dict=user_api_key_dict,
|
||||
version=version,
|
||||
proxy_config=proxy_config,
|
||||
router_schedules_request=llm_router is not None and route_type in ROUTER_SCHEDULED_ROUTE_TYPES,
|
||||
)
|
||||
if logging_auth is not None:
|
||||
metadata_variable_name: Final = get_metadata_variable_name(request)
|
||||
|
|
|
|||
|
|
@ -114,15 +114,14 @@ _SESSION_ID_VALUE_RE: Final = re.compile(r"^[a-zA-Z0-9_\-]{8,}$")
|
|||
|
||||
_SHA256_HEX_RE: Final = re.compile(r"^[0-9a-f]{64}$")
|
||||
|
||||
SCHEDULER_PRIORITY_HEADER: Final = "x-litellm-priority"
|
||||
ROUTER_SCHEDULED_ROUTE_TYPES: Final = frozenset({"acompletion", "atext_completion"})
|
||||
|
||||
# W3C Trace Context traceparent header: https://www.w3.org/TR/trace-context/
|
||||
# e.g. "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"
|
||||
_TRACEPARENT_RE: Final = re.compile(r"^[0-9a-f]{2}-([0-9a-f]{32})-[0-9a-f]{16}-[0-9a-f]{2}$", re.IGNORECASE)
|
||||
|
||||
|
||||
def _is_scheduler_queued_route(path: str) -> bool:
|
||||
return path.rstrip("/").endswith("/completions")
|
||||
|
||||
|
||||
def _trace_id_from_traceparent(traceparent: str) -> str | None:
|
||||
"""Extract the trace-id from a W3C Trace Context traceparent header, e.g.
|
||||
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01" -> the 32-hex
|
||||
|
|
@ -2367,6 +2366,7 @@ async def add_litellm_data_to_request(
|
|||
proxy_config: ProxyConfig,
|
||||
general_settings: dict[str, Any] | None = None,
|
||||
version: str | None = None,
|
||||
router_schedules_request: bool = False,
|
||||
):
|
||||
"""
|
||||
Adds LiteLLM-specific data to the request.
|
||||
|
|
@ -2501,10 +2501,9 @@ async def add_litellm_data_to_request(
|
|||
)
|
||||
)
|
||||
|
||||
if _is_scheduler_queued_route(request.url.path):
|
||||
request_priority: Final = _headers.get("x-litellm-priority")
|
||||
if request_priority is not None:
|
||||
data["priority"] = request_priority # rebind-ok: data is an out-param
|
||||
header_priority: Final = _headers.get(SCHEDULER_PRIORITY_HEADER) if router_schedules_request else None
|
||||
if header_priority is not None:
|
||||
data["priority"] = header_priority # rebind-ok: data is an out-param
|
||||
|
||||
LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers(
|
||||
headers=_headers,
|
||||
|
|
|
|||
|
|
@ -657,7 +657,11 @@ from litellm.proxy.list_api.common import (
|
|||
problem_response,
|
||||
request_validation_problem,
|
||||
)
|
||||
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, add_litellm_data_to_request
|
||||
from litellm.proxy.litellm_pre_call_utils import (
|
||||
SCHEDULER_PRIORITY_HEADER,
|
||||
LiteLLMProxyRequestSetup,
|
||||
add_litellm_data_to_request,
|
||||
)
|
||||
from litellm.proxy.logging_endpoints.callback_logs_endpoints import (
|
||||
rust_control_plane_router,
|
||||
)
|
||||
|
|
@ -18267,6 +18271,8 @@ async def async_queue_request(
|
|||
raise HTTPException(status_code=500, detail={"error": CommonProxyErrors.no_llm_router.value})
|
||||
router: Final = llm_router
|
||||
|
||||
header_priority: Final = request.headers.get(SCHEDULER_PRIORITY_HEADER)
|
||||
data.update({} if header_priority is None else {"priority": header_priority})
|
||||
queued_priority, _ = router.split_request_priority(model=data["model"], kwargs=data)
|
||||
if "stream" in data and data["stream"] is True: # use generate_responses to stream responses
|
||||
|
||||
|
|
|
|||
|
|
@ -169,6 +169,47 @@ def test_queue_chat_completions_rejects_a_request_it_cannot_queue_as_an_invalid_
|
|||
assert add_request.await_count == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("priority_fields", "expected_priority"),
|
||||
[({"priority": 5}, 0), ({}, 0)],
|
||||
ids=("header-beats-body", "header-only"),
|
||||
)
|
||||
def test_queue_chat_completions_queues_at_the_priority_header(
|
||||
client,
|
||||
auth_as,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
priority_fields: dict[str, object],
|
||||
expected_priority: int,
|
||||
) -> None:
|
||||
router: Final = _queue_router(monkeypatch, default_priority=5)
|
||||
payload: Final = {"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}], **priority_fields}
|
||||
with patch.object(router.scheduler, "add_request", wraps=router.scheduler.add_request) as add_request:
|
||||
with auth_as():
|
||||
response: Final = client.post(
|
||||
"/queue/chat/completions", json=payload, headers={"x-litellm-priority": "0"}
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert [call.kwargs["request"].priority for call in add_request.await_args_list] == [expected_priority]
|
||||
assert response.headers.get("x-litellm-priority") == str(expected_priority)
|
||||
|
||||
|
||||
def test_queue_chat_completions_rejects_a_malformed_priority_header_like_a_malformed_body_value(
|
||||
client, auth_as, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
router: Final = _queue_router(monkeypatch, default_priority=5)
|
||||
payload: Final = {"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}], "priority": 1}
|
||||
with patch.object(router.scheduler, "add_request", wraps=router.scheduler.add_request) as add_request:
|
||||
with auth_as():
|
||||
response: Final = client.post(
|
||||
"/queue/chat/completions", json=payload, headers={"x-litellm-priority": "high"}
|
||||
)
|
||||
assert response.status_code == 400, response.text
|
||||
error: Final = response.json()["error"]
|
||||
assert (error["type"], error["param"]) == ("invalid_request_error", "priority")
|
||||
assert "priority must be an integer" in error["message"]
|
||||
assert add_request.await_count == 0
|
||||
|
||||
|
||||
def test_streamed_queue_responses_report_the_priority_the_router_queued_at(
|
||||
client, auth_as, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -512,6 +512,97 @@ class TestProxyBaseLLMRequestProcessing:
|
|||
assert exc_info.value.param == "model"
|
||||
add_litellm_data_to_request.assert_not_awaited()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("path", "route_type", "has_router", "priority_header", "body_priority", "expected_priority"),
|
||||
[
|
||||
("/v1/chat/completions", "acompletion", True, "0", 5, "0"),
|
||||
("/v1/completions", "atext_completion", True, "0", 5, "0"),
|
||||
("/v1/chat/completions", "acompletion", True, "1", None, "1"),
|
||||
("/v1/completions", "atext_completion", True, "1", None, "1"),
|
||||
("/v1/chat/completions", "acompletion", True, None, 5, 5),
|
||||
("/cursor/chat/completions", "aresponses", True, "0", None, None),
|
||||
("/v1/responses", "aresponses", True, "0", 5, 5),
|
||||
("/v1/embeddings", "aembedding", True, "0", None, None),
|
||||
("/v1/chat/completions", "acompletion", False, "0", None, None),
|
||||
("/v1/chat/completions", "acompletion", False, "0", 5, 5),
|
||||
],
|
||||
ids=(
|
||||
"chat-header-beats-body",
|
||||
"text-completion-header-beats-body",
|
||||
"chat-header-only",
|
||||
"text-completion-header-only",
|
||||
"chat-body-only",
|
||||
"cursor-responses-route-ignores-header",
|
||||
"responses-keeps-body",
|
||||
"embeddings-ignores-header",
|
||||
"routerless-ignores-header",
|
||||
"routerless-keeps-body",
|
||||
),
|
||||
)
|
||||
async def test_common_processing_pre_call_logic_applies_priority_header_only_where_the_router_schedules(
|
||||
self,
|
||||
path: str,
|
||||
route_type: Literal["acompletion", "atext_completion", "aresponses", "aembedding"],
|
||||
has_router: bool,
|
||||
priority_header: str | None,
|
||||
body_priority: int | None,
|
||||
expected_priority: str | int | None,
|
||||
) -> None:
|
||||
from litellm.proxy.proxy_server import ProxyConfig as RealProxyConfig
|
||||
|
||||
header_entries: Final[tuple[tuple[bytes, bytes], ...]] = (
|
||||
() if priority_header is None else ((b"x-litellm-priority", priority_header.encode()),)
|
||||
)
|
||||
request: Final = Request(
|
||||
{
|
||||
"type": "http",
|
||||
"method": "POST",
|
||||
"path": path,
|
||||
"scheme": "http",
|
||||
"server": ("localhost", 4000),
|
||||
"client": ("127.0.0.1", 12345),
|
||||
"query_string": b"",
|
||||
"headers": ((b"content-type", b"application/json"), *header_entries),
|
||||
}
|
||||
)
|
||||
router: Final = (
|
||||
Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "qa-model",
|
||||
"litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-test", "mock_response": "hi"},
|
||||
}
|
||||
]
|
||||
)
|
||||
if has_router
|
||||
else None
|
||||
)
|
||||
processing_obj: Final = ProxyBaseLLMRequestProcessing(
|
||||
data={
|
||||
"model": "qa-model",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
**({"priority": body_priority} if body_priority is not None else {}),
|
||||
}
|
||||
)
|
||||
proxy_logging_obj: Final = MagicMock(spec=ProxyLogging)
|
||||
proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"])
|
||||
|
||||
returned_data, _ = await processing_obj.common_processing_pre_call_logic(
|
||||
request=request,
|
||||
general_settings={},
|
||||
user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"),
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
proxy_config=RealProxyConfig(),
|
||||
route_type=route_type,
|
||||
llm_router=router,
|
||||
)
|
||||
|
||||
assert ("priority" in returned_data, returned_data.get("priority")) == (
|
||||
expected_priority is not None,
|
||||
expected_priority,
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("safe_memory_mode", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
|
|
|
|||
|
|
@ -2684,60 +2684,6 @@ def test_add_litellm_data_for_backend_llm_call_merges_keepalive_seconds_header()
|
|||
assert result.get("keepalive_seconds") == 20.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("path", "priority_header", "body_priority", "expected_priority", "expected_priority_present"),
|
||||
[
|
||||
("/v1/chat/completions", "0", 5, "0", True),
|
||||
("/v1/completions", "0", 5, "0", True),
|
||||
("/v1/embeddings", "0", None, None, False),
|
||||
("/v1/responses", "0", None, None, False),
|
||||
("/v1/embeddings", "0", 5, 5, True),
|
||||
("/v1/responses", "0", 5, 5, True),
|
||||
("/v1/chat/completions", None, 5, 5, True),
|
||||
],
|
||||
)
|
||||
async def test_add_litellm_data_to_request_applies_priority_header_only_to_completion_routes(
|
||||
path: str,
|
||||
priority_header: str | None,
|
||||
body_priority: int | None,
|
||||
expected_priority: int | str | None,
|
||||
expected_priority_present: bool,
|
||||
) -> None:
|
||||
from litellm.proxy.proxy_server import ProxyConfig
|
||||
|
||||
header_entries: Final[tuple[tuple[bytes, bytes], ...]] = (
|
||||
() if priority_header is None else ((b"x-litellm-priority", priority_header.encode()),)
|
||||
)
|
||||
request_headers: Final = ((b"content-type", b"application/json"), *header_entries)
|
||||
request: Final = Request(
|
||||
{
|
||||
"type": "http",
|
||||
"method": "POST",
|
||||
"path": path,
|
||||
"scheme": "http",
|
||||
"server": ("localhost", 4000),
|
||||
"client": ("127.0.0.1", 12345),
|
||||
"query_string": b"",
|
||||
"headers": request_headers,
|
||||
}
|
||||
)
|
||||
request_data: Final[dict[str, object]] = {
|
||||
"model": "qa-model",
|
||||
**({"priority": body_priority} if body_priority is not None else {}),
|
||||
}
|
||||
updated_data: Final = await add_litellm_data_to_request(
|
||||
data=request_data,
|
||||
request=request,
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
|
||||
proxy_config=ProxyConfig(),
|
||||
general_settings={},
|
||||
)
|
||||
assert ("priority" in updated_data) is expected_priority_present
|
||||
if expected_priority_present:
|
||||
assert updated_data["priority"] == expected_priority
|
||||
|
||||
|
||||
def test_add_user_api_key_auth_to_request_metadata():
|
||||
"""
|
||||
Test that add_user_api_key_auth_to_request_metadata properly adds user API key authentication data to request metadata
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue