mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-22 10:40:46 +00:00
fix(proxy): only apply x-litellm-priority on scheduler-queued routes
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
4ba2cc27dc
commit
e272786a87
3 changed files with 59 additions and 35 deletions
|
|
@ -4468,7 +4468,6 @@ class JWTKeyMappingResponse(LiteLLMPydanticObjectBase):
|
|||
class LitellmDataForBackendLLMCall(TypedDict, total=False):
|
||||
headers: dict
|
||||
organization: str
|
||||
priority: str # writable-ok: request setup adds caller priority before routing
|
||||
timeout: float | None
|
||||
stream_timeout: float | None
|
||||
user: str | None
|
||||
|
|
|
|||
|
|
@ -119,6 +119,10 @@ _SHA256_HEX_RE: Final = re.compile(r"^[0-9a-f]{64}$")
|
|||
_TRACEPARENT_RE: Final = re.compile(r"^[0-9a-f]{2}-([0-9a-f]{32})-[0-9a-f]{16}-[0-9a-f]{2}$", re.IGNORECASE)
|
||||
|
||||
|
||||
def _is_scheduler_queued_route(path: str) -> bool:
|
||||
return path.rstrip("/").endswith("/completions")
|
||||
|
||||
|
||||
def _trace_id_from_traceparent(traceparent: str) -> str | None:
|
||||
"""Extract the trace-id from a W3C Trace Context traceparent header, e.g.
|
||||
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01" -> the 32-hex
|
||||
|
|
@ -1724,13 +1728,6 @@ class LiteLLMProxyRequestSetup:
|
|||
if num_retries is not None:
|
||||
data["num_retries"] = num_retries
|
||||
|
||||
setup: Final = LiteLLMProxyRequestSetup
|
||||
priority: Final = setup._get_case_insensitive_header( # pyright: ignore[reportUnknownMemberType] # legacy dict
|
||||
headers, "x-litellm-priority"
|
||||
)
|
||||
if priority is not None:
|
||||
data["priority"] = priority
|
||||
|
||||
keepalive_seconds: Final = LiteLLMProxyRequestSetup._get_keepalive_seconds_from_request(headers)
|
||||
if keepalive_seconds is not None:
|
||||
data["keepalive_seconds"] = keepalive_seconds
|
||||
|
|
@ -2504,6 +2501,11 @@ async def add_litellm_data_to_request(
|
|||
)
|
||||
)
|
||||
|
||||
if _is_scheduler_queued_route(request.url.path):
|
||||
request_priority: Final = _headers.get("x-litellm-priority")
|
||||
if request_priority is not None:
|
||||
data["priority"] = request_priority # rebind-ok: data is an out-param
|
||||
|
||||
LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers(
|
||||
headers=_headers,
|
||||
data=data,
|
||||
|
|
|
|||
|
|
@ -2684,35 +2684,58 @@ def test_add_litellm_data_for_backend_llm_call_merges_keepalive_seconds_header()
|
|||
assert result.get("keepalive_seconds") == 20.0
|
||||
|
||||
|
||||
def test_add_litellm_data_for_backend_llm_call_priority_header_precedence() -> None:
|
||||
header_only: Final = LiteLLMProxyRequestSetup.add_litellm_data_for_backend_llm_call(
|
||||
headers={"x-litellm-priority": "0"},
|
||||
request_data={},
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("path", "priority_header", "body_priority", "expected_priority", "expected_priority_present"),
|
||||
[
|
||||
("/v1/chat/completions", "0", 5, "0", True),
|
||||
("/v1/completions", "0", 5, "0", True),
|
||||
("/v1/embeddings", "0", None, None, False),
|
||||
("/v1/responses", "0", None, None, False),
|
||||
("/v1/embeddings", "0", 5, 5, True),
|
||||
("/v1/responses", "0", 5, 5, True),
|
||||
("/v1/chat/completions", None, 5, 5, True),
|
||||
],
|
||||
)
|
||||
async def test_add_litellm_data_to_request_applies_priority_header_only_to_completion_routes(
|
||||
path: str,
|
||||
priority_header: str | None,
|
||||
body_priority: int | None,
|
||||
expected_priority: int | str | None,
|
||||
expected_priority_present: bool,
|
||||
) -> None:
|
||||
from litellm.proxy.proxy_server import ProxyConfig
|
||||
|
||||
header_entries: Final[tuple[tuple[bytes, bytes], ...]] = (
|
||||
() if priority_header is None else ((b"x-litellm-priority", priority_header.encode()),)
|
||||
)
|
||||
assert header_only.get("priority") == "0"
|
||||
|
||||
body_priority: Final = {"priority": 5}
|
||||
header_overrides_body: Final = {
|
||||
**body_priority,
|
||||
**LiteLLMProxyRequestSetup.add_litellm_data_for_backend_llm_call(
|
||||
headers={"x-litellm-priority": "0"},
|
||||
request_data=body_priority,
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
|
||||
),
|
||||
request_headers: Final = ((b"content-type", b"application/json"), *header_entries)
|
||||
request: Final = Request(
|
||||
{
|
||||
"type": "http",
|
||||
"method": "POST",
|
||||
"path": path,
|
||||
"scheme": "http",
|
||||
"server": ("localhost", 4000),
|
||||
"client": ("127.0.0.1", 12345),
|
||||
"query_string": b"",
|
||||
"headers": request_headers,
|
||||
}
|
||||
)
|
||||
request_data: Final[dict[str, object]] = {
|
||||
"model": "qa-model",
|
||||
**({"priority": body_priority} if body_priority is not None else {}),
|
||||
}
|
||||
assert header_overrides_body["priority"] == "0"
|
||||
|
||||
body_without_header: Final = {"priority": 5}
|
||||
body_priority_untouched: Final = {
|
||||
**body_without_header,
|
||||
**LiteLLMProxyRequestSetup.add_litellm_data_for_backend_llm_call(
|
||||
headers={},
|
||||
request_data=body_without_header,
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
|
||||
),
|
||||
}
|
||||
assert body_priority_untouched["priority"] == 5
|
||||
updated_data: Final = await add_litellm_data_to_request(
|
||||
data=request_data,
|
||||
request=request,
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
|
||||
proxy_config=ProxyConfig(),
|
||||
general_settings={},
|
||||
)
|
||||
assert ("priority" in updated_data) is expected_priority_present
|
||||
if expected_priority_present:
|
||||
assert updated_data["priority"] == expected_priority
|
||||
|
||||
|
||||
def test_add_user_api_key_auth_to_request_metadata():
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue