mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(proxy): treat an empty priority header as unset and pass a valid one on as an integer
This commit is contained in:
parent
15e63b4436
commit
c095bb60a0
5 changed files with 19 additions and 9 deletions
|
|
@ -5124,7 +5124,9 @@ class Router:
|
|||
raise priority_error(model=model, message=request_priority.message)
|
||||
return request_priority, {key: value for key, value in kwargs.items() if key != "priority"}
|
||||
|
||||
def priority_header_update(self, model: str, kwargs: Mapping[str, object], header: str | None) -> Mapping[str, str]:
|
||||
def priority_header_update(
|
||||
self, model: str, kwargs: Mapping[str, object], header: str | None
|
||||
) -> Mapping[str, int | str]:
|
||||
return priority_header_fields(header, drops_params=lambda: self._drops_params(model=model, kwargs=kwargs))
|
||||
|
||||
def _drops_params(self, model: str, kwargs: Mapping[str, object]) -> bool:
|
||||
|
|
|
|||
|
|
@ -47,10 +47,13 @@ def resolve_request_priority(
|
|||
return priority
|
||||
|
||||
|
||||
def priority_header_fields(header: str | None, drops_params: Callable[[], bool]) -> Mapping[str, str]:
|
||||
if header is None or (isinstance(_as_priority(header), InvalidPriority) and drops_params()):
|
||||
def priority_header_fields(header: str | None, drops_params: Callable[[], bool]) -> Mapping[str, int | str]:
|
||||
if not header:
|
||||
return {}
|
||||
return {"priority": header}
|
||||
priority: Final = _as_priority(header)
|
||||
if isinstance(priority, InvalidPriority):
|
||||
return {} if drops_params() else {"priority": header}
|
||||
return {"priority": priority}
|
||||
|
||||
|
||||
def parse_default_priority(value: object) -> int | None | InvalidPriority:
|
||||
|
|
|
|||
|
|
@ -155,6 +155,8 @@ ignored_function_names = [
|
|||
"_deployment_field", # Read by is_batch_capable, tested through the mixed-group tests in test_router.py
|
||||
"_deployment_provider", # Read by without_deployments_that_cannot_batch, tested through the mixed-group tests in test_router.py
|
||||
"_cannot_run_batches", # Tested through the Azure and OpenAI mixed-group acreate_batch tests in test_router.py
|
||||
"priority_header_update", # Tested through the priority header tests in test_common_request_processing.py
|
||||
"_drops_params", # Read by split_request_priority and priority_header_update, tested through both
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -175,8 +175,9 @@ def test_queue_chat_completions_rejects_a_request_it_cannot_queue_as_an_invalid_
|
|||
(5, "0", {"priority": 5}, 0),
|
||||
(5, "0", {}, 0),
|
||||
(None, "high", {"priority": 1, "drop_params": True}, 1),
|
||||
(None, "", {"priority": 1}, 1),
|
||||
],
|
||||
ids=("header-beats-body", "header-only", "drop-params-keeps-body"),
|
||||
ids=("header-beats-body", "header-only", "drop-params-keeps-body", "empty-header-keeps-body"),
|
||||
)
|
||||
def test_queue_chat_completions_queues_at_the_priority_header(
|
||||
client,
|
||||
|
|
|
|||
|
|
@ -517,11 +517,12 @@ class TestProxyBaseLLMRequestProcessing:
|
|||
@pytest.mark.parametrize(
|
||||
("path", "route_type", "has_router", "priority_header", "body_priority", "expected_priority"),
|
||||
[
|
||||
("/v1/chat/completions", "acompletion", True, "0", 5, "0"),
|
||||
("/v1/completions", "atext_completion", True, "0", 5, "0"),
|
||||
("/v1/chat/completions", "acompletion", True, "1", None, "1"),
|
||||
("/v1/completions", "atext_completion", True, "1", None, "1"),
|
||||
("/v1/chat/completions", "acompletion", True, "0", 5, 0),
|
||||
("/v1/completions", "atext_completion", True, "0", 5, 0),
|
||||
("/v1/chat/completions", "acompletion", True, "1", None, 1),
|
||||
("/v1/completions", "atext_completion", True, "1", None, 1),
|
||||
("/v1/chat/completions", "acompletion", True, None, 5, 5),
|
||||
("/v1/chat/completions", "acompletion", True, "", 5, 5),
|
||||
("/cursor/chat/completions", "aresponses", True, "0", None, None),
|
||||
("/v1/responses", "aresponses", True, "0", 5, 5),
|
||||
("/v1/embeddings", "aembedding", True, "0", None, None),
|
||||
|
|
@ -534,6 +535,7 @@ class TestProxyBaseLLMRequestProcessing:
|
|||
"chat-header-only",
|
||||
"text-completion-header-only",
|
||||
"chat-body-only",
|
||||
"chat-empty-header-keeps-body",
|
||||
"cursor-responses-route-ignores-header",
|
||||
"responses-keeps-body",
|
||||
"embeddings-ignores-header",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue