mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-28 06:23:07 +00:00
d698106ed7
* fix(router): schedule default_priority requests once and reject a non-integer priority A router configured with default_priority failed every request that carried no priority with a 500 naming a missing 'priority' argument, and an explicit priority re-entered acompletion and hit the same error. acompletion now resolves the priority once (the request's integer, else the default, with 0 kept as a real priority) and schedules through _schedule_factory directly, while schedule_acompletion delegates to acompletion so the two paths share one queue pass. A non-integer priority answers 400 naming the param, or is dropped under drop_params, instead of leaking to the provider or crashing the scheduler. * fix(router): honor router-level drop_params and schedule prompt-management models under a priority * fix(router): drop a non-integer priority when the model's deployments set drop_params * fix(router): resolve the drop_params deployment scope for team routes and deployment ids * fix(router): read the request and router drop_params flags without a loop-assigned Final * refactor(router): type the prompt-management wrapper's return and mock the provider in its test * perf(router): consult drop_params for the priority only when it is not an integer * test(integration): assert the new priority contract for non-integer and zero priorities * docs(router): describe default_priority as the priority every request without one takes * fix(router): parse default_priority once and apply it to text completions and the queue route A digit string default is taken and any other non-integer default is ignored with a warning instead of failing every request. atext_completion and /queue/chat/completions resolve the priority the way acompletion does: a request without one takes the default, an integer queues once, and a non-integer answers 400 or takes the default under drop_params. Integration cells cover every default-priority surface on an owned two-instance rig. * fix(router): type the priority dispatch so the lint budgets hold * fix(router): read priority like the scheduler did, require a priority on the queue route, and keep litellm's drop_params order * fix(router): drop a priority named in additional_drop_params and report the queued priority on streamed queue responses * fix(router): serve deployment-id requests without waiting behind the queue and ignore a boolean default_priority A request whose model is a deployment id now counts that deployment as healthy in the scheduler, since routing by id already ignores cooldown. Before, the scheduler found no healthy deployment for an id, so the request waited behind every entry ahead of it. A boolean default_priority (true, or YAML off) is ignored at startup with a warning, like any other value that is not a whole number. The scheduler integration tests now share worker state for the defaulted pair, cool a group through /v1/responses so no stale in-memory queue is left behind, and use "high" instead of "1" for the non-integer cases. The concurrent burst test no longer asserts that the queue drains, matching the explicit-priority burst test, since the scheduler's read-modify-write queue can race across workers. * test(router): keep the queue drain check as a BUG-skipped integration test * fix(router): queue default_priority chat requests once instead of failing them * test(router): open with the default_priority regression tests alone * fix(router): honor deployment drop_params for a malformed priority on a request routed by provider model name
112 lines
5.1 KiB
Python
112 lines
5.1 KiB
Python
from typing import Final
|
|
|
|
import pytest
|
|
|
|
import litellm
|
|
from litellm.router_utils.request_priority import (
|
|
InvalidPriority,
|
|
parse_default_priority,
|
|
request_drops_params,
|
|
resolve_request_priority,
|
|
unqueueable_priority_message,
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("requested", "default_priority", "expected"),
|
|
[
|
|
(None, None, None),
|
|
(None, 3, 3),
|
|
(2, 3, 2),
|
|
(0, 3, 0),
|
|
(0, None, 0),
|
|
(255, None, 255),
|
|
("1", 3, 1),
|
|
(" 2 ", None, 2),
|
|
("-1", 3, -1),
|
|
(1.0, 3, 1),
|
|
(True, None, 1),
|
|
],
|
|
)
|
|
def test_an_absent_or_null_priority_takes_the_default_and_an_integer_like_value_wins(
|
|
requested: object, default_priority: int | None, expected: int | None
|
|
) -> None:
|
|
assert resolve_request_priority(requested, default_priority, drops_params=lambda: False) == expected
|
|
assert resolve_request_priority(requested, default_priority, drops_params=lambda: True) == expected
|
|
|
|
|
|
@pytest.mark.parametrize("requested", ["high", "", [1], 1.5, {"level": 1}])
|
|
def test_a_non_integer_priority_is_rejected_unless_params_are_dropped(requested: object) -> None:
|
|
rejected: Final = resolve_request_priority(requested, 3, drops_params=lambda: False)
|
|
assert isinstance(rejected, InvalidPriority)
|
|
assert rejected.value == requested
|
|
assert rejected.message.startswith("priority must be an integer, got ")
|
|
assert resolve_request_priority(requested, 3, drops_params=lambda: True) == 3
|
|
assert resolve_request_priority(requested, None, drops_params=lambda: True) is None
|
|
|
|
|
|
def test_the_drop_params_decision_is_consulted_only_for_a_non_integer_priority() -> None:
|
|
def _never() -> bool:
|
|
raise AssertionError("drop_params was consulted")
|
|
|
|
assert resolve_request_priority(None, 3, drops_params=_never) == 3
|
|
assert resolve_request_priority(2, 3, drops_params=_never) == 2
|
|
assert resolve_request_priority(0, None, drops_params=_never) == 0
|
|
assert resolve_request_priority("1", 3, drops_params=_never) == 1
|
|
|
|
|
|
def test_the_global_drop_params_wins_then_the_request_then_the_router_defaults_then_the_deployments(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
monkeypatch.setattr(litellm, "drop_params", False)
|
|
assert request_drops_params({}, {}, ()) is False
|
|
assert request_drops_params({"drop_params": True}, {}, ()) is True
|
|
assert request_drops_params({"drop_params": "true"}, {}, ()) is True
|
|
assert request_drops_params({}, {"drop_params": True}, ()) is True
|
|
assert request_drops_params({"drop_params": False}, {"drop_params": True}, ()) is False
|
|
assert request_drops_params({}, {}, ({"drop_params": True},)) is True
|
|
assert request_drops_params({}, {}, ({"drop_params": True}, {"model": "openai/gpt-5.4-nano"})) is True
|
|
assert request_drops_params({}, {}, ({"drop_params": False}, {"model": "openai/gpt-5.4-nano"})) is False
|
|
assert request_drops_params({}, {"drop_params": False}, ({"drop_params": True},)) is False
|
|
monkeypatch.setattr(litellm, "drop_params", True)
|
|
assert request_drops_params({}, {}, ()) is True
|
|
assert request_drops_params({}, {}, ({"model": "openai/gpt-5.4-nano"},)) is True
|
|
assert request_drops_params({"drop_params": False}, {}, ()) is True
|
|
assert request_drops_params({}, {"drop_params": False}, ()) is True
|
|
assert request_drops_params({}, {}, ({"drop_params": False},)) is True
|
|
assert request_drops_params({"drop_params": "not-a-flag"}, {}, ()) is True
|
|
|
|
|
|
def test_a_priority_named_in_additional_drop_params_at_any_level_is_dropped(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(litellm, "drop_params", False)
|
|
by_name: Final = {"additional_drop_params": ["priority"]}
|
|
assert request_drops_params(by_name, {}, ()) is True
|
|
assert request_drops_params({**by_name, "drop_params": False}, {}, ()) is True
|
|
assert request_drops_params({}, by_name, ()) is True
|
|
assert request_drops_params({}, {"drop_params": False}, (by_name,)) is True
|
|
assert request_drops_params({"additional_drop_params": ["seed"]}, {}, ()) is False
|
|
assert request_drops_params({}, {}, (params for params in ({"drop_params": True},))) is True
|
|
|
|
|
|
def test_a_request_it_cannot_queue_is_told_why() -> None:
|
|
assert "priority is required" in unqueueable_priority_message(None)
|
|
assert unqueueable_priority_message("high").startswith("priority must be an integer, got str 'high'")
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("configured", "expected"),
|
|
[(None, None), (3, 3), (0, 0), ("3", 3), (" 12 ", 12), ("0", 0), ("-1", -1), (2.0, 2)],
|
|
ids=["unset", "int", "zero", "digit-string", "padded-digit-string", "zero-string", "negative-string", "whole-float"],
|
|
)
|
|
def test_the_default_priority_is_taken_as_an_integer_or_an_integer_like_value(
|
|
configured: object, expected: int | None
|
|
) -> None:
|
|
assert parse_default_priority(configured) == expected
|
|
|
|
|
|
@pytest.mark.parametrize("configured", ["high", "", "1.5", 1.5, [3], {"level": 3}, True, False])
|
|
def test_any_other_default_priority_is_invalid(configured: object) -> None:
|
|
parsed: Final = parse_default_priority(configured)
|
|
assert isinstance(parsed, InvalidPriority)
|
|
assert parsed.value == configured
|
|
assert parsed.message.startswith("priority must be an integer, got ")
|