mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
test(router): observe the priority header tests through wait_for_turn
Main added two /queue/chat/completions tests for the x-litellm-priority header that spy on Scheduler.add_request. A request a healthy deployment can take no longer reaches add_request, so they move to wait_for_turn like the other priority tests on this branch.
This commit is contained in:
parent
9a1f0c8076
commit
e2e070dbde
1 changed files with 4 additions and 4 deletions
|
|
@ -212,13 +212,13 @@ def test_queue_chat_completions_queues_at_the_priority_header(
|
|||
) -> None:
|
||||
router: Final = _queue_router(monkeypatch, default_priority=default_priority)
|
||||
payload: Final = {"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}], **priority_fields}
|
||||
with patch.object(router.scheduler, "add_request", wraps=router.scheduler.add_request) as add_request:
|
||||
with patch.object(router.scheduler, "wait_for_turn", wraps=router.scheduler.wait_for_turn) as wait_for_turn:
|
||||
with auth_as():
|
||||
response: Final = client.post(
|
||||
"/queue/chat/completions", json=payload, headers={"x-litellm-priority": priority_header}
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert [call.kwargs["request"].priority for call in add_request.await_args_list] == [expected_priority]
|
||||
assert [call.kwargs["request"].priority for call in wait_for_turn.await_args_list] == [expected_priority]
|
||||
assert response.headers.get("x-litellm-priority") == str(expected_priority)
|
||||
|
||||
|
||||
|
|
@ -227,7 +227,7 @@ def test_queue_chat_completions_rejects_a_malformed_priority_header_like_a_malfo
|
|||
) -> None:
|
||||
router: Final = _queue_router(monkeypatch, default_priority=5)
|
||||
payload: Final = {"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}], "priority": 1}
|
||||
with patch.object(router.scheduler, "add_request", wraps=router.scheduler.add_request) as add_request:
|
||||
with patch.object(router.scheduler, "wait_for_turn", wraps=router.scheduler.wait_for_turn) as wait_for_turn:
|
||||
with auth_as():
|
||||
response: Final = client.post(
|
||||
"/queue/chat/completions", json=payload, headers={"x-litellm-priority": "high"}
|
||||
|
|
@ -236,7 +236,7 @@ def test_queue_chat_completions_rejects_a_malformed_priority_header_like_a_malfo
|
|||
error: Final = response.json()["error"]
|
||||
assert (error["type"], error["param"]) == ("invalid_request_error", "priority")
|
||||
assert "priority must be an integer" in error["message"]
|
||||
assert add_request.await_count == 0
|
||||
assert wait_for_turn.await_count == 0
|
||||
|
||||
|
||||
def test_streamed_queue_responses_report_the_priority_the_router_queued_at(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue