mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(proxy): prevent transform request from blocking event loop
This commit is contained in:
parent
252c71c0b2
commit
0a8cadec98
2 changed files with 40 additions and 1 deletions
|
|
@ -13126,7 +13126,11 @@ async def transform_request(request: TransformRequestBody):
|
|||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail={"error": str(e)})
|
||||
|
||||
return return_raw_request(endpoint=request.call_type, kwargs=request.request_body)
|
||||
return await asyncio.to_thread(
|
||||
return_raw_request,
|
||||
endpoint=request.call_type,
|
||||
kwargs=request.request_body,
|
||||
)
|
||||
|
||||
|
||||
async def _check_if_model_is_user_added(
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from __future__ import annotations
|
|||
|
||||
import asyncio
|
||||
import json
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -344,3 +345,37 @@ def test_token_counter_fallback_prompt_with_tools_does_not_500(client, auth_as,
|
|||
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["total_tokens"] == litellm.token_counter(model="claude-fable-5", text=prompt)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transform_request_does_not_block_event_loop(monkeypatch):
|
||||
monkeypatch.setattr(proxy_server, "llm_router", None)
|
||||
monkeypatch.setattr(proxy_server, "is_request_body_safe", lambda **kwargs: True)
|
||||
|
||||
event_loop_progressed = threading.Event()
|
||||
|
||||
def blocking_return_raw_request(endpoint, kwargs):
|
||||
if not event_loop_progressed.wait(timeout=1):
|
||||
raise AssertionError("event loop was blocked")
|
||||
return {
|
||||
"raw_request_api_base": "https://example.com/v1/chat/completions",
|
||||
"raw_request_body": kwargs,
|
||||
"raw_request_headers": {},
|
||||
}
|
||||
|
||||
monkeypatch.setattr("litellm.utils.return_raw_request", blocking_return_raw_request)
|
||||
|
||||
async def heartbeat():
|
||||
event_loop_progressed.set()
|
||||
|
||||
heartbeat_task = asyncio.create_task(heartbeat())
|
||||
result = await proxy_server.transform_request(
|
||||
proxy_server.TransformRequestBody(
|
||||
call_type="completion",
|
||||
request_body={"model": "openai/test-model"},
|
||||
)
|
||||
)
|
||||
await heartbeat_task
|
||||
|
||||
assert event_loop_progressed.is_set()
|
||||
assert result["raw_request_body"] == {"model": "openai/test-model"}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue