mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(router): free a failed rust router call's request without waiting for a full gc (#51963)
* fix(router): free a failed rust router call's request without waiting for a full gc Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(router): release retained retry errors after successful calls Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
1e3a960a0e
commit
8c97c243fe
3 changed files with 54 additions and 4 deletions
|
|
@ -5545,6 +5545,7 @@ class _Driver:
|
|||
def finish(self, step: Mapping[str, object]) -> object:
|
||||
"""The response with the router's headers, in the order ``make_call`` and the wrappers above it
|
||||
stamp them: a usage read that raises leaves the counts off."""
|
||||
self._clear_errors()
|
||||
if not self._plan.sets_headers:
|
||||
self.log_outcomes_sync()
|
||||
return self._last_response
|
||||
|
|
@ -5565,6 +5566,7 @@ class _Driver:
|
|||
|
||||
async def finish_async(self, step: Mapping[str, object]) -> object:
|
||||
"""``finish``, with the usage read on the call's own loop."""
|
||||
self._clear_errors()
|
||||
if not self._plan.sets_headers:
|
||||
await await_all(self.outcomes.take_pending())
|
||||
return self._last_response
|
||||
|
|
@ -5586,6 +5588,18 @@ class _Driver:
|
|||
return self._request.get("messages") is not None or self._request.get("input") is not None
|
||||
|
||||
def failed(self, step: Mapping[str, object]) -> BaseException:
|
||||
try:
|
||||
return self._failure(step)
|
||||
finally:
|
||||
self._clear_errors()
|
||||
|
||||
def _clear_errors(self) -> None:
|
||||
self._last_error = None
|
||||
self._level_errors.clear()
|
||||
self._hop_first.clear()
|
||||
self._level_picks.clear()
|
||||
|
||||
def _failure(self, step: Mapping[str, object]) -> BaseException:
|
||||
model: Final = str(self._request.get("model"))
|
||||
primary: Final = self._level_errors.get(self._base_level)
|
||||
if primary is None:
|
||||
|
|
|
|||
|
|
@ -450,7 +450,10 @@ class ScriptedHost(PythonHost):
|
|||
error, response = self._finish(op, call, outcome)
|
||||
if error is not None:
|
||||
await self.log_async(call, error, response)
|
||||
raise error
|
||||
try:
|
||||
raise error
|
||||
finally:
|
||||
del error, outcome
|
||||
if not self._logs_at_stream_end(call, response):
|
||||
await self.log_async(call, None, response)
|
||||
return response
|
||||
|
|
@ -460,7 +463,10 @@ class ScriptedHost(PythonHost):
|
|||
error, response = self._finish(op, call, outcome)
|
||||
if error is not None:
|
||||
self.log_sync(call, error, response)
|
||||
raise error
|
||||
try:
|
||||
raise error
|
||||
finally:
|
||||
del error, outcome
|
||||
if not self._logs_at_stream_end(call, response):
|
||||
self.log_sync(call, None, response)
|
||||
return response
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ from tests.unit.test_router.core.scripted_host import (
|
|||
completion_response,
|
||||
deployment,
|
||||
embedding_response,
|
||||
server_error,
|
||||
)
|
||||
|
||||
MESSAGES: Final = [{"role": "user", "content": "hi"}]
|
||||
|
|
@ -462,17 +463,46 @@ async def _request_state_after_a_call(
|
|||
state: Final = _RequestState()
|
||||
_ = await core.call(endpoint, {"model": "g", "messages": MESSAGES, "max_tokens": 5, "metadata": {"state": state}})
|
||||
host.calls.clear()
|
||||
host.raised.clear()
|
||||
return weakref.ref(state)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("endpoint", ["acompletion", "aanthropic_messages", "completion"])
|
||||
@pytest.mark.parametrize("retry_then_succeed", [False, True])
|
||||
async def test_a_finished_call_frees_its_request_without_a_gc_pass(
|
||||
make_core: MakeCore, host: ScriptedHost, endpoint: str
|
||||
make_core: MakeCore, host: ScriptedHost, endpoint: str, retry_then_succeed: bool
|
||||
):
|
||||
core: Final = _one_group(make_core)
|
||||
if retry_then_succeed:
|
||||
host.answer("openai/d1", server_error("d1"), None)
|
||||
core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=1, retry_after=0)
|
||||
gc.disable()
|
||||
try:
|
||||
state: Final = await _request_state_after_a_call(core, host, endpoint)
|
||||
assert state() is None, "the finished call's request stays in a reference cycle until the next gc pass"
|
||||
finally:
|
||||
gc.enable()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("endpoint", ["acompletion", "aanthropic_messages", "completion"])
|
||||
async def test_a_failed_call_frees_its_request_without_a_gc_pass(
|
||||
make_core: MakeCore, host: ScriptedHost, endpoint: str
|
||||
):
|
||||
host.answer("openai/d1", bad_request(message="scripted bad input"))
|
||||
core: Final = _one_group(make_core)
|
||||
gc.disable()
|
||||
try:
|
||||
state: Final = await _request_state_after_a_failed_call(core, host, endpoint)
|
||||
assert state() is None, "the failed call's request stays in a reference cycle until the next gc pass"
|
||||
finally:
|
||||
gc.enable()
|
||||
|
||||
|
||||
async def _request_state_after_a_failed_call(
|
||||
core: RouterCore, host: ScriptedHost, endpoint: str
|
||||
) -> "weakref.ref[_RequestState]":
|
||||
state: Final = _RequestState()
|
||||
with pytest.raises(litellm.BadRequestError, match="scripted bad input"):
|
||||
await core.call(endpoint, {"model": "g", "messages": MESSAGES, "max_tokens": 5, "metadata": {"state": state}})
|
||||
host.calls.clear()
|
||||
host.raised.clear()
|
||||
return weakref.ref(state)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue