fix(router): free a failed rust router call's request without waiting for a full gc (#51963)

* fix(router): free a failed rust router call's request without waiting for a full gc

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(router): release retained retry errors after successful calls

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Mateo Wang 2026-11-15 17:04:38 -05:00 • committed by GitHub
parent 1e3a960a0e
commit 8c97c243fe
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 54 additions and 4 deletions

View file

@ -5545,6 +5545,7 @@ class _Driver:
def finish(self, step: Mapping[str, object]) -> object:
"""The response with the router's headers, in the order ``make_call`` and the wrappers above it
stamp them: a usage read that raises leaves the counts off."""
self._clear_errors()
if not self._plan.sets_headers:
self.log_outcomes_sync()
return self._last_response
@ -5565,6 +5566,7 @@ class _Driver:
async def finish_async(self, step: Mapping[str, object]) -> object:
"""``finish``, with the usage read on the call's own loop."""
self._clear_errors()
if not self._plan.sets_headers:
await await_all(self.outcomes.take_pending())
return self._last_response
@ -5586,6 +5588,18 @@ class _Driver:
return self._request.get("messages") is not None or self._request.get("input") is not None
def failed(self, step: Mapping[str, object]) -> BaseException:
try:
return self._failure(step)
finally:
self._clear_errors()
def _clear_errors(self) -> None:
self._last_error = None
self._level_errors.clear()
self._hop_first.clear()
self._level_picks.clear()
def _failure(self, step: Mapping[str, object]) -> BaseException:
model: Final = str(self._request.get("model"))
primary: Final = self._level_errors.get(self._base_level)
if primary is None:

View file

@ -450,7 +450,10 @@ class ScriptedHost(PythonHost):
error, response = self._finish(op, call, outcome)
if error is not None:
await self.log_async(call, error, response)
raise error
try:
raise error
finally:
del error, outcome
if not self._logs_at_stream_end(call, response):
await self.log_async(call, None, response)
return response
@ -460,7 +463,10 @@ class ScriptedHost(PythonHost):
error, response = self._finish(op, call, outcome)
if error is not None:
self.log_sync(call, error, response)
raise error
try:
raise error
finally:
del error, outcome
if not self._logs_at_stream_end(call, response):
self.log_sync(call, None, response)
return response

View file

@ -24,6 +24,7 @@ from tests.unit.test_router.core.scripted_host import (
completion_response,
deployment,
embedding_response,
server_error,
)
MESSAGES: Final = [{"role": "user", "content": "hi"}]
@ -462,17 +463,46 @@ async def _request_state_after_a_call(
state: Final = _RequestState()
_ = await core.call(endpoint, {"model": "g", "messages": MESSAGES, "max_tokens": 5, "metadata": {"state": state}})
host.calls.clear()
host.raised.clear()
return weakref.ref(state)
@pytest.mark.parametrize("endpoint", ["acompletion", "aanthropic_messages", "completion"])
@pytest.mark.parametrize("retry_then_succeed", [False, True])
async def test_a_finished_call_frees_its_request_without_a_gc_pass(
make_core: MakeCore, host: ScriptedHost, endpoint: str
make_core: MakeCore, host: ScriptedHost, endpoint: str, retry_then_succeed: bool
):
core: Final = _one_group(make_core)
if retry_then_succeed:
host.answer("openai/d1", server_error("d1"), None)
core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=1, retry_after=0)
gc.disable()
try:
state: Final = await _request_state_after_a_call(core, host, endpoint)
assert state() is None, "the finished call's request stays in a reference cycle until the next gc pass"
finally:
gc.enable()
@pytest.mark.parametrize("endpoint", ["acompletion", "aanthropic_messages", "completion"])
async def test_a_failed_call_frees_its_request_without_a_gc_pass(
make_core: MakeCore, host: ScriptedHost, endpoint: str
):
host.answer("openai/d1", bad_request(message="scripted bad input"))
core: Final = _one_group(make_core)
gc.disable()
try:
state: Final = await _request_state_after_a_failed_call(core, host, endpoint)
assert state() is None, "the failed call's request stays in a reference cycle until the next gc pass"
finally:
gc.enable()
async def _request_state_after_a_failed_call(
core: RouterCore, host: ScriptedHost, endpoint: str
) -> "weakref.ref[_RequestState]":
state: Final = _RequestState()
with pytest.raises(litellm.BadRequestError, match="scripted bad input"):
await core.call(endpoint, {"model": "g", "messages": MESSAGES, "max_tokens": 5, "metadata": {"state": state}})
host.calls.clear()
host.raised.clear()
return weakref.ref(state)