diff --git a/litellm/router_rust/core.py b/litellm/router_rust/core.py index 1f35f396b98..e8d9eeb6174 100644 --- a/litellm/router_rust/core.py +++ b/litellm/router_rust/core.py @@ -5545,6 +5545,7 @@ class _Driver: def finish(self, step: Mapping[str, object]) -> object: """The response with the router's headers, in the order ``make_call`` and the wrappers above it stamp them: a usage read that raises leaves the counts off.""" + self._clear_errors() if not self._plan.sets_headers: self.log_outcomes_sync() return self._last_response @@ -5565,6 +5566,7 @@ class _Driver: async def finish_async(self, step: Mapping[str, object]) -> object: """``finish``, with the usage read on the call's own loop.""" + self._clear_errors() if not self._plan.sets_headers: await await_all(self.outcomes.take_pending()) return self._last_response @@ -5586,6 +5588,18 @@ class _Driver: return self._request.get("messages") is not None or self._request.get("input") is not None def failed(self, step: Mapping[str, object]) -> BaseException: + try: + return self._failure(step) + finally: + self._clear_errors() + + def _clear_errors(self) -> None: + self._last_error = None + self._level_errors.clear() + self._hop_first.clear() + self._level_picks.clear() + + def _failure(self, step: Mapping[str, object]) -> BaseException: model: Final = str(self._request.get("model")) primary: Final = self._level_errors.get(self._base_level) if primary is None: diff --git a/tests/unit/test_router/core/scripted_host.py b/tests/unit/test_router/core/scripted_host.py index 066faad4db8..4930ee9ec85 100644 --- a/tests/unit/test_router/core/scripted_host.py +++ b/tests/unit/test_router/core/scripted_host.py @@ -450,7 +450,10 @@ class ScriptedHost(PythonHost): error, response = self._finish(op, call, outcome) if error is not None: await self.log_async(call, error, response) - raise error + try: + raise error + finally: + del error, outcome if not self._logs_at_stream_end(call, response): await self.log_async(call, None, response) return response @@ -460,7 +463,10 @@ class ScriptedHost(PythonHost): error, response = self._finish(op, call, outcome) if error is not None: self.log_sync(call, error, response) - raise error + try: + raise error + finally: + del error, outcome if not self._logs_at_stream_end(call, response): self.log_sync(call, None, response) return response diff --git a/tests/unit/test_router/core/test_endpoints.py b/tests/unit/test_router/core/test_endpoints.py index 4e038f39aef..4ef2cfd177b 100644 --- a/tests/unit/test_router/core/test_endpoints.py +++ b/tests/unit/test_router/core/test_endpoints.py @@ -24,6 +24,7 @@ from tests.unit.test_router.core.scripted_host import ( completion_response, deployment, embedding_response, + server_error, ) MESSAGES: Final = [{"role": "user", "content": "hi"}] @@ -462,17 +463,46 @@ async def _request_state_after_a_call( state: Final = _RequestState() _ = await core.call(endpoint, {"model": "g", "messages": MESSAGES, "max_tokens": 5, "metadata": {"state": state}}) host.calls.clear() + host.raised.clear() return weakref.ref(state) @pytest.mark.parametrize("endpoint", ["acompletion", "aanthropic_messages", "completion"]) +@pytest.mark.parametrize("retry_then_succeed", [False, True]) async def test_a_finished_call_frees_its_request_without_a_gc_pass( - make_core: MakeCore, host: ScriptedHost, endpoint: str + make_core: MakeCore, host: ScriptedHost, endpoint: str, retry_then_succeed: bool ): - core: Final = _one_group(make_core) + if retry_then_succeed: + host.answer("openai/d1", server_error("d1"), None) + core: Final = make_core(model_list=[deployment("g", "d1")], num_retries=1, retry_after=0) gc.disable() try: state: Final = await _request_state_after_a_call(core, host, endpoint) assert state() is None, "the finished call's request stays in a reference cycle until the next gc pass" finally: gc.enable() + + +@pytest.mark.parametrize("endpoint", ["acompletion", "aanthropic_messages", "completion"]) +async def test_a_failed_call_frees_its_request_without_a_gc_pass( + make_core: MakeCore, host: ScriptedHost, endpoint: str +): + host.answer("openai/d1", bad_request(message="scripted bad input")) + core: Final = _one_group(make_core) + gc.disable() + try: + state: Final = await _request_state_after_a_failed_call(core, host, endpoint) + assert state() is None, "the failed call's request stays in a reference cycle until the next gc pass" + finally: + gc.enable() + + +async def _request_state_after_a_failed_call( + core: RouterCore, host: ScriptedHost, endpoint: str +) -> "weakref.ref[_RequestState]": + state: Final = _RequestState() + with pytest.raises(litellm.BadRequestError, match="scripted bad input"): + await core.call(endpoint, {"model": "g", "messages": MESSAGES, "max_tokens": 5, "metadata": {"state": state}}) + host.calls.clear() + host.raised.clear() + return weakref.ref(state)