From d49706f55d707b5b280b22b722e5256f9878b5df Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:31:07 -0500 Subject: [PATCH] fix(router): stop the rust router keeping every finished request in memory until a full gc (#51888) * fix(router): stop the rust router keeping every finished request in memory until a full gc Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): prove finished /v1/messages and chat calls free their log record without a gc pass Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): read the memory endpoint with typed helpers instead of a type ignore Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router_rust/core.py | 15 ++- .../test_finished_requests_freed_wire.py | 122 ++++++++++++++++++ tests/unit/test_router/core/test_endpoints.py | 28 ++++ 3 files changed, 158 insertions(+), 7 deletions(-) create mode 100644 tests/integration/messages_endpoint/test_finished_requests_freed_wire.py diff --git a/litellm/router_rust/core.py b/litellm/router_rust/core.py index fe6ff6d6806..1f35f396b98 100644 --- a/litellm/router_rust/core.py +++ b/litellm/router_rust/core.py @@ -4434,6 +4434,13 @@ def _log_attempt(plan: Plan, model_name: object, error: BaseException | None = N verbose_router_logger.info("%s(model=%s)\x1b[32m 200 OK\x1b[0m", served_name, model_name) +def _params_base_model(core: RustCore, deployment_id: str) -> str | None: + """``_deployment_params_base_model``: the ``base_model`` under the deployment's ``litellm_params``.""" + record: Final = as_str_mapping(core._handle.deployment(deployment_id)) or {} # pyright: ignore[reportPrivateUsage] # the driver serves its own core + base_model: Final = (as_str_mapping(record.get("litellm_params")) or {}).get("base_model") + return base_model if isinstance(base_model, str) else None + + class _Driver: """Answers the attempt and fallback steps of one call, and shapes its result. @@ -4496,7 +4503,7 @@ class _Driver: self._route_phase: contextlib.ExitStack | None = None self._route_model: str = "" self._walk: Final = WalkMetadata( - request, plan.metadata_field, self._params_base_model, live=fresh and not plan.single_attempt + request, plan.metadata_field, partial(_params_base_model, core), live=fresh and not plan.single_attempt ) self.outcomes: Final = FallbackOutcomes(request) # The fallback level the walk is at, from its fallback events. @@ -4548,12 +4555,6 @@ class _Driver: endpoints.counters.retract_success(self._plan, call) endpoints.attempt_failed(self._plan, error, deployment, call, call.group) - def _params_base_model(self, deployment_id: str) -> str | None: - """``_deployment_params_base_model``: the ``base_model`` under the deployment's ``litellm_params``.""" - record: Final = as_str_mapping(self._core._handle.deployment(deployment_id)) or {} # pyright: ignore[reportPrivateUsage] # the driver serves its own core - base_model: Final = (as_str_mapping(record.get("litellm_params")) or {}).get("base_model") - return base_model if isinstance(base_model, str) else None - def _logged_error(self, event: Mapping[str, object]) -> BaseException | None: """The error a ``retry_logged`` event records.""" return self._cause_error(event) diff --git a/tests/integration/messages_endpoint/test_finished_requests_freed_wire.py b/tests/integration/messages_endpoint/test_finished_requests_freed_wire.py new file mode 100644 index 00000000000..12863d5c0f8 --- /dev/null +++ b/tests/integration/messages_endpoint/test_finished_requests_freed_wire.py @@ -0,0 +1,122 @@ +import uuid +from collections.abc import Iterator +from pathlib import Path +from typing import Final + +import pytest +import yaml +from integration._support.anthropic_sse import message_json +from integration._support.client import ( + JSON_OBJECT, + Gateway, + eventually, + gateway_from_environment, + object_value, + string_value, +) +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue + +_MODEL: Final = "claude-haiku-5-5" +_API_KEY: Final = "synthetic-anthropic-key" +_CALLS: Final = 12 +_SLACK: Final = 2 +_NEVER: Final = 1_000_000_000 + + +def _log_records(gateway: Gateway) -> int: + response: Final = gateway.request( + "GET", "/debug/memory/details", params={"top_n": "100000", "include_process_info": "false"} + ) + assert response.status_code == 200, response.text + objects: Final = object_value(JSON_OBJECT.validate_python(response.json())["objects"]) + top_types: Final = objects["top_types"] + assert isinstance(top_types, list), objects + counts: Final = {string_value(object_value(entry)["type"]): object_value(entry)["count"] for entry in top_types} + count: Final = counts.get("Logging", 0) + assert isinstance(count, int), counts + return count + + +_CONFIG_MODEL: Final = "freed-config-model" + + +@pytest.fixture(scope="module") +def uncached_gateway(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + upstream: Final = wire_server(_answer) + wire: Final = upstream.__enter__() + directory: Final = tmp_path_factory.mktemp("finished-requests-freed") + stock: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + settings: Final = { + key: value + for key, value in JSON_OBJECT.validate_python(stock.get("litellm_settings") or {}).items() + if key not in ("cache", "cache_params") + } + config: Final = directory / "finished-requests-freed.yaml" + model_list: Final = [ + { + "model_name": _CONFIG_MODEL, + "litellm_params": {"model": f"anthropic/{_MODEL}", "api_base": wire.url, "api_key": _API_KEY}, + } + ] + config.write_text(yaml.safe_dump({**stock, "litellm_settings": settings, "model_list": model_list})) + try: + with gateway_from_environment() as shared, owned_proxy(shared, directory, {}, config=config) as owned: + yield owned + finally: + upstream.__exit__(None, None, None) + + +@pytest.fixture +def no_cleanup_pass(uncached_gateway: Gateway) -> Iterator[None]: + gateway: Final = uncached_gateway + configured: Final = gateway.request("POST", "/debug/memory/gc/configure", params={"generation_0": str(_NEVER)}) + assert configured.status_code == 200, configured.text + previous: Final = [ + part.strip() + for part in string_value(JSON_OBJECT.validate_python(configured.json())["previous_thresholds"]).split(",") + ] + try: + yield + finally: + restored: Final = gateway.request( + "POST", + "/debug/memory/gc/configure", + params={"generation_0": previous[0], "generation_1": previous[1], "generation_2": previous[2]}, + ) + assert restored.status_code == 200, restored.text + + +def _answer(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/v1/messages" + return Reply(body=message_json("msg_freed", _MODEL, "Hi!")) + + +def _call(gateway: Gateway, path: str, model: str) -> None: + messages: Final[list[JsonValue]] = [{"role": "user", "content": f"Say hi {uuid.uuid4().hex}"}] + body: Final[dict[str, JsonValue]] = ( + {"model": model, "max_tokens": 16, "messages": messages} + if path == "/v1/messages" + else {"model": model, "messages": messages} + ) + response: Final = gateway.request("POST", path, body) + assert response.status_code == 200, response.text + + +@pytest.mark.parametrize("path", ("/v1/messages", "/v1/chat/completions")) +def test_finished_calls_are_freed_without_a_cleanup_pass( + uncached_gateway: Gateway, no_cleanup_pass: None, path: str +) -> None: + gateway: Final = uncached_gateway + _call(gateway, path, _CONFIG_MODEL) + before: Final = _log_records(gateway) + for _ in range(_CALLS): + _call(gateway, path, _CONFIG_MODEL) + after: Final = eventually( + lambda: _log_records(gateway), lambda count: count <= before + _SLACK, seconds=15, return_last_on_timeout=True + ) + assert after <= before + _SLACK, ( + f"{after - before} of {_CALLS} finished {path} calls still hold their log record with Python's cycle " + "collector held off, so each one waits for a full cleanup pass before its memory comes back" + ) diff --git a/tests/unit/test_router/core/test_endpoints.py b/tests/unit/test_router/core/test_endpoints.py index d129dc9f408..4e038f39aef 100644 --- a/tests/unit/test_router/core/test_endpoints.py +++ b/tests/unit/test_router/core/test_endpoints.py @@ -5,7 +5,9 @@ Every test builds its core through ``make_core`` on the ``host`` fixture, so it """ import datetime +import gc import io +import weakref from collections.abc import AsyncIterator, Callable, Coroutine, Mapping from typing import Final @@ -448,3 +450,29 @@ async def test_streaming_acompletion_yields_the_host_stream_chunks_with_routing_ assert stream._hidden_params["model_id"] == "d1" assert stream._hidden_params["additional_headers"]["x-litellm-model-group"] == "g" assert (host.calls[0].op, host.calls[0].kwargs["stream"]) == ("acompletion", True) + + +class _RequestState: + pass + + +async def _request_state_after_a_call( + core: RouterCore, host: ScriptedHost, endpoint: str +) -> "weakref.ref[_RequestState]": + state: Final = _RequestState() + _ = await core.call(endpoint, {"model": "g", "messages": MESSAGES, "max_tokens": 5, "metadata": {"state": state}}) + host.calls.clear() + return weakref.ref(state) + + +@pytest.mark.parametrize("endpoint", ["acompletion", "aanthropic_messages", "completion"]) +async def test_a_finished_call_frees_its_request_without_a_gc_pass( + make_core: MakeCore, host: ScriptedHost, endpoint: str +): + core: Final = _one_group(make_core) + gc.disable() + try: + state: Final = await _request_state_after_a_call(core, host, endpoint) + assert state() is None, "the finished call's request stays in a reference cycle until the next gc pass" + finally: + gc.enable()