fix(router): stop the rust router keeping every finished request in memory until a full gc (#51888)

* fix(router): stop the rust router keeping every finished request in memory until a full gc

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(integration): prove finished /v1/messages and chat calls free their log record without a gc pass

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(integration): read the memory endpoint with typed helpers instead of a type ignore

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Mateo Wang 2026-11-15 14:31:07 -05:00 • committed by GitHub
parent 61a1648572
commit d49706f55d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 158 additions and 7 deletions

View file

@ -4434,6 +4434,13 @@ def _log_attempt(plan: Plan, model_name: object, error: BaseException | None = N
verbose_router_logger.info("%s(model=%s)\x1b[32m 200 OK\x1b[0m", served_name, model_name)
def _params_base_model(core: RustCore, deployment_id: str) -> str | None:
"""``_deployment_params_base_model``: the ``base_model`` under the deployment's ``litellm_params``."""
record: Final = as_str_mapping(core._handle.deployment(deployment_id)) or {} # pyright: ignore[reportPrivateUsage] # the driver serves its own core
base_model: Final = (as_str_mapping(record.get("litellm_params")) or {}).get("base_model")
return base_model if isinstance(base_model, str) else None
class _Driver:
"""Answers the attempt and fallback steps of one call, and shapes its result.
@ -4496,7 +4503,7 @@ class _Driver:
self._route_phase: contextlib.ExitStack | None = None
self._route_model: str = ""
self._walk: Final = WalkMetadata(
request, plan.metadata_field, self._params_base_model, live=fresh and not plan.single_attempt
request, plan.metadata_field, partial(_params_base_model, core), live=fresh and not plan.single_attempt
)
self.outcomes: Final = FallbackOutcomes(request)
# The fallback level the walk is at, from its fallback events.
@ -4548,12 +4555,6 @@ class _Driver:
endpoints.counters.retract_success(self._plan, call)
endpoints.attempt_failed(self._plan, error, deployment, call, call.group)
def _params_base_model(self, deployment_id: str) -> str | None:
"""``_deployment_params_base_model``: the ``base_model`` under the deployment's ``litellm_params``."""
record: Final = as_str_mapping(self._core._handle.deployment(deployment_id)) or {} # pyright: ignore[reportPrivateUsage] # the driver serves its own core
base_model: Final = (as_str_mapping(record.get("litellm_params")) or {}).get("base_model")
return base_model if isinstance(base_model, str) else None
def _logged_error(self, event: Mapping[str, object]) -> BaseException | None:
"""The error a ``retry_logged`` event records."""
return self._cause_error(event)

View file

@ -0,0 +1,122 @@
import uuid
from collections.abc import Iterator
from pathlib import Path
from typing import Final
import pytest
import yaml
from integration._support.anthropic_sse import message_json
from integration._support.client import (
JSON_OBJECT,
Gateway,
eventually,
gateway_from_environment,
object_value,
string_value,
)
from integration._support.process import owned_proxy
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue
_MODEL: Final = "claude-haiku-5-5"
_API_KEY: Final = "synthetic-anthropic-key"
_CALLS: Final = 12
_SLACK: Final = 2
_NEVER: Final = 1_000_000_000
def _log_records(gateway: Gateway) -> int:
response: Final = gateway.request(
"GET", "/debug/memory/details", params={"top_n": "100000", "include_process_info": "false"}
)
assert response.status_code == 200, response.text
objects: Final = object_value(JSON_OBJECT.validate_python(response.json())["objects"])
top_types: Final = objects["top_types"]
assert isinstance(top_types, list), objects
counts: Final = {string_value(object_value(entry)["type"]): object_value(entry)["count"] for entry in top_types}
count: Final = counts.get("Logging", 0)
assert isinstance(count, int), counts
return count
_CONFIG_MODEL: Final = "freed-config-model"
@pytest.fixture(scope="module")
def uncached_gateway(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]:
upstream: Final = wire_server(_answer)
wire: Final = upstream.__enter__()
directory: Final = tmp_path_factory.mktemp("finished-requests-freed")
stock: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
settings: Final = {
key: value
for key, value in JSON_OBJECT.validate_python(stock.get("litellm_settings") or {}).items()
if key not in ("cache", "cache_params")
}
config: Final = directory / "finished-requests-freed.yaml"
model_list: Final = [
{
"model_name": _CONFIG_MODEL,
"litellm_params": {"model": f"anthropic/{_MODEL}", "api_base": wire.url, "api_key": _API_KEY},
}
]
config.write_text(yaml.safe_dump({**stock, "litellm_settings": settings, "model_list": model_list}))
try:
with gateway_from_environment() as shared, owned_proxy(shared, directory, {}, config=config) as owned:
yield owned
finally:
upstream.__exit__(None, None, None)
@pytest.fixture
def no_cleanup_pass(uncached_gateway: Gateway) -> Iterator[None]:
gateway: Final = uncached_gateway
configured: Final = gateway.request("POST", "/debug/memory/gc/configure", params={"generation_0": str(_NEVER)})
assert configured.status_code == 200, configured.text
previous: Final = [
part.strip()
for part in string_value(JSON_OBJECT.validate_python(configured.json())["previous_thresholds"]).split(",")
]
try:
yield
finally:
restored: Final = gateway.request(
"POST",
"/debug/memory/gc/configure",
params={"generation_0": previous[0], "generation_1": previous[1], "generation_2": previous[2]},
)
assert restored.status_code == 200, restored.text
def _answer(request: Request) -> Reply:
assert request.method == "POST" and request.target == "/v1/messages"
return Reply(body=message_json("msg_freed", _MODEL, "Hi!"))
def _call(gateway: Gateway, path: str, model: str) -> None:
messages: Final[list[JsonValue]] = [{"role": "user", "content": f"Say hi {uuid.uuid4().hex}"}]
body: Final[dict[str, JsonValue]] = (
{"model": model, "max_tokens": 16, "messages": messages}
if path == "/v1/messages"
else {"model": model, "messages": messages}
)
response: Final = gateway.request("POST", path, body)
assert response.status_code == 200, response.text
@pytest.mark.parametrize("path", ("/v1/messages", "/v1/chat/completions"))
def test_finished_calls_are_freed_without_a_cleanup_pass(
uncached_gateway: Gateway, no_cleanup_pass: None, path: str
) -> None:
gateway: Final = uncached_gateway
_call(gateway, path, _CONFIG_MODEL)
before: Final = _log_records(gateway)
for _ in range(_CALLS):
_call(gateway, path, _CONFIG_MODEL)
after: Final = eventually(
lambda: _log_records(gateway), lambda count: count <= before + _SLACK, seconds=15, return_last_on_timeout=True
)
assert after <= before + _SLACK, (
f"{after - before} of {_CALLS} finished {path} calls still hold their log record with Python's cycle "
"collector held off, so each one waits for a full cleanup pass before its memory comes back"
)

View file

@ -5,7 +5,9 @@ Every test builds its core through ``make_core`` on the ``host`` fixture, so it
"""
import datetime
import gc
import io
import weakref
from collections.abc import AsyncIterator, Callable, Coroutine, Mapping
from typing import Final
@ -448,3 +450,29 @@ async def test_streaming_acompletion_yields_the_host_stream_chunks_with_routing_
assert stream._hidden_params["model_id"] == "d1"
assert stream._hidden_params["additional_headers"]["x-litellm-model-group"] == "g"
assert (host.calls[0].op, host.calls[0].kwargs["stream"]) == ("acompletion", True)
class _RequestState:
pass
async def _request_state_after_a_call(
core: RouterCore, host: ScriptedHost, endpoint: str
) -> "weakref.ref[_RequestState]":
state: Final = _RequestState()
_ = await core.call(endpoint, {"model": "g", "messages": MESSAGES, "max_tokens": 5, "metadata": {"state": state}})
host.calls.clear()
return weakref.ref(state)
@pytest.mark.parametrize("endpoint", ["acompletion", "aanthropic_messages", "completion"])
async def test_a_finished_call_frees_its_request_without_a_gc_pass(
make_core: MakeCore, host: ScriptedHost, endpoint: str
):
core: Final = _one_group(make_core)
gc.disable()
try:
state: Final = await _request_state_after_a_call(core, host, endpoint)
assert state() is None, "the finished call's request stays in a reference cycle until the next gc pass"
finally:
gc.enable()