mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(router): stop the rust router keeping every finished request in memory until a full gc (#51888)
* fix(router): stop the rust router keeping every finished request in memory until a full gc Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): prove finished /v1/messages and chat calls free their log record without a gc pass Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): read the memory endpoint with typed helpers instead of a type ignore Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
61a1648572
commit
d49706f55d
3 changed files with 158 additions and 7 deletions
|
|
@ -4434,6 +4434,13 @@ def _log_attempt(plan: Plan, model_name: object, error: BaseException | None = N
|
|||
verbose_router_logger.info("%s(model=%s)\x1b[32m 200 OK\x1b[0m", served_name, model_name)
|
||||
|
||||
|
||||
def _params_base_model(core: RustCore, deployment_id: str) -> str | None:
|
||||
"""``_deployment_params_base_model``: the ``base_model`` under the deployment's ``litellm_params``."""
|
||||
record: Final = as_str_mapping(core._handle.deployment(deployment_id)) or {} # pyright: ignore[reportPrivateUsage] # the driver serves its own core
|
||||
base_model: Final = (as_str_mapping(record.get("litellm_params")) or {}).get("base_model")
|
||||
return base_model if isinstance(base_model, str) else None
|
||||
|
||||
|
||||
class _Driver:
|
||||
"""Answers the attempt and fallback steps of one call, and shapes its result.
|
||||
|
||||
|
|
@ -4496,7 +4503,7 @@ class _Driver:
|
|||
self._route_phase: contextlib.ExitStack | None = None
|
||||
self._route_model: str = ""
|
||||
self._walk: Final = WalkMetadata(
|
||||
request, plan.metadata_field, self._params_base_model, live=fresh and not plan.single_attempt
|
||||
request, plan.metadata_field, partial(_params_base_model, core), live=fresh and not plan.single_attempt
|
||||
)
|
||||
self.outcomes: Final = FallbackOutcomes(request)
|
||||
# The fallback level the walk is at, from its fallback events.
|
||||
|
|
@ -4548,12 +4555,6 @@ class _Driver:
|
|||
endpoints.counters.retract_success(self._plan, call)
|
||||
endpoints.attempt_failed(self._plan, error, deployment, call, call.group)
|
||||
|
||||
def _params_base_model(self, deployment_id: str) -> str | None:
|
||||
"""``_deployment_params_base_model``: the ``base_model`` under the deployment's ``litellm_params``."""
|
||||
record: Final = as_str_mapping(self._core._handle.deployment(deployment_id)) or {} # pyright: ignore[reportPrivateUsage] # the driver serves its own core
|
||||
base_model: Final = (as_str_mapping(record.get("litellm_params")) or {}).get("base_model")
|
||||
return base_model if isinstance(base_model, str) else None
|
||||
|
||||
def _logged_error(self, event: Mapping[str, object]) -> BaseException | None:
|
||||
"""The error a ``retry_logged`` event records."""
|
||||
return self._cause_error(event)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,122 @@
|
|||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.anthropic_sse import message_json
|
||||
from integration._support.client import (
|
||||
JSON_OBJECT,
|
||||
Gateway,
|
||||
eventually,
|
||||
gateway_from_environment,
|
||||
object_value,
|
||||
string_value,
|
||||
)
|
||||
from integration._support.process import owned_proxy
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import JsonValue
|
||||
|
||||
_MODEL: Final = "claude-haiku-5-5"
|
||||
_API_KEY: Final = "synthetic-anthropic-key"
|
||||
_CALLS: Final = 12
|
||||
_SLACK: Final = 2
|
||||
_NEVER: Final = 1_000_000_000
|
||||
|
||||
|
||||
def _log_records(gateway: Gateway) -> int:
|
||||
response: Final = gateway.request(
|
||||
"GET", "/debug/memory/details", params={"top_n": "100000", "include_process_info": "false"}
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
objects: Final = object_value(JSON_OBJECT.validate_python(response.json())["objects"])
|
||||
top_types: Final = objects["top_types"]
|
||||
assert isinstance(top_types, list), objects
|
||||
counts: Final = {string_value(object_value(entry)["type"]): object_value(entry)["count"] for entry in top_types}
|
||||
count: Final = counts.get("Logging", 0)
|
||||
assert isinstance(count, int), counts
|
||||
return count
|
||||
|
||||
|
||||
_CONFIG_MODEL: Final = "freed-config-model"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def uncached_gateway(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]:
|
||||
upstream: Final = wire_server(_answer)
|
||||
wire: Final = upstream.__enter__()
|
||||
directory: Final = tmp_path_factory.mktemp("finished-requests-freed")
|
||||
stock: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
|
||||
settings: Final = {
|
||||
key: value
|
||||
for key, value in JSON_OBJECT.validate_python(stock.get("litellm_settings") or {}).items()
|
||||
if key not in ("cache", "cache_params")
|
||||
}
|
||||
config: Final = directory / "finished-requests-freed.yaml"
|
||||
model_list: Final = [
|
||||
{
|
||||
"model_name": _CONFIG_MODEL,
|
||||
"litellm_params": {"model": f"anthropic/{_MODEL}", "api_base": wire.url, "api_key": _API_KEY},
|
||||
}
|
||||
]
|
||||
config.write_text(yaml.safe_dump({**stock, "litellm_settings": settings, "model_list": model_list}))
|
||||
try:
|
||||
with gateway_from_environment() as shared, owned_proxy(shared, directory, {}, config=config) as owned:
|
||||
yield owned
|
||||
finally:
|
||||
upstream.__exit__(None, None, None)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def no_cleanup_pass(uncached_gateway: Gateway) -> Iterator[None]:
|
||||
gateway: Final = uncached_gateway
|
||||
configured: Final = gateway.request("POST", "/debug/memory/gc/configure", params={"generation_0": str(_NEVER)})
|
||||
assert configured.status_code == 200, configured.text
|
||||
previous: Final = [
|
||||
part.strip()
|
||||
for part in string_value(JSON_OBJECT.validate_python(configured.json())["previous_thresholds"]).split(",")
|
||||
]
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
restored: Final = gateway.request(
|
||||
"POST",
|
||||
"/debug/memory/gc/configure",
|
||||
params={"generation_0": previous[0], "generation_1": previous[1], "generation_2": previous[2]},
|
||||
)
|
||||
assert restored.status_code == 200, restored.text
|
||||
|
||||
|
||||
def _answer(request: Request) -> Reply:
|
||||
assert request.method == "POST" and request.target == "/v1/messages"
|
||||
return Reply(body=message_json("msg_freed", _MODEL, "Hi!"))
|
||||
|
||||
|
||||
def _call(gateway: Gateway, path: str, model: str) -> None:
|
||||
messages: Final[list[JsonValue]] = [{"role": "user", "content": f"Say hi {uuid.uuid4().hex}"}]
|
||||
body: Final[dict[str, JsonValue]] = (
|
||||
{"model": model, "max_tokens": 16, "messages": messages}
|
||||
if path == "/v1/messages"
|
||||
else {"model": model, "messages": messages}
|
||||
)
|
||||
response: Final = gateway.request("POST", path, body)
|
||||
assert response.status_code == 200, response.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ("/v1/messages", "/v1/chat/completions"))
|
||||
def test_finished_calls_are_freed_without_a_cleanup_pass(
|
||||
uncached_gateway: Gateway, no_cleanup_pass: None, path: str
|
||||
) -> None:
|
||||
gateway: Final = uncached_gateway
|
||||
_call(gateway, path, _CONFIG_MODEL)
|
||||
before: Final = _log_records(gateway)
|
||||
for _ in range(_CALLS):
|
||||
_call(gateway, path, _CONFIG_MODEL)
|
||||
after: Final = eventually(
|
||||
lambda: _log_records(gateway), lambda count: count <= before + _SLACK, seconds=15, return_last_on_timeout=True
|
||||
)
|
||||
assert after <= before + _SLACK, (
|
||||
f"{after - before} of {_CALLS} finished {path} calls still hold their log record with Python's cycle "
|
||||
"collector held off, so each one waits for a full cleanup pass before its memory comes back"
|
||||
)
|
||||
|
|
@ -5,7 +5,9 @@ Every test builds its core through ``make_core`` on the ``host`` fixture, so it
|
|||
"""
|
||||
|
||||
import datetime
|
||||
import gc
|
||||
import io
|
||||
import weakref
|
||||
from collections.abc import AsyncIterator, Callable, Coroutine, Mapping
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -448,3 +450,29 @@ async def test_streaming_acompletion_yields_the_host_stream_chunks_with_routing_
|
|||
assert stream._hidden_params["model_id"] == "d1"
|
||||
assert stream._hidden_params["additional_headers"]["x-litellm-model-group"] == "g"
|
||||
assert (host.calls[0].op, host.calls[0].kwargs["stream"]) == ("acompletion", True)
|
||||
|
||||
|
||||
class _RequestState:
|
||||
pass
|
||||
|
||||
|
||||
async def _request_state_after_a_call(
|
||||
core: RouterCore, host: ScriptedHost, endpoint: str
|
||||
) -> "weakref.ref[_RequestState]":
|
||||
state: Final = _RequestState()
|
||||
_ = await core.call(endpoint, {"model": "g", "messages": MESSAGES, "max_tokens": 5, "metadata": {"state": state}})
|
||||
host.calls.clear()
|
||||
return weakref.ref(state)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("endpoint", ["acompletion", "aanthropic_messages", "completion"])
|
||||
async def test_a_finished_call_frees_its_request_without_a_gc_pass(
|
||||
make_core: MakeCore, host: ScriptedHost, endpoint: str
|
||||
):
|
||||
core: Final = _one_group(make_core)
|
||||
gc.disable()
|
||||
try:
|
||||
state: Final = await _request_state_after_a_call(core, host, endpoint)
|
||||
assert state() is None, "the finished call's request stays in a reference cycle until the next gc pass"
|
||||
finally:
|
||||
gc.enable()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue