From a9ce9454b42072f9c75916d8d0332f719e129064 Mon Sep 17 00:00:00 2001 From: mateo Date: Sun, 11 Oct 2026 01:48:42 +0000 Subject: [PATCH 01/13] fix(proxy): support reload with multiple workers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- AGENTS.md | 4 +- litellm/proxy/dev_reload.py | 73 +++++++++++++++ litellm/proxy/proxy_cli.py | 11 ++- .../configuration/test_multi_worker_reload.py | 90 +++++++++++++++++++ 4 files changed, 175 insertions(+), 3 deletions(-) create mode 100644 litellm/proxy/dev_reload.py create mode 100644 tests/integration/configuration/test_multi_worker_reload.py diff --git a/AGENTS.md b/AGENTS.md index 7f8e9cedc26..7fb3033dc59 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,7 +39,9 @@ Same applies for filing bug reports and feature requests, with .github/ISSUE_TEM If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just drop the section -Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`. The Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into. I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, or the main use case runs through a headful agentic coding tool like Claude Code or Codex, drive that surface yourself and embed your own before and after screenshots of it in the PR (the Admin UI page, or what the coding tool shows), next to an ordered list of the URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, and what fields to fill out so a reviewer can reproduce it +Run every QA proxy with `--num_workers 2 --reload` and a dedicated Redis instance configured through `router_settings.redis_host`, `redis_port`, and credentials when required. Verify two application worker PIDs are serving, then edit a watched Python, config YAML, or `.env` file and verify both workers are replaced and requests still succeed. Use the same worker count and Redis topology for before and after proof + +Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --num_workers 2 --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`. The Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into. I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, or the main use case runs through a headful agentic coding tool like Claude Code or Codex, drive that surface yourself and embed your own before and after screenshots of it in the PR (the Admin UI page, or what the coding tool shows), next to an ordered list of the URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, and what fields to fill out so a reviewer can reproduce it If you ever write any human-facing text (pull requests, issues, commit messages, discussion posts, github comments, release notes, docs, etc.), always follow these guidelines to sound less AI-y: - don't use emojis diff --git a/litellm/proxy/dev_reload.py b/litellm/proxy/dev_reload.py new file mode 100644 index 00000000000..f60aa9a93f8 --- /dev/null +++ b/litellm/proxy/dev_reload.py @@ -0,0 +1,73 @@ +from collections.abc import Sequence +from functools import partial +from inspect import signature +from socket import socket +from typing import Final, Literal + +from pydantic import ConfigDict, JsonValue, TypeAdapter, with_config +from typing_extensions import ReadOnly, Required, TypedDict +from uvicorn import Config, Server +from uvicorn.supervisors import ChangeReload, Multiprocess + + +class WorkerOptions(TypedDict, total=False): + app: Required[ReadOnly[str]] + host: ReadOnly[str] + port: ReadOnly[int] + server_header: ReadOnly[bool] + log_config: ReadOnly[str | dict[str, JsonValue]] # mutable-ok: uvicorn.Config requires a concrete logging dict + log_level: ReadOnly[str | int] + timeout_keep_alive: ReadOnly[int] + timeout_worker_healthcheck: ReadOnly[int] + limit_max_requests: ReadOnly[int] + limit_max_requests_jitter: ReadOnly[int] + limit_concurrency: ReadOnly[int] + ssl_keyfile: ReadOnly[str] + ssl_certfile: ReadOnly[str] + loop: ReadOnly[Literal["none", "auto", "asyncio", "uvloop"]] + + +@with_config(ConfigDict(extra="forbid")) +class ReloadOptions(WorkerOptions): + reload: ReadOnly[bool] + reload_dirs: ReadOnly[tuple[str, ...]] + reload_includes: ReadOnly[tuple[str, ...]] + + +RELOAD_OPTIONS: Final = TypeAdapter(ReloadOptions) +WORKER_OPTIONS: Final = TypeAdapter(WorkerOptions) + + +def _run_worker_pool(config: Config, sockets: Sequence[socket] | None) -> None: + shared_sockets: Final = list(sockets or ()) + pool: Final = ( + Multiprocess( + config=config, + target=Server(config=config).run, # pyright: ignore[reportCallIssue] # Older uvicorn versions require target + sockets=shared_sockets, + ) + if "target" in signature(Multiprocess).parameters + else Multiprocess(config=config, sockets=shared_sockets) + ) + pool.run() + + +def run_multi_worker_reload(options: ReloadOptions, workers: int) -> None: + worker_options: Final = WORKER_OPTIONS.validate_python(options) + worker_config: Final = Config(**worker_options, workers=workers) + reload_config: Final = Config( + **worker_options, + workers=1, + reload=options["reload"], + reload_dirs=list(options["reload_dirs"]), + reload_includes=list(options["reload_includes"]), + ) + shared_socket: Final = worker_config.bind_socket() + try: + ChangeReload( + config=reload_config, + target=partial(_run_worker_pool, worker_config), + sockets=[shared_socket], + ).run() + finally: + shared_socket.close() diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index c7371f294f0..035f2021cb0 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -15,7 +15,7 @@ import click import httpx from click.core import ParameterSource from dotenv import load_dotenv -from pydantic import ConfigDict +from pydantic import ConfigDict, TypeAdapter import litellm from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY @@ -1033,7 +1033,7 @@ class ProxyInitializationHelpers: "--reload", is_flag=True, default=False, - help="Enable uvicorn hot reload (dev only). Also reloads when the --config YAML file changes. Incompatible with --num_workers>1, --run_gunicorn, and --run_hypercorn.", + help="Enable uvicorn hot reload (dev only), including all --num_workers workers. Also reloads when the --config YAML or .env changes. Incompatible with --run_gunicorn, --run_hypercorn, and --run_granian.", ) @click.option( "--prometheus_metrics_port", @@ -1596,6 +1596,13 @@ def run_server( if num_workers > 1: start_query_engine_reaper() + if reload and num_workers > 1: + from litellm.proxy.dev_reload import RELOAD_OPTIONS, run_multi_worker_reload + + run_multi_worker_reload( + RELOAD_OPTIONS.validate_python(uvicorn_args), workers=TypeAdapter(int).validate_python(num_workers) + ) + return uvicorn.run( **uvicorn_args, workers=num_workers, diff --git a/tests/integration/configuration/test_multi_worker_reload.py b/tests/integration/configuration/test_multi_worker_reload.py new file mode 100644 index 00000000000..64f6521a13e --- /dev/null +++ b/tests/integration/configuration/test_multi_worker_reload.py @@ -0,0 +1,90 @@ +import json +import os +import re +from pathlib import Path +from typing import Final + +import psutil +import pytest + +from tests.integration._support.client import Gateway, eventually, object_value +from tests.integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process + +STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]") + + +def _worker_pids(owned: OwnedProxy) -> frozenset[int]: + return frozenset(int(match.group(1)) for match in STARTED_WORKER.finditer(owned.log.read_text())) + + +def _config(model: str) -> str: + return json.dumps( + { + "model_list": [{"model_name": model, "litellm_params": {"model": "openai/reload-test"}}], + "general_settings": {"master_key": "os.environ/LITELLM_MASTER_KEY"}, + "router_settings": {"redis_host": "os.environ/REDIS_HOST", "redis_port": "os.environ/REDIS_PORT"}, + } + ) + + +def _assert_replaced(owned: OwnedProxy, previous: frozenset[int], workers: int) -> frozenset[int]: + started: Final = eventually( + lambda: _worker_pids(owned), + lambda pids: len(pids - previous) == workers, + seconds=graceful_stop_seconds(), + ) + assert all(not psutil.pid_exists(pid) for pid in previous), owned.log.read_text() + assert owned.process.poll() is None, owned.log.read_text() + assert len(started) == len(previous) + workers, owned.log.read_text() + return started + + +def _assert_models(gateway: Gateway) -> None: + data: Final = gateway.get("/v1/models")["data"] + assert isinstance(data, list), data + assert tuple(object_value(model)["id"] for model in data) == ("after-reload",) + + +@pytest.mark.parametrize("workers", (1, 2)) +@pytest.mark.timeout(6 * graceful_stop_seconds()) +def test_reload_replaces_every_worker_and_shutdown_leaves_none( + gateway: Gateway, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, workers: int +) -> None: + root: Final = Path(__file__).resolve().parents[3] + monkeypatch.setenv("INTEGRATION_PROXY_ROOT", str(tmp_path)) + config: Final = tmp_path / "config.yaml" + source: Final = tmp_path / "reload_source.py" + env: Final = tmp_path / ".env" + config.write_text(_config("before-reload")) + source.write_text("VALUE = 1\n") + env.write_text("RELOAD_TEST_VALUE=before\n") + with owned_proxy_process( + gateway, + tmp_path, + { + "PYTHONPATH": os.pathsep.join((str(root), str(root / "tests"), os.environ.get("PYTHONPATH", ""))), + "STORE_MODEL_IN_DB": "False", + }, + config=config, + remove_environment=("DATABASE_URL", "DATABASE_URL_READ_REPLICA", "PYTHON_DOTENV_DISABLED"), + workers=workers, + database_setup=(), + extra_arguments=("--reload",), + ) as owned: + initial: Final = eventually( + lambda: _worker_pids(owned), lambda pids: len(pids) == workers, seconds=graceful_stop_seconds() + ) + config.write_text(_config("after-reload")) + after_config: Final = _assert_replaced(owned, initial, workers) + for _ in range(2 * workers): + _assert_models(owned.gateway) + source.write_text("VALUE = 2\n") + after_source: Final = _assert_replaced(owned, after_config, workers) + env.write_text("RELOAD_TEST_VALUE=after\n") + after_env: Final = _assert_replaced(owned, after_source, workers) + response: Final = owned.gateway.request("GET", "/health/readiness") + assert response.status_code == 200, response.text + owned.process.terminate() + owned.process.wait(timeout=graceful_stop_seconds()) + assert all(not psutil.pid_exists(pid) for pid in after_env), owned.log.read_text() + assert owned.process.returncode == 0, owned.log.read_text() From 48729c7e7359a71e40643a6f88316a3a6cfa730c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 13 Oct 2026 10:35:49 -0700 Subject: [PATCH 02/13] fix(proxy): stop the reload worker pool on a broken edit instead of respawning it Workers now load the app through a factory that exits with uvicorn's STARTUP_FAILURE code on any import error, so the pool stops and the reloader waits for the next save, the same as single-worker reload. Multi-worker reload needs uvicorn>=0.50.0 for that, so older versions keep the single-worker reload with a warning. --- AGENTS.md | 2 +- litellm/proxy/dev_reload.py | 44 ++++++++++++++++++++++------ litellm/proxy/proxy_cli.py | 22 ++++++++++---- tests/unit/proxy/test_dev_reload.py | 45 +++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 16 deletions(-) create mode 100644 tests/unit/proxy/test_dev_reload.py diff --git a/AGENTS.md b/AGENTS.md index 7fb3033dc59..eafa9ba8e9c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,7 +39,7 @@ Same applies for filing bug reports and feature requests, with .github/ISSUE_TEM If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just drop the section -Run every QA proxy with `--num_workers 2 --reload` and a dedicated Redis instance configured through `router_settings.redis_host`, `redis_port`, and credentials when required. Verify two application worker PIDs are serving, then edit a watched Python, config YAML, or `.env` file and verify both workers are replaced and requests still succeed. Use the same worker count and Redis topology for before and after proof +Run every QA proxy with `--num_workers 2 --reload` and a dedicated Redis instance configured through `router_settings.redis_host`, `redis_port`, and credentials when required. `litellm/proxy/dev_config.yaml` sets no Redis, so QA from a copy of it that adds them. Verify two application worker PIDs are serving, then edit a watched Python, config YAML, or `.env` file and verify both workers are replaced and requests still succeed. Use the same worker count and Redis topology for before and after proof Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --num_workers 2 --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`. The Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into. I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, or the main use case runs through a headful agentic coding tool like Claude Code or Codex, drive that surface yourself and embed your own before and after screenshots of it in the PR (the Admin UI page, or what the coding tool shows), next to an ordered list of the URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, and what fields to fill out so a reviewer can reproduce it diff --git a/litellm/proxy/dev_reload.py b/litellm/proxy/dev_reload.py index f60aa9a93f8..cf97bc2ff08 100644 --- a/litellm/proxy/dev_reload.py +++ b/litellm/proxy/dev_reload.py @@ -1,17 +1,19 @@ -from collections.abc import Sequence +import logging +import sys +from collections.abc import Callable, Sequence from functools import partial from inspect import signature from socket import socket from typing import Final, Literal -from pydantic import ConfigDict, JsonValue, TypeAdapter, with_config +from pydantic import ConfigDict, ImportString, JsonValue, TypeAdapter, with_config from typing_extensions import ReadOnly, Required, TypedDict from uvicorn import Config, Server +from uvicorn import config as uvicorn_config from uvicorn.supervisors import ChangeReload, Multiprocess -class WorkerOptions(TypedDict, total=False): - app: Required[ReadOnly[str]] +class ServerOptions(TypedDict, total=False): host: ReadOnly[str] port: ReadOnly[int] server_header: ReadOnly[bool] @@ -28,14 +30,38 @@ class WorkerOptions(TypedDict, total=False): @with_config(ConfigDict(extra="forbid")) -class ReloadOptions(WorkerOptions): +class ReloadOptions(ServerOptions): + app: Required[ReadOnly[str]] reload: ReadOnly[bool] reload_dirs: ReadOnly[tuple[str, ...]] reload_includes: ReadOnly[tuple[str, ...]] RELOAD_OPTIONS: Final = TypeAdapter(ReloadOptions) -WORKER_OPTIONS: Final = TypeAdapter(WorkerOptions) +SERVER_OPTIONS: Final = TypeAdapter(ServerOptions) +ASGI_APP: Final = TypeAdapter(ImportString[Callable[..., object]]) +LOGGER: Final = logging.getLogger("uvicorn.error") + + +def multi_worker_reload_supported() -> bool: + return hasattr(uvicorn_config, "STARTUP_FAILURE") + + +def load_app_or_exit(import_path: str) -> Callable[..., object]: + try: + return ASGI_APP.validate_python(import_path) + except Exception: + LOGGER.exception("Error loading ASGI app %s", import_path) + sys.exit(uvicorn_config.STARTUP_FAILURE) + + +def pool_config(options: ReloadOptions, workers: int) -> Config: + return Config( + partial(load_app_or_exit, options["app"]), + factory=True, + workers=workers, + **SERVER_OPTIONS.validate_python(options), + ) def _run_worker_pool(config: Config, sockets: Sequence[socket] | None) -> None: @@ -53,14 +79,14 @@ def _run_worker_pool(config: Config, sockets: Sequence[socket] | None) -> None: def run_multi_worker_reload(options: ReloadOptions, workers: int) -> None: - worker_options: Final = WORKER_OPTIONS.validate_python(options) - worker_config: Final = Config(**worker_options, workers=workers) + worker_config: Final = pool_config(options, workers) reload_config: Final = Config( - **worker_options, + options["app"], workers=1, reload=options["reload"], reload_dirs=list(options["reload_dirs"]), reload_includes=list(options["reload_includes"]), + **SERVER_OPTIONS.validate_python(options), ) shared_socket: Final = worker_config.bind_socket() try: diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 035f2021cb0..59a4d00622a 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -1033,7 +1033,7 @@ class ProxyInitializationHelpers: "--reload", is_flag=True, default=False, - help="Enable uvicorn hot reload (dev only), including all --num_workers workers. Also reloads when the --config YAML or .env changes. Incompatible with --run_gunicorn, --run_hypercorn, and --run_granian.", + help="Enable uvicorn hot reload (dev only), including every --num_workers worker on uvicorn>=0.50.0. Also reloads when the --config YAML or .env changes. Incompatible with --run_gunicorn, --run_hypercorn, and --run_granian.", ) @click.option( "--prometheus_metrics_port", @@ -1597,12 +1597,22 @@ def run_server( if num_workers > 1: start_query_engine_reaper() if reload and num_workers > 1: - from litellm.proxy.dev_reload import RELOAD_OPTIONS, run_multi_worker_reload - - run_multi_worker_reload( - RELOAD_OPTIONS.validate_python(uvicorn_args), workers=TypeAdapter(int).validate_python(num_workers) + from litellm.proxy.dev_reload import ( + RELOAD_OPTIONS, + multi_worker_reload_supported, + run_multi_worker_reload, + ) + + if multi_worker_reload_supported(): + run_multi_worker_reload( + RELOAD_OPTIONS.validate_python(uvicorn_args), + workers=TypeAdapter(int).validate_python(num_workers), + ) + return + print( + "\033[1;33mLiteLLM Proxy: --reload with --num_workers>1 requires uvicorn>=0.50.0, " + "so only a single worker reloads.\033[0m" ) - return uvicorn.run( **uvicorn_args, workers=num_workers, diff --git a/tests/unit/proxy/test_dev_reload.py b/tests/unit/proxy/test_dev_reload.py new file mode 100644 index 00000000000..017737859c9 --- /dev/null +++ b/tests/unit/proxy/test_dev_reload.py @@ -0,0 +1,45 @@ +import importlib +from pathlib import Path +from typing import Final + +import pytest +from uvicorn import config as uvicorn_config + +from litellm.proxy.dev_reload import ReloadOptions, load_app_or_exit, pool_config + +UNCHANGED_LOGGING: Final = {"version": 1, "incremental": True} + + +def _options(app: str) -> ReloadOptions: + return { + "app": app, + "log_config": UNCHANGED_LOGGING, + "reload": True, + "reload_dirs": (), + "reload_includes": ("*.py",), + } + + +@pytest.mark.parametrize( + ("module", "source"), + ( + ("reload_syntax_error_app", "def app(\n"), + ("reload_name_error_app", "app = undefined_name\n"), + ("reload_runtime_error_app", "raise RuntimeError('broken edit')\n"), + ("reload_missing_attribute_app", "value = 1\n"), + ), +) +def test_a_broken_app_exits_the_worker_as_a_startup_failure( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, module: str, source: str +) -> None: + (tmp_path / f"{module}.py").write_text(source) + monkeypatch.syspath_prepend(str(tmp_path)) + with pytest.raises(SystemExit) as exit_info: + pool_config(_options(f"{module}:app"), workers=2).load() + assert exit_info.value.code == uvicorn_config.STARTUP_FAILURE + + +def test_a_working_app_loads_as_the_module_defines_it(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + (tmp_path / "reload_working_app.py").write_text("async def app(scope, receive, send):\n return None\n") + monkeypatch.syspath_prepend(str(tmp_path)) + assert load_app_or_exit("reload_working_app:app") is importlib.import_module("reload_working_app").app From 687730c5d0c31901a81338e1f933f268537f7703 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 13 Oct 2026 11:21:22 -0700 Subject: [PATCH 03/13] test(integration): stop the reload test's proxy through the harness shutdown helper Main's TQ010 rule rejects a raw .terminate() under tests/integration, so the multi-worker reload test now calls OwnedProxy.begin_shutdown(), which sends the same SIGTERM --- tests/integration/configuration/test_multi_worker_reload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/configuration/test_multi_worker_reload.py b/tests/integration/configuration/test_multi_worker_reload.py index 64f6521a13e..3df262a7cd7 100644 --- a/tests/integration/configuration/test_multi_worker_reload.py +++ b/tests/integration/configuration/test_multi_worker_reload.py @@ -84,7 +84,7 @@ def test_reload_replaces_every_worker_and_shutdown_leaves_none( after_env: Final = _assert_replaced(owned, after_source, workers) response: Final = owned.gateway.request("GET", "/health/readiness") assert response.status_code == 200, response.text - owned.process.terminate() + owned.begin_shutdown() owned.process.wait(timeout=graceful_stop_seconds()) assert all(not psutil.pid_exists(pid) for pid in after_env), owned.log.read_text() assert owned.process.returncode == 0, owned.log.read_text() From 0fc06accb1cbc2362d001db81e5075b12f995fa3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 13 Oct 2026 13:26:43 -0700 Subject: [PATCH 04/13] test(proxy): cover the reload worker pool end to end and edit the config the proxy loads --- .../configuration/test_multi_worker_reload.py | 7 +- tests/unit/proxy/test_dev_reload.py | 121 +++++++++++++++++- 2 files changed, 126 insertions(+), 2 deletions(-) diff --git a/tests/integration/configuration/test_multi_worker_reload.py b/tests/integration/configuration/test_multi_worker_reload.py index 3df262a7cd7..280f760f7e1 100644 --- a/tests/integration/configuration/test_multi_worker_reload.py +++ b/tests/integration/configuration/test_multi_worker_reload.py @@ -17,6 +17,11 @@ def _worker_pids(owned: OwnedProxy) -> frozenset[int]: return frozenset(int(match.group(1)) for match in STARTED_WORKER.finditer(owned.log.read_text())) +def _launched_config(owned: OwnedProxy) -> Path: + arguments: Final = psutil.Process(owned.process.pid).cmdline() + return Path(arguments[arguments.index("--config") + 1]) + + def _config(model: str) -> str: return json.dumps( { @@ -74,7 +79,7 @@ def test_reload_replaces_every_worker_and_shutdown_leaves_none( initial: Final = eventually( lambda: _worker_pids(owned), lambda pids: len(pids) == workers, seconds=graceful_stop_seconds() ) - config.write_text(_config("after-reload")) + _launched_config(owned).write_text(_config("after-reload")) after_config: Final = _assert_replaced(owned, initial, workers) for _ in range(2 * workers): _assert_models(owned.gateway) diff --git a/tests/unit/proxy/test_dev_reload.py b/tests/unit/proxy/test_dev_reload.py index 017737859c9..f6baf11bd3b 100644 --- a/tests/unit/proxy/test_dev_reload.py +++ b/tests/unit/proxy/test_dev_reload.py @@ -1,13 +1,58 @@ +import contextlib import importlib +import os +import re +import signal +import socket +import subprocess +import sys +import time +from collections.abc import Callable from pathlib import Path from typing import Final +import httpx +import psutil import pytest from uvicorn import config as uvicorn_config -from litellm.proxy.dev_reload import ReloadOptions, load_app_or_exit, pool_config +from litellm.proxy.dev_reload import ReloadOptions, load_app_or_exit, multi_worker_reload_supported, pool_config UNCHANGED_LOGGING: Final = {"version": 1, "incremental": True} +POOL_WORKERS: Final = 2 +POOL_TIMEOUT_SECONDS: Final = 120 +WORKER_HEALTHCHECK_SECONDS: Final = 30 +WORKER_STARTED: Final = re.compile(r"Started server process \[(\d+)\]") +POOL_STOPPED: Final = "failed to start, stopping the parent process" +APP_LOAD_ERROR: Final = "Error loading ASGI app" +PID_APP: Final = """import os + + +async def app(scope, receive, send): + if scope["type"] != "http": + return + await send({"type": "http.response.start", "status": 200, "headers": []}) + await send({"type": "http.response.body", "body": str(os.getpid()).encode()}) +""" +POOL_RUNNER: Final = """import sys + +sys.path.insert(0, sys.argv[1]) + +from litellm.proxy.dev_reload import run_multi_worker_reload + +run_multi_worker_reload( + { + "app": "reload_pool_app:app", + "host": "127.0.0.1", + "port": int(sys.argv[2]), + "timeout_worker_healthcheck": int(sys.argv[3]), + "reload": True, + "reload_dirs": (sys.argv[1],), + "reload_includes": ("*.py",), + }, + workers=int(sys.argv[4]), +) +""" def _options(app: str) -> ReloadOptions: @@ -43,3 +88,77 @@ def test_a_working_app_loads_as_the_module_defines_it(tmp_path: Path, monkeypatc (tmp_path / "reload_working_app.py").write_text("async def app(scope, receive, send):\n return None\n") monkeypatch.syspath_prepend(str(tmp_path)) assert load_app_or_exit("reload_working_app:app") is importlib.import_module("reload_working_app").app + + +def _free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +def _started_workers(log_text: str) -> frozenset[int]: + return frozenset(int(match.group(1)) for match in WORKER_STARTED.finditer(log_text)) + + +def _wait_for_log(log: Path, process: subprocess.Popen[bytes], seen: Callable[[str], bool]) -> str: + deadline: Final = time.monotonic() + POOL_TIMEOUT_SECONDS + while time.monotonic() < deadline: + log_text: Final = log.read_text() + if seen(log_text): + return log_text + assert process.poll() is None, log_text + time.sleep(0.2) + raise AssertionError(log.read_text()) + + +def _next_pool(log: Path, process: subprocess.Popen[bytes], port: int, previous: frozenset[int]) -> frozenset[int]: + log_text: Final = _wait_for_log(log, process, lambda text: len(_started_workers(text) - previous) == POOL_WORKERS) + pool: Final = _started_workers(log_text) - previous + assert not any(psutil.pid_exists(pid) for pid in previous), log_text + served: Final = httpx.get(f"http://127.0.0.1:{port}/", timeout=POOL_TIMEOUT_SECONDS) + assert int(served.text) in pool, log_text + return pool + + +@pytest.mark.skipif( + not multi_worker_reload_supported(), reason="uvicorn<0.50.0 cannot stop a worker pool on a startup failure" +) +@pytest.mark.timeout(5 * POOL_TIMEOUT_SECONDS) +def test_a_save_replaces_every_worker_and_a_broken_save_stops_the_pool_until_the_next_good_one( + tmp_path: Path, +) -> None: + app: Final = tmp_path / "reload_pool_app.py" + app.write_text(PID_APP) + log: Final = tmp_path / "reload.log" + port: Final = _free_port() + with log.open("wb") as output: + process: Final = subprocess.Popen( + ( + sys.executable, + "-I", + "-c", + POOL_RUNNER, + str(tmp_path), + str(port), + str(WORKER_HEALTHCHECK_SECONDS), + str(POOL_WORKERS), + ), + stdout=output, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + try: + first_pool: Final = _next_pool(log, process, port, frozenset()) + app.write_text("async def app(\n") + _wait_for_log(log, process, lambda text: POOL_STOPPED in text) + assert not any(psutil.pid_exists(pid) for pid in first_pool), log.read_text() + app.write_text(PID_APP) + last_pool: Final = _next_pool(log, process, port, first_pool) + process.send_signal(signal.SIGINT) + assert process.wait(timeout=POOL_TIMEOUT_SECONDS) == 0, log.read_text() + assert not any(psutil.pid_exists(pid) for pid in last_pool), log.read_text() + assert 1 <= log.read_text().count(APP_LOAD_ERROR) <= POOL_WORKERS, log.read_text() + finally: + with contextlib.suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + process.wait() From c43fff79ab9c6c87d46ce1349462be184c8c4307 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 13 Oct 2026 13:26:44 -0700 Subject: [PATCH 05/13] docs(agents): scope the two-worker reload QA rule to uvicorn proxies that use Redis --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index c1f9791ae95..28033af1327 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,7 +39,7 @@ Same applies for filing bug reports and feature requests, with .github/ISSUE_TEM If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just drop the section -Run every QA proxy with `--num_workers 2 --reload` and a dedicated Redis instance configured through `router_settings.redis_host`, `redis_port`, and credentials when required. `litellm/proxy/dev_config.yaml` sets no Redis, so QA from a copy of it that adds them. Verify two application worker PIDs are serving, then edit a watched Python, config YAML, or `.env` file and verify both workers are replaced and requests still succeed. Use the same worker count and Redis topology for before and after proof +Run QA proxies the way the command below does, with `--num_workers 2 --reload`, and add a dedicated Redis instance through `router_settings.redis_host`, `redis_port`, and credentials when required. `litellm/proxy/dev_config.yaml` sets no Redis, so pass `--config` a copy of it that adds them. Check that two worker PIDs are serving, then edit a watched Python, config YAML, or `.env` file and check that both workers are replaced and requests still succeed. Use the same worker count and Redis setup for the before and after proof. When the PR is about `--run_gunicorn`, `--run_hypercorn`, or `--run_granian`, which `--reload` does not support, drop `--reload` and the reload check. When it is about behavior without Redis, drop Redis. Say in the proof which part you dropped and why Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --num_workers 2 --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`. The Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into. I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, or the main use case runs through a headful agentic coding tool like Claude Code or Codex, drive that surface yourself and embed your own before and after screenshots of it in the PR (the Admin UI page, or what the coding tool shows), next to an ordered list of the URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, and what fields to fill out so a reviewer can reproduce it From 0a2b8da1bf9a9c6d91ea106713b066a5d54d35fc Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:07:13 -0700 Subject: [PATCH 06/13] fix(proxy): keep the original traceback when a reloaded worker fails to import the app --- litellm/proxy/dev_reload.py | 3 +++ tests/unit/proxy/test_dev_reload.py | 30 +++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/litellm/proxy/dev_reload.py b/litellm/proxy/dev_reload.py index cf97bc2ff08..693ba44578a 100644 --- a/litellm/proxy/dev_reload.py +++ b/litellm/proxy/dev_reload.py @@ -1,3 +1,4 @@ +import importlib import logging import sys from collections.abc import Callable, Sequence @@ -48,7 +49,9 @@ def multi_worker_reload_supported() -> bool: def load_app_or_exit(import_path: str) -> Callable[..., object]: + module_name: Final = import_path.partition(":")[0] try: + importlib.import_module(module_name) return ASGI_APP.validate_python(import_path) except Exception: LOGGER.exception("Error loading ASGI app %s", import_path) diff --git a/tests/unit/proxy/test_dev_reload.py b/tests/unit/proxy/test_dev_reload.py index f6baf11bd3b..5ebf7c152d9 100644 --- a/tests/unit/proxy/test_dev_reload.py +++ b/tests/unit/proxy/test_dev_reload.py @@ -1,5 +1,7 @@ import contextlib import importlib +import logging +import logging.handlers import os import re import signal @@ -7,6 +9,7 @@ import socket import subprocess import sys import time +import traceback from collections.abc import Callable from pathlib import Path from typing import Final @@ -84,6 +87,33 @@ def test_a_broken_app_exits_the_worker_as_a_startup_failure( assert exit_info.value.code == uvicorn_config.STARTUP_FAILURE +@pytest.mark.parametrize( + ("module", "source"), + ( + ("reload_import_error_app", "import os\nfrom os import missing_name\n"), + ("reload_value_error_app", "import os\nraise ValueError('broken edit')\n"), + ), +) +def test_a_broken_import_logs_the_line_that_raised( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, module: str, source: str +) -> None: + app_file: Final = tmp_path / f"{module}.py" + app_file.write_text(source) + monkeypatch.syspath_prepend(str(tmp_path)) + uvicorn_logger: Final = logging.getLogger("uvicorn.error") + captured: Final = logging.handlers.BufferingHandler(capacity=POOL_WORKERS) + uvicorn_logger.addHandler(captured) + try: + with pytest.raises(SystemExit): + load_app_or_exit(f"{module}:app") + finally: + uvicorn_logger.removeHandler(captured) + (record,) = captured.buffer + assert record.exc_info is not None + raised_at: Final = frozenset((frame.filename, frame.lineno) for frame in traceback.extract_tb(record.exc_info[2])) + assert (str(app_file), 2) in raised_at, record.exc_text + + def test_a_working_app_loads_as_the_module_defines_it(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: (tmp_path / "reload_working_app.py").write_text("async def app(scope, receive, send):\n return None\n") monkeypatch.syspath_prepend(str(tmp_path)) From a2ba53ae839353553c90715fc62fe83b7f72abce Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:12:21 -0700 Subject: [PATCH 07/13] docs(agents): point both router and coordination Redis at the QA Redis and cover single-worker trees --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 28033af1327..4a884355052 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,7 +39,7 @@ Same applies for filing bug reports and feature requests, with .github/ISSUE_TEM If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just drop the section -Run QA proxies the way the command below does, with `--num_workers 2 --reload`, and add a dedicated Redis instance through `router_settings.redis_host`, `redis_port`, and credentials when required. `litellm/proxy/dev_config.yaml` sets no Redis, so pass `--config` a copy of it that adds them. Check that two worker PIDs are serving, then edit a watched Python, config YAML, or `.env` file and check that both workers are replaced and requests still succeed. Use the same worker count and Redis setup for the before and after proof. When the PR is about `--run_gunicorn`, `--run_hypercorn`, or `--run_granian`, which `--reload` does not support, drop `--reload` and the reload check. When it is about behavior without Redis, drop Redis. Say in the proof which part you dropped and why +Run QA proxies the way the command below does, with `--num_workers 2 --reload`, and point both `router_settings` (`redis_host`, `redis_port`) and `general_settings.coordination_redis` (`host`, `port`) at one dedicated Redis instance, with credentials when required, so the workers share router state and spend, budget, and rate-limit counters. `litellm/proxy/dev_config.yaml` sets no Redis, so pass `--config` a copy of it that adds them. Check that two worker PIDs are serving, then edit a watched Python, config YAML, or `.env` file and check that both workers are replaced and requests still succeed. Use the same worker count and Redis setup for the before and after proof. When the PR is about `--run_gunicorn`, `--run_hypercorn`, or `--run_granian`, which `--reload` does not support, drop `--reload` and the reload check. When it is about behavior without Redis, drop Redis. When the proxy prints `"workers" flag is ignored when reloading is enabled`, that checkout or its uvicorn reloads only one worker, so drop the two-worker checks for it. Say in the proof which part you dropped and why Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --num_workers 2 --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`. The Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into. I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, or the main use case runs through a headful agentic coding tool like Claude Code or Codex, drive that surface yourself and embed your own before and after screenshots of it in the PR (the Admin UI page, or what the coding tool shows), next to an ordered list of the URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, and what fields to fill out so a reviewer can reproduce it From 82c0ed923e6f4d55925f33a125d5b3b7431f709b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 13 Oct 2026 16:47:37 -0700 Subject: [PATCH 08/13] fix(proxy): stop the reload worker pool when the reloader exits and give reloads a longer worker healthcheck Closing the terminal or pressing Ctrl-\ killed the reloader and left the pool parent holding the port while it respawned workers that could not start. The pool now stops once its parent reloader is gone. The reload pool also defaults the worker healthcheck to 30 seconds unless --timeout_worker_healthcheck is set, since uvicorn's 5 second default killed workers that were still booting on a reload. --- litellm/proxy/dev_reload.py | 14 ++++- tests/unit/proxy/test_dev_reload.py | 93 +++++++++++++++++++++++------ 2 files changed, 89 insertions(+), 18 deletions(-) diff --git a/litellm/proxy/dev_reload.py b/litellm/proxy/dev_reload.py index 693ba44578a..a29319b8542 100644 --- a/litellm/proxy/dev_reload.py +++ b/litellm/proxy/dev_reload.py @@ -1,6 +1,8 @@ import importlib import logging +import os import sys +import threading from collections.abc import Callable, Sequence from functools import partial from inspect import signature @@ -42,6 +44,8 @@ RELOAD_OPTIONS: Final = TypeAdapter(ReloadOptions) SERVER_OPTIONS: Final = TypeAdapter(ServerOptions) ASGI_APP: Final = TypeAdapter(ImportString[Callable[..., object]]) LOGGER: Final = logging.getLogger("uvicorn.error") +RELOAD_WORKER_HEALTHCHECK_SECONDS: Final = 30 +RELOADER_CHECK_SECONDS: Final = 0.5 def multi_worker_reload_supported() -> bool: @@ -63,10 +67,17 @@ def pool_config(options: ReloadOptions, workers: int) -> Config: partial(load_app_or_exit, options["app"]), factory=True, workers=workers, - **SERVER_OPTIONS.validate_python(options), + **SERVER_OPTIONS.validate_python({"timeout_worker_healthcheck": RELOAD_WORKER_HEALTHCHECK_SECONDS, **options}), ) +def stop_pool_when_reloader_exits(pool: Multiprocess, reloader_pid: int) -> None: + while not pool.should_exit.wait(RELOADER_CHECK_SECONDS): + if os.getppid() != reloader_pid: + LOGGER.info("Reloader process [%d] exited, stopping the worker pool.", reloader_pid) + pool.should_exit.set() + + def _run_worker_pool(config: Config, sockets: Sequence[socket] | None) -> None: shared_sockets: Final = list(sockets or ()) pool: Final = ( @@ -78,6 +89,7 @@ def _run_worker_pool(config: Config, sockets: Sequence[socket] | None) -> None: if "target" in signature(Multiprocess).parameters else Multiprocess(config=config, sockets=shared_sockets) ) + threading.Thread(target=stop_pool_when_reloader_exits, args=(pool, os.getppid()), daemon=True).start() pool.run() diff --git a/tests/unit/proxy/test_dev_reload.py b/tests/unit/proxy/test_dev_reload.py index 5ebf7c152d9..5f7b4093855 100644 --- a/tests/unit/proxy/test_dev_reload.py +++ b/tests/unit/proxy/test_dev_reload.py @@ -19,7 +19,13 @@ import psutil import pytest from uvicorn import config as uvicorn_config -from litellm.proxy.dev_reload import ReloadOptions, load_app_or_exit, multi_worker_reload_supported, pool_config +from litellm.proxy.dev_reload import ( + RELOAD_WORKER_HEALTHCHECK_SECONDS, + ReloadOptions, + load_app_or_exit, + multi_worker_reload_supported, + pool_config, +) UNCHANGED_LOGGING: Final = {"version": 1, "incremental": True} POOL_WORKERS: Final = 2 @@ -114,6 +120,15 @@ def test_a_broken_import_logs_the_line_that_raised( assert (str(app_file), 2) in raised_at, record.exc_text +def test_the_reload_pool_defaults_the_worker_healthcheck_and_keeps_an_explicit_one() -> None: + explicit: Final = RELOAD_WORKER_HEALTHCHECK_SECONDS + 7 + assert pool_config(_options("app:app"), workers=2).timeout_worker_healthcheck == RELOAD_WORKER_HEALTHCHECK_SECONDS + assert ( + pool_config({**_options("app:app"), "timeout_worker_healthcheck": explicit}, workers=2).timeout_worker_healthcheck + == explicit + ) + + def test_a_working_app_loads_as_the_module_defines_it(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: (tmp_path / "reload_working_app.py").write_text("async def app(scope, receive, send):\n return None\n") monkeypatch.syspath_prepend(str(tmp_path)) @@ -141,6 +156,30 @@ def _wait_for_log(log: Path, process: subprocess.Popen[bytes], seen: Callable[[s raise AssertionError(log.read_text()) +def _start_pool(app_dir: Path, port: int, log: Path) -> subprocess.Popen[bytes]: + with log.open("wb") as output: + return subprocess.Popen( + ( + sys.executable, + "-I", + "-c", + POOL_RUNNER, + str(app_dir), + str(port), + str(WORKER_HEALTHCHECK_SECONDS), + str(POOL_WORKERS), + ), + stdout=output, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + + +def _port_refuses_connections(port: int) -> bool: + with socket.socket() as sock: + return sock.connect_ex(("127.0.0.1", port)) != 0 + + def _next_pool(log: Path, process: subprocess.Popen[bytes], port: int, previous: frozenset[int]) -> frozenset[int]: log_text: Final = _wait_for_log(log, process, lambda text: len(_started_workers(text) - previous) == POOL_WORKERS) pool: Final = _started_workers(log_text) - previous @@ -161,22 +200,7 @@ def test_a_save_replaces_every_worker_and_a_broken_save_stops_the_pool_until_the app.write_text(PID_APP) log: Final = tmp_path / "reload.log" port: Final = _free_port() - with log.open("wb") as output: - process: Final = subprocess.Popen( - ( - sys.executable, - "-I", - "-c", - POOL_RUNNER, - str(tmp_path), - str(port), - str(WORKER_HEALTHCHECK_SECONDS), - str(POOL_WORKERS), - ), - stdout=output, - stderr=subprocess.STDOUT, - start_new_session=True, - ) + process: Final = _start_pool(tmp_path, port, log) try: first_pool: Final = _next_pool(log, process, port, frozenset()) app.write_text("async def app(\n") @@ -192,3 +216,38 @@ def test_a_save_replaces_every_worker_and_a_broken_save_stops_the_pool_until_the with contextlib.suppress(ProcessLookupError): os.killpg(process.pid, signal.SIGKILL) process.wait() + + +@pytest.mark.skipif( + not multi_worker_reload_supported(), reason="uvicorn<0.50.0 cannot stop a worker pool on a startup failure" +) +@pytest.mark.timeout(3 * POOL_TIMEOUT_SECONDS) +@pytest.mark.parametrize( + "kill_reloader", + ( + pytest.param(lambda reloader: os.kill(reloader, signal.SIGKILL), id="sigkill-reloader"), + pytest.param(lambda reloader: os.killpg(reloader, signal.SIGHUP), id="terminal-closed"), + pytest.param(lambda reloader: os.killpg(reloader, signal.SIGQUIT), id="ctrl-backslash"), + ), +) +def test_the_pool_stops_and_frees_the_port_when_the_reloader_dies( + tmp_path: Path, kill_reloader: Callable[[int], None] +) -> None: + (tmp_path / "reload_pool_app.py").write_text(PID_APP) + log: Final = tmp_path / "reload.log" + port: Final = _free_port() + process: Final = _start_pool(tmp_path, port, log) + try: + workers: Final = _next_pool(log, process, port, frozenset()) + pool_parent: Final = psutil.Process(min(workers)).ppid() + assert psutil.Process(pool_parent).ppid() == process.pid, log.read_text() + kill_reloader(process.pid) + process.wait(timeout=POOL_TIMEOUT_SECONDS) + deadline: Final = time.monotonic() + POOL_TIMEOUT_SECONDS + while psutil.pid_exists(pool_parent) or not _port_refuses_connections(port): + assert time.monotonic() < deadline, log.read_text() + time.sleep(0.2) + finally: + with contextlib.suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + process.wait() From afe2687a5f59174a964cb68399f898daf3f6df01 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:33:08 -0700 Subject: [PATCH 09/13] fix(proxy): watch the reloader through its process sentinel so the pool also stops on Windows --- litellm/proxy/dev_reload.py | 17 +++++++++-------- tests/unit/proxy/test_dev_reload.py | 17 +++++++++++++++++ 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/dev_reload.py b/litellm/proxy/dev_reload.py index a29319b8542..2ccd8e9ec45 100644 --- a/litellm/proxy/dev_reload.py +++ b/litellm/proxy/dev_reload.py @@ -1,11 +1,12 @@ import importlib import logging -import os +import multiprocessing import sys import threading from collections.abc import Callable, Sequence from functools import partial from inspect import signature +from multiprocessing.process import BaseProcess from socket import socket from typing import Final, Literal @@ -45,7 +46,6 @@ SERVER_OPTIONS: Final = TypeAdapter(ServerOptions) ASGI_APP: Final = TypeAdapter(ImportString[Callable[..., object]]) LOGGER: Final = logging.getLogger("uvicorn.error") RELOAD_WORKER_HEALTHCHECK_SECONDS: Final = 30 -RELOADER_CHECK_SECONDS: Final = 0.5 def multi_worker_reload_supported() -> bool: @@ -71,11 +71,10 @@ def pool_config(options: ReloadOptions, workers: int) -> Config: ) -def stop_pool_when_reloader_exits(pool: Multiprocess, reloader_pid: int) -> None: - while not pool.should_exit.wait(RELOADER_CHECK_SECONDS): - if os.getppid() != reloader_pid: - LOGGER.info("Reloader process [%d] exited, stopping the worker pool.", reloader_pid) - pool.should_exit.set() +def stop_pool_when_reloader_exits(reloader: BaseProcess, pool_should_exit: threading.Event) -> None: + reloader.join() + LOGGER.info("Reloader process [%s] exited, stopping the worker pool.", reloader.pid) + pool_should_exit.set() def _run_worker_pool(config: Config, sockets: Sequence[socket] | None) -> None: @@ -89,7 +88,9 @@ def _run_worker_pool(config: Config, sockets: Sequence[socket] | None) -> None: if "target" in signature(Multiprocess).parameters else Multiprocess(config=config, sockets=shared_sockets) ) - threading.Thread(target=stop_pool_when_reloader_exits, args=(pool, os.getppid()), daemon=True).start() + reloader: Final = multiprocessing.parent_process() + if reloader is not None: + threading.Thread(target=stop_pool_when_reloader_exits, args=(reloader, pool.should_exit), daemon=True).start() pool.run() diff --git a/tests/unit/proxy/test_dev_reload.py b/tests/unit/proxy/test_dev_reload.py index 5f7b4093855..fd6926c153f 100644 --- a/tests/unit/proxy/test_dev_reload.py +++ b/tests/unit/proxy/test_dev_reload.py @@ -2,12 +2,14 @@ import contextlib import importlib import logging import logging.handlers +import multiprocessing import os import re import signal import socket import subprocess import sys +import threading import time import traceback from collections.abc import Callable @@ -25,6 +27,7 @@ from litellm.proxy.dev_reload import ( load_app_or_exit, multi_worker_reload_supported, pool_config, + stop_pool_when_reloader_exits, ) UNCHANGED_LOGGING: Final = {"version": 1, "incremental": True} @@ -129,6 +132,20 @@ def test_the_reload_pool_defaults_the_worker_healthcheck_and_keeps_an_explicit_o ) +def test_the_pool_is_told_to_stop_once_the_reloader_process_exits() -> None: + reloader: Final = multiprocessing.get_context("spawn").Process(target=time.sleep, args=(POOL_TIMEOUT_SECONDS,)) + reloader.start() + pool_should_exit: Final = threading.Event() + threading.Thread(target=stop_pool_when_reloader_exits, args=(reloader, pool_should_exit), daemon=True).start() + try: + assert not pool_should_exit.wait(0.5) + reloader.kill() + assert pool_should_exit.wait(POOL_TIMEOUT_SECONDS) + finally: + reloader.kill() + reloader.join() + + def test_a_working_app_loads_as_the_module_defines_it(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: (tmp_path / "reload_working_app.py").write_text("async def app(scope, receive, send):\n return None\n") monkeypatch.syspath_prepend(str(tmp_path)) From a02c0c75829ca2dceadbbf6ed278b49fcd153000 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:33:38 -0700 Subject: [PATCH 10/13] fix(proxy): carry the graceful shutdown timeout into the reload worker pool --- litellm/proxy/dev_reload.py | 5 +++-- tests/unit/proxy/test_dev_reload.py | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/dev_reload.py b/litellm/proxy/dev_reload.py index 2ccd8e9ec45..f2494bd41bd 100644 --- a/litellm/proxy/dev_reload.py +++ b/litellm/proxy/dev_reload.py @@ -21,6 +21,7 @@ class ServerOptions(TypedDict, total=False): host: ReadOnly[str] port: ReadOnly[int] server_header: ReadOnly[bool] + timeout_graceful_shutdown: ReadOnly[float] log_config: ReadOnly[str | dict[str, JsonValue]] # mutable-ok: uvicorn.Config requires a concrete logging dict log_level: ReadOnly[str | int] timeout_keep_alive: ReadOnly[int] @@ -67,7 +68,7 @@ def pool_config(options: ReloadOptions, workers: int) -> Config: partial(load_app_or_exit, options["app"]), factory=True, workers=workers, - **SERVER_OPTIONS.validate_python({"timeout_worker_healthcheck": RELOAD_WORKER_HEALTHCHECK_SECONDS, **options}), + **SERVER_OPTIONS.validate_python({"timeout_worker_healthcheck": RELOAD_WORKER_HEALTHCHECK_SECONDS, **options}), # pyright: ignore[reportArgumentType] # uvicorn types the float graceful shutdown timeout as int ) @@ -102,7 +103,7 @@ def run_multi_worker_reload(options: ReloadOptions, workers: int) -> None: reload=options["reload"], reload_dirs=list(options["reload_dirs"]), reload_includes=list(options["reload_includes"]), - **SERVER_OPTIONS.validate_python(options), + **SERVER_OPTIONS.validate_python(options), # pyright: ignore[reportArgumentType] # uvicorn types the float graceful shutdown timeout as int ) shared_socket: Final = worker_config.bind_socket() try: diff --git a/tests/unit/proxy/test_dev_reload.py b/tests/unit/proxy/test_dev_reload.py index fd6926c153f..39f9de17101 100644 --- a/tests/unit/proxy/test_dev_reload.py +++ b/tests/unit/proxy/test_dev_reload.py @@ -22,6 +22,7 @@ import pytest from uvicorn import config as uvicorn_config from litellm.proxy.dev_reload import ( + RELOAD_OPTIONS, RELOAD_WORKER_HEALTHCHECK_SECONDS, ReloadOptions, load_app_or_exit, @@ -29,6 +30,7 @@ from litellm.proxy.dev_reload import ( pool_config, stop_pool_when_reloader_exits, ) +from litellm.proxy.proxy_cli import ProxyInitializationHelpers UNCHANGED_LOGGING: Final = {"version": 1, "incremental": True} POOL_WORKERS: Final = 2 @@ -132,6 +134,24 @@ def test_the_reload_pool_defaults_the_worker_healthcheck_and_keeps_an_explicit_o ) +def test_the_reload_pool_accepts_and_keeps_every_uvicorn_arg_the_proxy_cli_builds(tmp_path: Path) -> None: + cli_args: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args( + "127.0.0.1", 4123, keepalive_timeout=61, timeout_worker_healthcheck=RELOAD_WORKER_HEALTHCHECK_SECONDS + 7 + ) + options: Final = RELOAD_OPTIONS.validate_python( + { + **cli_args, + "log_config": UNCHANGED_LOGGING, + "reload": True, + "reload_dirs": [str(tmp_path)], + "reload_includes": ["*.py"], + } + ) + config: Final = pool_config(options, workers=POOL_WORKERS) + server_args: Final = {key: value for key, value in cli_args.items() if key not in ("app", "log_config")} + assert {key: getattr(config, key) for key in server_args} == server_args + + def test_the_pool_is_told_to_stop_once_the_reloader_process_exits() -> None: reloader: Final = multiprocessing.get_context("spawn").Process(target=time.sleep, args=(POOL_TIMEOUT_SECONDS,)) reloader.start() From 3bd5c1460556f970308e7ef65caa3860e71b2df6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:47:28 -0700 Subject: [PATCH 11/13] fix(proxy): stop the reload worker pool on Ctrl-\ instead of waiting out a dying worker's healthcheck --- litellm/proxy/dev_reload.py | 10 +++++-- tests/unit/proxy/test_dev_reload.py | 43 +++++++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/dev_reload.py b/litellm/proxy/dev_reload.py index f2494bd41bd..cc8bf2beb7b 100644 --- a/litellm/proxy/dev_reload.py +++ b/litellm/proxy/dev_reload.py @@ -78,16 +78,22 @@ def stop_pool_when_reloader_exits(reloader: BaseProcess, pool_should_exit: threa pool_should_exit.set() +class ReloadWorkerPool(Multiprocess): + def handle_quit(self) -> None: + LOGGER.info("Received SIGQUIT, exiting.") + self.should_exit.set() + + def _run_worker_pool(config: Config, sockets: Sequence[socket] | None) -> None: shared_sockets: Final = list(sockets or ()) pool: Final = ( - Multiprocess( + ReloadWorkerPool( config=config, target=Server(config=config).run, # pyright: ignore[reportCallIssue] # Older uvicorn versions require target sockets=shared_sockets, ) if "target" in signature(Multiprocess).parameters - else Multiprocess(config=config, sockets=shared_sockets) + else ReloadWorkerPool(config=config, sockets=shared_sockets) ) reloader: Final = multiprocessing.parent_process() if reloader is not None: diff --git a/tests/unit/proxy/test_dev_reload.py b/tests/unit/proxy/test_dev_reload.py index 39f9de17101..3387e4776a6 100644 --- a/tests/unit/proxy/test_dev_reload.py +++ b/tests/unit/proxy/test_dev_reload.py @@ -48,6 +48,20 @@ async def app(scope, receive, send): await send({"type": "http.response.start", "status": 200, "headers": []}) await send({"type": "http.response.body", "body": str(os.getpid()).encode()}) """ +CORE_DUMP_SECONDS: Final = 3 +SLOW_CORE_DUMP_ON_QUIT: Final = f""" +import os +import signal +import subprocess + + +def freeze_until_killed(signum, frame): + subprocess.Popen(("sh", "-c", f"sleep {CORE_DUMP_SECONDS}; kill -9 {{os.getpid()}}")) + os.kill(os.getpid(), signal.SIGSTOP) + + +signal.signal(signal.SIGQUIT, freeze_until_killed) +""" POOL_RUNNER: Final = """import sys sys.path.insert(0, sys.argv[1]) @@ -193,14 +207,14 @@ def _wait_for_log(log: Path, process: subprocess.Popen[bytes], seen: Callable[[s raise AssertionError(log.read_text()) -def _start_pool(app_dir: Path, port: int, log: Path) -> subprocess.Popen[bytes]: +def _start_pool(app_dir: Path, port: int, log: Path, runner: str = POOL_RUNNER) -> subprocess.Popen[bytes]: with log.open("wb") as output: return subprocess.Popen( ( sys.executable, "-I", "-c", - POOL_RUNNER, + runner, str(app_dir), str(port), str(WORKER_HEALTHCHECK_SECONDS), @@ -288,3 +302,28 @@ def test_the_pool_stops_and_frees_the_port_when_the_reloader_dies( with contextlib.suppress(ProcessLookupError): os.killpg(process.pid, signal.SIGKILL) process.wait() + + +@pytest.mark.skipif( + not multi_worker_reload_supported(), reason="uvicorn<0.50.0 cannot stop a worker pool on a startup failure" +) +@pytest.mark.timeout(3 * POOL_TIMEOUT_SECONDS) +def test_ctrl_backslash_frees_the_port_without_waiting_out_the_worker_healthcheck_when_core_dumps_are_slow( + tmp_path: Path, +) -> None: + (tmp_path / "reload_pool_app.py").write_text(PID_APP + SLOW_CORE_DUMP_ON_QUIT) + log: Final = tmp_path / "reload.log" + port: Final = _free_port() + process: Final = _start_pool(tmp_path, port, log, runner=SLOW_CORE_DUMP_ON_QUIT + POOL_RUNNER) + try: + workers: Final = _next_pool(log, process, port, frozenset()) + pool_parent: Final = psutil.Process(min(workers)).ppid() + os.killpg(process.pid, signal.SIGQUIT) + deadline: Final = time.monotonic() + WORKER_HEALTHCHECK_SECONDS / 2 + while process.poll() is None or psutil.pid_exists(pool_parent) or not _port_refuses_connections(port): + assert time.monotonic() < deadline, log.read_text() + time.sleep(0.2) + finally: + with contextlib.suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + process.wait() From ce123a7d11b2093d7881a9e640005829d987178c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:46:35 -0700 Subject: [PATCH 12/13] test(proxy): skip the startup-failure reload tests on uvicorn without a startup failure code --- tests/unit/proxy/test_dev_reload.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/unit/proxy/test_dev_reload.py b/tests/unit/proxy/test_dev_reload.py index 3387e4776a6..7498b249bee 100644 --- a/tests/unit/proxy/test_dev_reload.py +++ b/tests/unit/proxy/test_dev_reload.py @@ -48,6 +48,9 @@ async def app(scope, receive, send): await send({"type": "http.response.start", "status": 200, "headers": []}) await send({"type": "http.response.body", "body": str(os.getpid()).encode()}) """ +REQUIRES_UVICORN_STARTUP_FAILURE: Final = pytest.mark.skipif( + not multi_worker_reload_supported(), reason="uvicorn<0.50.0 has no startup failure exit code" +) CORE_DUMP_SECONDS: Final = 3 SLOW_CORE_DUMP_ON_QUIT: Final = f""" import os @@ -93,6 +96,7 @@ def _options(app: str) -> ReloadOptions: } +@REQUIRES_UVICORN_STARTUP_FAILURE @pytest.mark.parametrize( ("module", "source"), ( @@ -112,6 +116,7 @@ def test_a_broken_app_exits_the_worker_as_a_startup_failure( assert exit_info.value.code == uvicorn_config.STARTUP_FAILURE +@REQUIRES_UVICORN_STARTUP_FAILURE @pytest.mark.parametrize( ("module", "source"), ( @@ -240,9 +245,7 @@ def _next_pool(log: Path, process: subprocess.Popen[bytes], port: int, previous: return pool -@pytest.mark.skipif( - not multi_worker_reload_supported(), reason="uvicorn<0.50.0 cannot stop a worker pool on a startup failure" -) +@REQUIRES_UVICORN_STARTUP_FAILURE @pytest.mark.timeout(5 * POOL_TIMEOUT_SECONDS) def test_a_save_replaces_every_worker_and_a_broken_save_stops_the_pool_until_the_next_good_one( tmp_path: Path, @@ -269,9 +272,7 @@ def test_a_save_replaces_every_worker_and_a_broken_save_stops_the_pool_until_the process.wait() -@pytest.mark.skipif( - not multi_worker_reload_supported(), reason="uvicorn<0.50.0 cannot stop a worker pool on a startup failure" -) +@REQUIRES_UVICORN_STARTUP_FAILURE @pytest.mark.timeout(3 * POOL_TIMEOUT_SECONDS) @pytest.mark.parametrize( "kill_reloader", @@ -304,9 +305,7 @@ def test_the_pool_stops_and_frees_the_port_when_the_reloader_dies( process.wait() -@pytest.mark.skipif( - not multi_worker_reload_supported(), reason="uvicorn<0.50.0 cannot stop a worker pool on a startup failure" -) +@REQUIRES_UVICORN_STARTUP_FAILURE @pytest.mark.timeout(3 * POOL_TIMEOUT_SECONDS) def test_ctrl_backslash_frees_the_port_without_waiting_out_the_worker_healthcheck_when_core_dumps_are_slow( tmp_path: Path, From dc68ca731b7ba17ee6bb477ca36ba944ea09a506 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:21:14 -0700 Subject: [PATCH 13/13] docs(agents): keep the QA proxy rule to two workers, Redis, and reload --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index fd23e3149e2..68969a4a7e7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,7 +39,7 @@ Same applies for filing bug reports and feature requests, with .github/ISSUE_TEM If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just drop the section -Run QA proxies the way the command below does, with `--num_workers 2 --reload`, and point both `router_settings` (`redis_host`, `redis_port`) and `general_settings.coordination_redis` (`host`, `port`) at one dedicated Redis instance, with credentials when required, so the workers share router state and spend, budget, and rate-limit counters. `litellm/proxy/dev_config.yaml` sets no Redis, so pass `--config` a copy of it that adds them. Check that two worker PIDs are serving, then edit a watched Python, config YAML, or `.env` file and check that both workers are replaced and requests still succeed. Use the same worker count and Redis setup for the before and after proof. When the PR is about `--run_gunicorn`, `--run_hypercorn`, or `--run_granian`, which `--reload` does not support, drop `--reload` and the reload check. When it is about behavior without Redis, drop Redis. When the proxy prints `"workers" flag is ignored when reloading is enabled`, that checkout or its uvicorn reloads only one worker, so drop the two-worker checks for it. Say in the proof which part you dropped and why +Run QA proxies with the command below, which passes `--num_workers 2 --reload`, and swap its `--config` for a copy of `litellm/proxy/dev_config.yaml` that points both `router_settings` (`redis_host`, `redis_port`) and `general_settings.coordination_redis` (`host`, `port`) at one dedicated Redis instance, with credentials when required, so the workers share router state and spend, budget, and rate-limit counters. Check that two worker PIDs are serving, and use the same worker count and Redis setup for the before and after proof. When the PR is about `--run_gunicorn`, `--run_hypercorn`, or `--run_granian`, which `--reload` does not support, drop `--reload`. When it is about behavior without Redis, drop Redis. When the proxy prints `"workers" flag is ignored when reloading is enabled`, that checkout or its uvicorn reloads only one worker, so drop the two-worker check for it. Say in the proof which part you dropped and why Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --num_workers 2 --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`. The Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into. I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, or the main use case runs through a headful agentic coding tool like Claude Code or Codex, drive that surface yourself and embed your own before and after screenshots of it in the PR (the Admin UI page, or what the coding tool shows), next to an ordered list of the URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, and what fields to fill out so a reviewer can reproduce it