From 4c7afd979851f0b5c24bb72eaa944bf5d3e88838 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 23 Oct 2026 23:57:45 +0000 Subject: [PATCH 1/2] fix(gateway): clean up PgBouncer startup cancellation --- gateway/launch.py | 27 +- litellm/proxy/db/pgbouncer.py | 225 ++++++-- .../configuration/test_pgbouncer_lifecycle.py | 496 ++++++++++++++++++ 3 files changed, 690 insertions(+), 58 deletions(-) create mode 100644 tests/integration/configuration/test_pgbouncer_lifecycle.py diff --git a/gateway/launch.py b/gateway/launch.py index d67432caee1..69e2b7c0506 100644 --- a/gateway/launch.py +++ b/gateway/launch.py @@ -21,9 +21,11 @@ from uvicorn.main import main as uvicorn_main from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import ( + PGBOUNCER_MANAGED_SHUTDOWN_SECONDS, PgBouncerError, PgBouncerSettings, export_pooled_database_url, + pgbouncer_startup_signals, start_in_container_pgbouncer, ) @@ -55,7 +57,12 @@ def pool_database_url( upstream_url: Final = environ.get("DATABASE_URL") if upstream_url is None: return PgBouncerError("LITELLM_PGBOUNCER_ENABLED is set but no DATABASE_URL could be assembled") - return start_in_container_pgbouncer(pgbouncer, upstream_url, token_auth=settings.token_auth()) + return start_in_container_pgbouncer( + pgbouncer, + upstream_url, + token_auth=settings.token_auth(), + shutdown_timeout_seconds=PGBOUNCER_MANAGED_SHUTDOWN_SECONDS, + ) def _serve(argv: Sequence[str]) -> None: @@ -63,14 +70,16 @@ def _serve(argv: Sequence[str]) -> None: def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None: - settings: Final = DatabaseURLSettings.from_env() - settings.apply_to_env() - pooled_url: Final = pool_database_url(settings, PgBouncerSettings(), os.environ) - if isinstance(pooled_url, PgBouncerError): - sys.exit(f"LiteLLM gateway: in-container pgbouncer could not start: {pooled_url.reason}") - if pooled_url is not None: - export_pooled_database_url(pooled_url) - serve(uvicorn_argv(argv, os.environ)) + pgbouncer: Final = PgBouncerSettings() + with pgbouncer_startup_signals(pgbouncer.enabled): + settings: Final = DatabaseURLSettings.from_env() + settings.apply_to_env() + pooled_url: Final = pool_database_url(settings, pgbouncer, os.environ) + if isinstance(pooled_url, PgBouncerError): + sys.exit(f"LiteLLM gateway: in-container pgbouncer could not start: {pooled_url.reason}") + if pooled_url is not None: + export_pooled_database_url(pooled_url) + serve(uvicorn_argv(argv, os.environ)) if __name__ == "__main__": diff --git a/litellm/proxy/db/pgbouncer.py b/litellm/proxy/db/pgbouncer.py index c9fbabd3585..60eee582474 100644 --- a/litellm/proxy/db/pgbouncer.py +++ b/litellm/proxy/db/pgbouncer.py @@ -46,12 +46,13 @@ import tempfile import threading import time import urllib.parse -from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass +from collections.abc import Callable, Generator, Mapping, Sequence +from contextlib import ExitStack, contextmanager +from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path -from types import MappingProxyType -from typing import Final +from types import FrameType, MappingProxyType +from typing import Final, NoReturn from pydantic import Field from pydantic_settings import BaseSettings, SettingsConfigDict @@ -75,6 +76,7 @@ PGBOUNCER_CA_NAME: Final = "server-ca.pem" PGBOUNCER_RESTART_DELAY_SECONDS: Final = 1.0 PGBOUNCER_READY_TIMEOUT_SECONDS: Final = 15.0 PGBOUNCER_STOP_GRACE_SECONDS: Final = 10.0 +PGBOUNCER_MANAGED_SHUTDOWN_SECONDS: Final = 30.0 PGBOUNCER_UNPRIVILEGED_USER: Final = "nobody" PGBOUNCER_MIN_VERSION: Final = (1, 19) PGBOUNCER_MAX_PASSWORD_BYTES: Final = 2048 @@ -396,6 +398,8 @@ class PgBouncerTokenRefresher: token: Final = self._source.mint() except Exception as mint_error: return PgBouncerError(f"could not mint a {label} for the in-container pgbouncer: {mint_error!r}") + if self._stopping.is_set(): + return self._fallback_seconds if len(token.encode()) >= PGBOUNCER_MAX_PASSWORD_BYTES: return PgBouncerError( f"the {label} is {len(token.encode())} bytes long, but PgBouncer's auth file holds passwords of at " @@ -440,10 +444,16 @@ class PgBouncerTokenRefresher: ) return outcome - def stop(self) -> None: + def request_stop(self) -> None: self._stopping.set() + + def stop(self, deadline: float | None = None) -> None: + self.request_stop() if self._thread is not None: - self._thread.join() + if deadline is None: + self._thread.join() + else: + self._thread.join(timeout=max(0.0, deadline - time.monotonic())) def _port_open(port: int) -> bool: @@ -487,15 +497,27 @@ def pgbouncer_version(binary: str) -> tuple[int, int] | PgBouncerError: return int(found[1]), int(found[2]) -def _end(process: subprocess.Popen[bytes]) -> None: +def _end(process: subprocess.Popen[bytes], deadline: float | None = None) -> None: if process.poll() is not None: return process.terminate() try: - process.wait(timeout=PGBOUNCER_STOP_GRACE_SECONDS) + remaining: Final = None if deadline is None else max(0.0, deadline - time.monotonic()) + timeout: Final = ( + PGBOUNCER_STOP_GRACE_SECONDS + if remaining is None + else max(0.0, min(PGBOUNCER_STOP_GRACE_SECONDS, remaining - 1.0)) + ) + process.wait(timeout=timeout) except subprocess.TimeoutExpired: process.kill() - process.wait() + if deadline is None: + process.wait() + else: + try: + process.wait(timeout=max(0.0, deadline - time.monotonic())) + except subprocess.TimeoutExpired: + pass class PgBouncerProcess: @@ -537,6 +559,12 @@ class PgBouncerProcess: with self._lock: return None if self._process is None else self._process.pid + @contextmanager + def owned_pid_guard(self) -> Generator[int | None, None, None]: + with self._lock: + process: Final = self._process + yield None if process is None or process.poll() is not None else process.pid + def _spawn(self) -> subprocess.Popen[bytes] | PgBouncerError | None: """Start a child, or None once ``stop`` ran; both take the lock so no child can slip in after a stop. @@ -572,7 +600,7 @@ class PgBouncerProcess: f"within {self.ready_timeout_seconds:.0f}s" ) - def start(self) -> PgBouncerError | None: + def start(self, deadline: float | None = None) -> PgBouncerError | None: """Spawn PgBouncer, wait until it listens on port and unix socket, then supervise it from a daemon thread.""" process: Final = self._spawn() if process is None: @@ -581,7 +609,7 @@ class PgBouncerProcess: return process not_ready: Final = self._wait_ready(process) if not_ready is not None: - self.stop() + self.stop(deadline) return not_ready self._watch(process) return None @@ -631,12 +659,12 @@ class PgBouncerProcess: if self._process is not None: self._process.send_signal(signal.SIGHUP) - def stop(self) -> None: + def stop(self, deadline: float | None = None) -> None: with self._lock: self._stopping.set() process: Final = self._process if process is not None: - _end(process) + _end(process, deadline) def install_pgbouncer_token( @@ -653,6 +681,10 @@ def _install_upstream_password( pooler: PgBouncerProcess, token_auth: DatabaseTokenAuth | None, upstream_url: str, + startup_cleanup: ExitStack, + refresher_factory: Callable[ + [PgBouncerTokenSource, Callable[[str], None]], PgBouncerTokenRefresher + ] = PgBouncerTokenRefresher, ) -> PgBouncerTokenRefresher | None | PgBouncerError: if token_auth is None: if plan.upstream_password is None: @@ -662,10 +694,11 @@ def _install_upstream_password( ) write_userlist(plan.userlist(plan.upstream_password), runtime_dir, run_as_user) return None - refresher: Final = PgBouncerTokenRefresher( + refresher: Final = refresher_factory( PgBouncerTokenSource(auth=token_auth, endpoint=parse_iam_endpoint_from_url(upstream_url)), functools.partial(install_pgbouncer_token, plan, runtime_dir, run_as_user, pooler), ) + startup_cleanup.callback(_stop_managed_refresher, refresher) failed: Final = refresher.start() if failed is not None: return failed @@ -683,11 +716,130 @@ def _only_in_this_process(action: Callable[[], None]) -> Callable[[], None]: return run +def _raise_for_startup_signal(signum: int, _frame: FrameType | None) -> NoReturn: + raise SystemExit(128 + signum) + + +@contextmanager +def pgbouncer_startup_signals(enabled: bool) -> Generator[None, None, None]: + if not enabled: + yield + return + previous_term: Final = signal.getsignal(signal.SIGTERM) + previous_int: Final = signal.getsignal(signal.SIGINT) + signal.signal(signal.SIGTERM, _raise_for_startup_signal) + try: + signal.signal(signal.SIGINT, _raise_for_startup_signal) + try: + yield + finally: + signal.signal(signal.SIGINT, previous_int) + finally: + signal.signal(signal.SIGTERM, previous_term) + + +def _stop_managed_pooler(pooler: PgBouncerProcess) -> None: + pooler.stop(time.monotonic() + PGBOUNCER_STOP_GRACE_SECONDS) + + +def _stop_managed_refresher(refresher: PgBouncerTokenRefresher) -> None: + refresher.stop(time.monotonic() + PGBOUNCER_STOP_GRACE_SECONDS) + + +@dataclass(frozen=True, slots=True) +class ManagedPgBouncerRuntime: + pooled_url: str = field(repr=False) + pooler: PgBouncerProcess + refresher: PgBouncerTokenRefresher | None + runtime_dir: Path + + @contextmanager + def owned_pid_guard(self) -> Generator[int | None, None, None]: + with self.pooler.owned_pid_guard() as pid: + yield pid + + def stop(self, deadline: float | None = None) -> None: + if self.refresher is not None: + self.refresher.request_stop() + try: + self.pooler.stop(deadline) + finally: + try: + if self.refresher is not None: + self.refresher.stop(deadline) + finally: + shutil.rmtree(self.runtime_dir, ignore_errors=True) + + +def start_managed_pgbouncer( + settings: PgBouncerSettings, + upstream_url: str, + token_auth: DatabaseTokenAuth | None = None, + refresher_factory: Callable[ + [PgBouncerTokenSource, Callable[[str], None]], PgBouncerTokenRefresher + ] = PgBouncerTokenRefresher, +) -> ManagedPgBouncerRuntime | PgBouncerError: + version: Final = pgbouncer_version(settings.binary) + if isinstance(version, PgBouncerError): + return version + if version < PGBOUNCER_MIN_VERSION: + return PgBouncerError( + f"PgBouncer {version[0]}.{version[1]} keeps running after failing to bind its TCP port, so the proxy " + f"cannot tell it apart from another listener; {PGBOUNCER_MIN_VERSION[0]}.{PGBOUNCER_MIN_VERSION[1]} " + "or newer is required" + ) + with ExitStack() as startup_cleanup: + runtime_dir: Final = Path(tempfile.mkdtemp(prefix="litellm-pgbouncer-")) + startup_cleanup.callback(shutil.rmtree, runtime_dir, ignore_errors=True) + run_as_user: Final = PGBOUNCER_UNPRIVILEGED_USER if os.geteuid() == 0 else None + plan: Final = plan_pgbouncer(upstream_url, settings, runtime_dir, run_as_user) + if isinstance(plan, PgBouncerError): + return plan + ini_path: Final = write_pgbouncer_ini(plan, runtime_dir, run_as_user) + if isinstance(ini_path, PgBouncerError): + return ini_path + pooler: Final = PgBouncerProcess( + argv=(settings.binary, str(ini_path)), + port=settings.port, + socket_path=unix_socket_path(runtime_dir, settings.port), + ) + startup_cleanup.callback(_stop_managed_pooler, pooler) + refresher_result: Final = _install_upstream_password( + plan, + runtime_dir, + run_as_user, + pooler, + token_auth, + upstream_url, + startup_cleanup, + refresher_factory, + ) + if isinstance(refresher_result, PgBouncerError): + return refresher_result + failed: Final = pooler.start(time.monotonic() + PGBOUNCER_STOP_GRACE_SECONDS) + if failed is not None: + return failed + runtime: Final = ManagedPgBouncerRuntime( + pooled_url=plan.pooled_url, + pooler=pooler, + refresher=refresher_result, + runtime_dir=runtime_dir, + ) + startup_cleanup.pop_all() + return runtime + + +def _stop_managed_runtime_before_exit(runtime: ManagedPgBouncerRuntime, timeout_seconds: float) -> None: + runtime.stop(time.monotonic() + timeout_seconds) + + def start_in_container_pgbouncer( settings: PgBouncerSettings, upstream_url: str, token_auth: DatabaseTokenAuth | None = None, register_exit_hook: Callable[[Callable[[], None]], object] = atexit.register, + *, + shutdown_timeout_seconds: float | None = None, ) -> str | PgBouncerError: """Start the pooler for ``upstream_url`` and return the loopback URL the workers must use. @@ -698,46 +850,21 @@ def start_in_container_pgbouncer( ``nobody``. With ``token_auth`` the password on ``upstream_url`` is ignored: the pooler mints its own tokens and renews them for as long as it runs. """ - version: Final = pgbouncer_version(settings.binary) - if isinstance(version, PgBouncerError): - return version - if version < PGBOUNCER_MIN_VERSION: - return PgBouncerError( - f"PgBouncer {version[0]}.{version[1]} keeps running after failing to bind its TCP port, so the proxy " - f"cannot tell it apart from another listener; {PGBOUNCER_MIN_VERSION[0]}.{PGBOUNCER_MIN_VERSION[1]} " - "or newer is required" - ) - runtime_dir: Final = Path(tempfile.mkdtemp(prefix="litellm-pgbouncer-")) - register_exit_hook(_only_in_this_process(lambda: shutil.rmtree(runtime_dir, ignore_errors=True))) - run_as_user: Final = PGBOUNCER_UNPRIVILEGED_USER if os.geteuid() == 0 else None - plan: Final = plan_pgbouncer(upstream_url, settings, runtime_dir, run_as_user) - if isinstance(plan, PgBouncerError): - return plan - ini_path: Final = write_pgbouncer_ini(plan, runtime_dir, run_as_user) - if isinstance(ini_path, PgBouncerError): - return ini_path - pooler: Final = PgBouncerProcess( - argv=(settings.binary, str(ini_path)), - port=settings.port, - socket_path=unix_socket_path(runtime_dir, settings.port), + runtime: Final = start_managed_pgbouncer(settings, upstream_url, token_auth) + if isinstance(runtime, PgBouncerError): + return runtime + cleanup: Final = ( + runtime.stop + if shutdown_timeout_seconds is None + else functools.partial(_stop_managed_runtime_before_exit, runtime, shutdown_timeout_seconds) ) - refresher: Final = _install_upstream_password(plan, runtime_dir, run_as_user, pooler, token_auth, upstream_url) - if isinstance(refresher, PgBouncerError): - return refresher - failed: Final = pooler.start() - if failed is not None: - if refresher is not None: - refresher.stop() - return failed - register_exit_hook(_only_in_this_process(pooler.stop)) - if refresher is not None: - register_exit_hook(_only_in_this_process(refresher.stop)) + register_exit_hook(_only_in_this_process(cleanup)) verbose_proxy_logger.info( "In-container pgbouncer (pid %s) listening on %s:%s; capping this pod at %s upstream database connections%s.", - pooler.pid, + runtime.pooler.pid, PGBOUNCER_LISTEN_ADDR, settings.port, settings.max_db_connections, "" if token_auth is None else f" and renewing its {token_auth.label} before each one expires", ) - return plan.pooled_url + return runtime.pooled_url diff --git a/tests/integration/configuration/test_pgbouncer_lifecycle.py b/tests/integration/configuration/test_pgbouncer_lifecycle.py new file mode 100644 index 00000000000..eb890d9300f --- /dev/null +++ b/tests/integration/configuration/test_pgbouncer_lifecycle.py @@ -0,0 +1,496 @@ +import http.client +import json +import os +import signal +import subprocess +import sys +import textwrap +import threading +import time +from pathlib import Path +from typing import Final, cast + +import pytest + +from tests.integration._support.ports import allocate_port +from tests.integration._support.timing import assert_enforced_within + +ROOT: Final = Path(os.getenv("LITELLM_TEST_ROOT", str(Path(__file__).resolve().parents[3]))) +LAUNCHER_SOURCE: Final = """\ +import ctypes +import json +import os +import shutil +import signal +import socket +import subprocess +import sys +import time +from pathlib import Path + +PR_SET_CHILD_SUBREAPER = 36 +if ctypes.CDLL(None, use_errno=True).prctl(PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0) != 0: + raise OSError(ctypes.get_errno(), "prctl(PR_SET_CHILD_SUBREAPER) failed") + +root, launcher_pid_file, pooler_pid_file, config_file, port_file, result_file, *command = sys.argv[1:] +started = time.monotonic() +launcher = subprocess.Popen(command, cwd=root, env=os.environ.copy()) +Path(launcher_pid_file).write_text(str(launcher.pid)) +status = launcher.wait() +try: + pooler_pid = int(Path(pooler_pid_file).read_text()) +except (OSError, ValueError): + pooler_pid = None +try: + config_path = Path(Path(config_file).read_text()) +except OSError: + config_path = None +try: + port = int(Path(port_file).read_text()) +except (OSError, ValueError): + port = None +pooler_present = pooler_pid is not None and Path(f"/proc/{pooler_pid}").exists() +listener_open = False +if port is not None: + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.2): + listener_open = True + except OSError: + pass +result = { + "launcher_exit_code": status, + "elapsed": time.monotonic() - started, + "pooler_pid": pooler_pid, + "pooler_present": pooler_present, + "listener_open": listener_open, + "runtime_dir_present": config_path is not None and config_path.parent.exists(), +} +Path(result_file).write_text(json.dumps(result)) +if pooler_pid is not None and Path(f"/proc/{pooler_pid}").exists(): + try: + os.kill(pooler_pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + os.waitpid(pooler_pid, 0) + except ChildProcessError: + pass +if config_path is not None: + shutil.rmtree(config_path.parent, ignore_errors=True) +raise SystemExit(status) +""" + +PGBOUNCER_SOURCE: Final = """\ +import configparser +import os +import select +import signal +import socket +import sys +import time +from pathlib import Path + +signal.signal(signal.SIGTERM, signal.SIG_IGN) +signal.signal(signal.SIGINT, signal.SIG_IGN) +signal.signal(signal.SIGHUP, signal.SIG_IGN) + +if sys.argv[1:] == ["--version"]: + print("PgBouncer 1.26.0") + raise SystemExit(0) + +config_path = Path(sys.argv[1]) +parser = configparser.ConfigParser() +parser.read(config_path) +port = parser.getint("pgbouncer", "listen_port") +socket_path = Path(parser.get("pgbouncer", "unix_socket_dir")) / f".s.PGSQL.{port}" +Path(os.environ["LITELLM_TEST_POOLER_PID_FILE"]).write_text(str(os.getpid())) +Path(os.environ["LITELLM_TEST_POOLER_CONFIG_FILE"]).write_text(str(config_path)) +Path(os.environ["LITELLM_TEST_POOLER_PORT_FILE"]).write_text(str(port)) +if os.environ["LITELLM_TEST_POOLER_MODE"] == "slow": + while True: + signal.pause() +if os.environ["LITELLM_TEST_POOLER_MODE"] == "exit": + time.sleep(0.25) + raise SystemExit(3) + +tcp = socket.socket() +tcp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +tcp.bind(("127.0.0.1", port)) +tcp.listen() +unix = socket.socket(socket.AF_UNIX) +unix.bind(str(socket_path)) +unix.listen() +while True: + for listener in select.select((tcp, unix), (), ())[0]: + listener.accept()[0].close() +""" + +SERVE_SOURCE: Final = """\ +import sys +import uvicorn +from gateway.launch import main + +async def app(scope, receive, send): + if scope["type"] == "http": + await send({"type": "http.response.start", "status": 200, "headers": []}) + await send({"type": "http.response.body", "body": b"ready"}) + +def serve(argv): + port = int(argv[argv.index("--port") + 1]) + uvicorn.Server( + uvicorn.Config(app, host="127.0.0.1", port=port, lifespan="off", workers=1) + ).run() + +main(sys.argv[1:], serve=serve) +""" + + +def _write_script(path: Path, source: str) -> Path: + path.write_text(f"#!/usr/bin/env python3\n{textwrap.dedent(source)}") + path.chmod(0o700) + return path + + +def _wait_for_file(path: Path, timeout_seconds: float = 15.0) -> str: + deadline: Final = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + try: + return path.read_text() + except OSError: + time.sleep(0.05) + pytest.fail(f"timed out waiting for {path}") + + +def _wait_for_new_pid(path: Path, previous_pid: int, timeout_seconds: float = 5.0) -> int: + deadline: Final = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + try: + pid: Final = int(path.read_text()) + if pid != previous_pid: + return pid + except (OSError, ValueError): + pass + time.sleep(0.05) + pytest.fail(f"timed out waiting for a replacement process in {path}") + + +def _http_health(port: int) -> bool: + try: + connection: Final = http.client.HTTPConnection("127.0.0.1", port, timeout=0.5) + connection.request("GET", "/health/liveliness") + response: Final = connection.getresponse() + response.read() + connection.close() + return response.status == 200 + except (OSError, http.client.HTTPException): + return False + + +def _wait_for_health(port: int, timeout_seconds: float = 15.0) -> None: + deadline: Final = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + if _http_health(port): + return + time.sleep(0.05) + pytest.fail(f"gateway did not become healthy on port {port}") + + +def _run_black_box( + tmp_path: Path, pooler_mode: str, signal_number: signal.Signals, require_health: bool +) -> dict[str, object]: + launcher_pid_file: Final = tmp_path / "launcher.pid" + pooler_pid_file: Final = tmp_path / "pooler.pid" + config_file: Final = tmp_path / "pooler-config" + port_file: Final = tmp_path / "pooler-port" + result_file: Final = tmp_path / "result.json" + pooler_port: Final = allocate_port() + gateway_port: Final = allocate_port() + pooler_binary: Final = _write_script(tmp_path / "fake-pgbouncer", PGBOUNCER_SOURCE) + wrapper: Final = _write_script(tmp_path / "subreaper-wrapper.py", LAUNCHER_SOURCE) + environment: Final = { + **os.environ, + "PYTHONPATH": os.pathsep.join(filter(None, (str(ROOT), os.environ.get("PYTHONPATH", "")))), + "DATABASE_URL": "postgresql://app:password@127.0.0.1:5432/litellm", + "LITELLM_PGBOUNCER_ENABLED": "true", + "LITELLM_PGBOUNCER_BINARY": str(pooler_binary), + "LITELLM_PGBOUNCER_PORT": str(pooler_port), + "LITELLM_TEST_POOLER_MODE": pooler_mode, + "LITELLM_TEST_POOLER_PID_FILE": str(pooler_pid_file), + "LITELLM_TEST_POOLER_CONFIG_FILE": str(config_file), + "LITELLM_TEST_POOLER_PORT_FILE": str(port_file), + } + serve_script: Final = _write_script(tmp_path / "serve-gateway.py", SERVE_SOURCE) + command: Final = ( + ( + sys.executable, + str(serve_script), + "--host", + "127.0.0.1", + "--port", + str(gateway_port), + "--workers", + "1", + ) + if require_health + else ( + sys.executable, + "-m", + "gateway.launch", + "--host", + "127.0.0.1", + "--port", + str(gateway_port), + "--workers", + "1", + ) + ) + launch_log: Final = (tmp_path / "gateway.log").open("w") + wrapper_process: Final = subprocess.Popen( + ( + sys.executable, + "-P", + str(wrapper), + str(ROOT), + str(launcher_pid_file), + str(pooler_pid_file), + str(config_file), + str(port_file), + str(result_file), + *command, + ), + cwd=ROOT, + env=environment, + stdin=subprocess.DEVNULL, + stdout=launch_log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + try: + launcher_pid: Final = int(_wait_for_file(launcher_pid_file)) + _wait_for_file(pooler_pid_file) + if require_health: + try: + _wait_for_health(gateway_port) + except BaseException as error: + raise AssertionError((tmp_path / "gateway.log").read_text()) from error + os.kill(launcher_pid, signal_number) # test-quality-ok: signal starts the lifecycle under test + wrapper_process.wait(timeout=35) + return cast(dict[str, object], json.loads(result_file.read_text())) + finally: + if wrapper_process.poll() is None: + try: + launcher_pid = int(launcher_pid_file.read_text()) + os.kill(launcher_pid, signal.SIGKILL) # test-quality-ok: bounded fixture cleanup + except (OSError, ValueError): + pass + try: + wrapper_process.wait(timeout=15) + except subprocess.TimeoutExpired: + os.kill(wrapper_process.pid, signal.SIGKILL) # test-quality-ok: bounded fixture cleanup + wrapper_process.wait() + launch_log.close() + + +@pytest.mark.parametrize( + ("signal_number", "expected_exit"), + ((signal.SIGTERM, 143), (signal.SIGINT, 130)), +) +def test_startup_signal_removes_slow_pooler_and_runtime( + tmp_path: Path, signal_number: signal.Signals, expected_exit: int +) -> None: + result: Final = _run_black_box(tmp_path, "slow", signal_number, require_health=False) + assert result["pooler_present"] is False, result + assert result["listener_open"] is False, result + assert result["runtime_dir_present"] is False, result + assert result["launcher_exit_code"] == expected_exit, result + assert cast(float, result["elapsed"]) <= 30 + + +@pytest.mark.parametrize( + ("signal_number", "expected_exit"), + ((signal.SIGTERM, 143), (signal.SIGINT, 130)), +) +def test_running_gateway_signal_removes_pooler_and_runtime( + tmp_path: Path, signal_number: signal.Signals, expected_exit: int +) -> None: + result: Final = _run_black_box(tmp_path, "serve", signal_number, require_health=True) + assert result["pooler_present"] is False, result + assert result["listener_open"] is False, result + assert result["runtime_dir_present"] is False, result + assert result["launcher_exit_code"] == expected_exit, result + assert cast(float, result["elapsed"]) <= 30 + + +def test_managed_pooler_stop_reaps_owned_pid_and_runtime(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy.db.pgbouncer import ManagedPgBouncerRuntime, PgBouncerSettings, start_managed_pgbouncer + + pid_file: Final = tmp_path / "pooler.pid" + config_file: Final = tmp_path / "pooler-config" + port_file: Final = tmp_path / "pooler-port" + pooler_binary: Final = _write_script(tmp_path / "fake-pgbouncer", PGBOUNCER_SOURCE) + settings: Final = PgBouncerSettings(enabled=True, binary=str(pooler_binary), port=allocate_port()) + environment: Final = { + "LITELLM_TEST_POOLER_MODE": "serve", + "LITELLM_TEST_POOLER_PID_FILE": str(pid_file), + "LITELLM_TEST_POOLER_CONFIG_FILE": str(config_file), + "LITELLM_TEST_POOLER_PORT_FILE": str(port_file), + } + for key, value in environment.items(): + monkeypatch.setenv(key, value) + runtime_result: Final = start_managed_pgbouncer(settings, "postgresql://app:password@db/litellm") + assert isinstance(runtime_result, ManagedPgBouncerRuntime), runtime_result + runtime: Final = runtime_result + pid: Final = runtime.pooler.pid + try: + assert pid is not None + with runtime.owned_pid_guard() as guarded_pid: + assert guarded_pid == pid + finally: + runtime.stop(time.monotonic() + 15) + assert pid is not None + assert not Path(f"/proc/{pid}").exists() + assert not runtime.runtime_dir.exists() + + +def test_pooler_start_failure_stops_injected_fast_refresher(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from collections.abc import Callable + + from litellm.proxy.db.pgbouncer import ( + PgBouncerError, + PgBouncerSettings, + PgBouncerTokenRefresher, + PgBouncerTokenSource, + start_managed_pgbouncer, + ) + from litellm.proxy.db.token_auth import AzureEntraTokenAuth + + pid_file: Final = tmp_path / "pooler.pid" + config_file: Final = tmp_path / "pooler-config" + port_file: Final = tmp_path / "pooler-port" + pooler_binary: Final = _write_script(tmp_path / "fake-pgbouncer", PGBOUNCER_SOURCE) + settings: Final = PgBouncerSettings(enabled=True, binary=str(pooler_binary), port=allocate_port()) + monkeypatch.setenv("LITELLM_TEST_POOLER_MODE", "exit") + monkeypatch.setenv("LITELLM_TEST_POOLER_PID_FILE", str(pid_file)) + monkeypatch.setenv("LITELLM_TEST_POOLER_CONFIG_FILE", str(config_file)) + monkeypatch.setenv("LITELLM_TEST_POOLER_PORT_FILE", str(port_file)) + first_mint: Final = threading.Event() + refreshed_during_start: Final = threading.Event() + start_failed: Final = threading.Event() + refreshed_after_failure: Final = threading.Event() + initial_install_done: Final = threading.Event() + + def provide_token() -> str: + if first_mint.is_set(): + refreshed_during_start.set() + if start_failed.is_set(): + refreshed_after_failure.set() + else: + first_mint.set() + return "opaque-token" + + def fast_refresher_factory(source: PgBouncerTokenSource, install: Callable[[str], None]) -> PgBouncerTokenRefresher: + def install_once(token: str) -> None: + if initial_install_done.is_set(): + return + install(token) + initial_install_done.set() + + return PgBouncerTokenRefresher(source, install_once, fallback_seconds=0.02, retry_seconds=0.02) + + outcome: Final = start_managed_pgbouncer( + settings, + "postgresql://app@db/litellm", + token_auth=AzureEntraTokenAuth(token_provider=provide_token), + refresher_factory=fast_refresher_factory, + ) + assert isinstance(outcome, PgBouncerError), outcome + assert pid_file.exists(), outcome + assert refreshed_during_start.wait(2) + start_failed.set() + time.sleep(0.1) + assert not refreshed_after_failure.is_set() + runtime_dir: Final = Path(config_file.read_text()).parent + assert not runtime_dir.exists() + + +def test_managed_stop_reaps_replacement_before_refresher_deadline_and_skips_late_install( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from collections.abc import Callable + + from litellm.proxy.db.pgbouncer import ( + ManagedPgBouncerRuntime, + PgBouncerSettings, + PgBouncerTokenRefresher, + PgBouncerTokenSource, + start_managed_pgbouncer, + ) + from litellm.proxy.db.token_auth import AzureEntraTokenAuth + + pid_file: Final = tmp_path / "pooler.pid" + config_file: Final = tmp_path / "pooler-config" + port_file: Final = tmp_path / "pooler-port" + monkeypatch.setenv("LITELLM_TEST_POOLER_MODE", "serve") + monkeypatch.setenv("LITELLM_TEST_POOLER_PID_FILE", str(pid_file)) + monkeypatch.setenv("LITELLM_TEST_POOLER_CONFIG_FILE", str(config_file)) + monkeypatch.setenv("LITELLM_TEST_POOLER_PORT_FILE", str(port_file)) + first_mint: Final = threading.Event() + mint_blocked: Final = threading.Event() + release_mint: Final = threading.Event() + first_install: Final = threading.Event() + late_install: Final = threading.Event() + + def provide_token() -> str: + if not first_mint.is_set(): + first_mint.set() + return "initial-token" + mint_blocked.set() + release_mint.wait(10) + return "late-token" + + def refresher_factory(source: PgBouncerTokenSource, install: Callable[[str], None]) -> PgBouncerTokenRefresher: + def observe_install(token: str) -> None: + if first_install.is_set(): + late_install.set() + return + first_install.set() + install(token) + + return PgBouncerTokenRefresher(source, observe_install, fallback_seconds=0.01, retry_seconds=0.01) + + pooler_binary: Final = _write_script(tmp_path / "fake-pgbouncer", PGBOUNCER_SOURCE) + settings: Final = PgBouncerSettings(enabled=True, binary=str(pooler_binary), port=allocate_port()) + outcome: Final = start_managed_pgbouncer( + settings, + "postgresql://app@db/litellm", + token_auth=AzureEntraTokenAuth(token_provider=provide_token), + refresher_factory=refresher_factory, + ) + assert isinstance(outcome, ManagedPgBouncerRuntime), outcome + runtime: Final = outcome + refresher: Final = runtime.refresher + assert refresher is not None + try: + assert first_install.is_set() + assert mint_blocked.wait(3) + initial_pid: Final = runtime.pooler.pid + assert initial_pid is not None + os.kill(initial_pid, signal.SIGKILL) # test-quality-ok: crash the owned child to exercise watchdog replacement + replacement_pid: Final = _wait_for_new_pid(pid_file, initial_pid) + assert runtime.pooler.pid == replacement_pid + started: Final = time.monotonic() + stop_budget_seconds: Final = 0.23 + runtime.stop(started + stop_budget_seconds) + assert_enforced_within( + time.monotonic() - started, + configured=stop_budget_seconds, + slack=0.27, + ) + assert not Path(f"/proc/{replacement_pid}").exists() + assert not runtime.runtime_dir.exists() + release_mint.set() + refresher.stop(time.monotonic() + 2) + assert not late_install.is_set() + finally: + release_mint.set() + runtime.stop(time.monotonic() + 5) From 14fab5ed41d80e3e0d125f6ad26952a9a185c0cd Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 00:40:36 +0000 Subject: [PATCH 2/2] test(gateway): verify PgBouncer shutdown ownership --- tests/unit/proxy/db/test_pgbouncer.py | 47 +++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/unit/proxy/db/test_pgbouncer.py b/tests/unit/proxy/db/test_pgbouncer.py index c69c8d015a3..082a58db870 100644 --- a/tests/unit/proxy/db/test_pgbouncer.py +++ b/tests/unit/proxy/db/test_pgbouncer.py @@ -23,6 +23,7 @@ import pytest from litellm._logging import verbose_proxy_logger from litellm.proxy.db.pgbouncer import ( PGBOUNCER_POOLED_ENV_VAR, + ManagedPgBouncerRuntime, PgBouncerError, PgBouncerPlan, PgBouncerProcess, @@ -32,6 +33,7 @@ from litellm.proxy.db.pgbouncer import ( database_url_is_pooled, export_pooled_database_url, install_pgbouncer_token, + pgbouncer_startup_signals, pgbouncer_version, plan_pgbouncer, start_in_container_pgbouncer, @@ -603,6 +605,51 @@ def _token_source(*tokens: str | Exception) -> PgBouncerTokenSource: return PgBouncerTokenSource(auth=AzureEntraTokenAuth(token_provider=provide), endpoint=ENDPOINT) +@pytest.mark.skipif(sys.platform == "win32", reason="Windows does not deliver POSIX process signals") +@pytest.mark.parametrize("signum", (signal.SIGTERM, signal.SIGINT)) +@pytest.mark.parametrize("enabled", (False, True)) +def test_startup_signals_preserve_the_application_handler_after_interruption(signum: int, enabled: bool) -> None: + previous: Final = signal.signal(signum, signal.default_int_handler) + expected_exception: Final = SystemExit if enabled else KeyboardInterrupt + try: + with pytest.raises((SystemExit, KeyboardInterrupt)) as interrupted: + with pgbouncer_startup_signals(enabled): + signal.raise_signal(signum) + assert type(interrupted.value) is expected_exception + assert interrupted.value.args == ((128 + signum,) if enabled else ()) + with pytest.raises(KeyboardInterrupt): + signal.raise_signal(signum) + finally: + signal.signal(signum, previous) + + +def test_managed_shutdown_cancels_pending_token_install_and_restart(tmp_path: Path) -> None: + runtime_dir: Final = tmp_path / "runtime" + runtime_dir.mkdir() + auth_file: Final = runtime_dir / "userlist.txt" + + def install(token: str) -> None: + auth_file.write_text(token) + + refresher: Final = PgBouncerTokenRefresher(_token_source("before", "after"), install, fallback_seconds=600) + pooler: Final = PgBouncerProcess(argv=(), port=6543, socket_path=runtime_dir / "socket") + runtime: Final = ManagedPgBouncerRuntime( + pooled_url="postgresql://app:password@127.0.0.1:6543/litellm", + pooler=pooler, + refresher=refresher, + runtime_dir=runtime_dir, + ) + assert refresher.refresh() == 600 + assert auth_file.read_text() == "before" + with runtime.owned_pid_guard() as pid: + assert pid is None + runtime.stop(deadline=0.0) + assert not runtime_dir.exists() + assert refresher.refresh() == 600 + assert not runtime_dir.exists() + assert pooler.start() == PgBouncerError("pgbouncer was stopped before it started") + + class TestPgBouncerTokenRefresher: def _refresher( self,