This commit is contained in:
yujonglee 2026-10-26 12:12:04 -04:00 • committed by GitHub
commit 0c85038dc3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 737 additions and 58 deletions

View file

@ -21,9 +21,11 @@ from uvicorn.main import main as uvicorn_main
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
from litellm.proxy.db.pgbouncer import (
PGBOUNCER_MANAGED_SHUTDOWN_SECONDS,
PgBouncerError,
PgBouncerSettings,
export_pooled_database_url,
pgbouncer_startup_signals,
start_in_container_pgbouncer,
)
@ -55,7 +57,12 @@ def pool_database_url(
upstream_url: Final = environ.get("DATABASE_URL")
if upstream_url is None:
return PgBouncerError("LITELLM_PGBOUNCER_ENABLED is set but no DATABASE_URL could be assembled")
return start_in_container_pgbouncer(pgbouncer, upstream_url, token_auth=settings.token_auth())
return start_in_container_pgbouncer(
pgbouncer,
upstream_url,
token_auth=settings.token_auth(),
shutdown_timeout_seconds=PGBOUNCER_MANAGED_SHUTDOWN_SECONDS,
)
def _serve(argv: Sequence[str]) -> None:
@ -63,14 +70,16 @@ def _serve(argv: Sequence[str]) -> None:
def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None:
settings: Final = DatabaseURLSettings.from_env()
settings.apply_to_env()
pooled_url: Final = pool_database_url(settings, PgBouncerSettings(), os.environ)
if isinstance(pooled_url, PgBouncerError):
sys.exit(f"LiteLLM gateway: in-container pgbouncer could not start: {pooled_url.reason}")
if pooled_url is not None:
export_pooled_database_url(pooled_url)
serve(uvicorn_argv(argv, os.environ))
pgbouncer: Final = PgBouncerSettings()
with pgbouncer_startup_signals(pgbouncer.enabled):
settings: Final = DatabaseURLSettings.from_env()
settings.apply_to_env()
pooled_url: Final = pool_database_url(settings, pgbouncer, os.environ)
if isinstance(pooled_url, PgBouncerError):
sys.exit(f"LiteLLM gateway: in-container pgbouncer could not start: {pooled_url.reason}")
if pooled_url is not None:
export_pooled_database_url(pooled_url)
serve(uvicorn_argv(argv, os.environ))
if __name__ == "__main__":

View file

@ -46,12 +46,13 @@ import tempfile
import threading
import time
import urllib.parse
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass
from collections.abc import Callable, Generator, Mapping, Sequence
from contextlib import ExitStack, contextmanager
from dataclasses import dataclass, field
from datetime import datetime, timezone
from pathlib import Path
from types import MappingProxyType
from typing import Final
from types import FrameType, MappingProxyType
from typing import Final, NoReturn
from pydantic import Field
from pydantic_settings import BaseSettings, SettingsConfigDict
@ -75,6 +76,7 @@ PGBOUNCER_CA_NAME: Final = "server-ca.pem"
PGBOUNCER_RESTART_DELAY_SECONDS: Final = 1.0
PGBOUNCER_READY_TIMEOUT_SECONDS: Final = 15.0
PGBOUNCER_STOP_GRACE_SECONDS: Final = 10.0
PGBOUNCER_MANAGED_SHUTDOWN_SECONDS: Final = 30.0
PGBOUNCER_UNPRIVILEGED_USER: Final = "nobody"
PGBOUNCER_MIN_VERSION: Final = (1, 19)
PGBOUNCER_MAX_PASSWORD_BYTES: Final = 2048
@ -396,6 +398,8 @@ class PgBouncerTokenRefresher:
token: Final = self._source.mint()
except Exception as mint_error:
return PgBouncerError(f"could not mint a {label} for the in-container pgbouncer: {mint_error!r}")
if self._stopping.is_set():
return self._fallback_seconds
if len(token.encode()) >= PGBOUNCER_MAX_PASSWORD_BYTES:
return PgBouncerError(
f"the {label} is {len(token.encode())} bytes long, but PgBouncer's auth file holds passwords of at "
@ -440,10 +444,16 @@ class PgBouncerTokenRefresher:
)
return outcome
def stop(self) -> None:
def request_stop(self) -> None:
self._stopping.set()
def stop(self, deadline: float | None = None) -> None:
self.request_stop()
if self._thread is not None:
self._thread.join()
if deadline is None:
self._thread.join()
else:
self._thread.join(timeout=max(0.0, deadline - time.monotonic()))
def _port_open(port: int) -> bool:
@ -487,15 +497,27 @@ def pgbouncer_version(binary: str) -> tuple[int, int] | PgBouncerError:
return int(found[1]), int(found[2])
def _end(process: subprocess.Popen[bytes]) -> None:
def _end(process: subprocess.Popen[bytes], deadline: float | None = None) -> None:
if process.poll() is not None:
return
process.terminate()
try:
process.wait(timeout=PGBOUNCER_STOP_GRACE_SECONDS)
remaining: Final = None if deadline is None else max(0.0, deadline - time.monotonic())
timeout: Final = (
PGBOUNCER_STOP_GRACE_SECONDS
if remaining is None
else max(0.0, min(PGBOUNCER_STOP_GRACE_SECONDS, remaining - 1.0))
)
process.wait(timeout=timeout)
except subprocess.TimeoutExpired:
process.kill()
process.wait()
if deadline is None:
process.wait()
else:
try:
process.wait(timeout=max(0.0, deadline - time.monotonic()))
except subprocess.TimeoutExpired:
pass
class PgBouncerProcess:
@ -537,6 +559,12 @@ class PgBouncerProcess:
with self._lock:
return None if self._process is None else self._process.pid
@contextmanager
def owned_pid_guard(self) -> Generator[int | None, None, None]:
with self._lock:
process: Final = self._process
yield None if process is None or process.poll() is not None else process.pid
def _spawn(self) -> subprocess.Popen[bytes] | PgBouncerError | None:
"""Start a child, or None once ``stop`` ran; both take the lock so no child can slip in after a stop.
@ -572,7 +600,7 @@ class PgBouncerProcess:
f"within {self.ready_timeout_seconds:.0f}s"
)
def start(self) -> PgBouncerError | None:
def start(self, deadline: float | None = None) -> PgBouncerError | None:
"""Spawn PgBouncer, wait until it listens on port and unix socket, then supervise it from a daemon thread."""
process: Final = self._spawn()
if process is None:
@ -581,7 +609,7 @@ class PgBouncerProcess:
return process
not_ready: Final = self._wait_ready(process)
if not_ready is not None:
self.stop()
self.stop(deadline)
return not_ready
self._watch(process)
return None
@ -631,12 +659,12 @@ class PgBouncerProcess:
if self._process is not None:
self._process.send_signal(signal.SIGHUP)
def stop(self) -> None:
def stop(self, deadline: float | None = None) -> None:
with self._lock:
self._stopping.set()
process: Final = self._process
if process is not None:
_end(process)
_end(process, deadline)
def install_pgbouncer_token(
@ -653,6 +681,10 @@ def _install_upstream_password(
pooler: PgBouncerProcess,
token_auth: DatabaseTokenAuth | None,
upstream_url: str,
startup_cleanup: ExitStack,
refresher_factory: Callable[
[PgBouncerTokenSource, Callable[[str], None]], PgBouncerTokenRefresher
] = PgBouncerTokenRefresher,
) -> PgBouncerTokenRefresher | None | PgBouncerError:
if token_auth is None:
if plan.upstream_password is None:
@ -662,10 +694,11 @@ def _install_upstream_password(
)
write_userlist(plan.userlist(plan.upstream_password), runtime_dir, run_as_user)
return None
refresher: Final = PgBouncerTokenRefresher(
refresher: Final = refresher_factory(
PgBouncerTokenSource(auth=token_auth, endpoint=parse_iam_endpoint_from_url(upstream_url)),
functools.partial(install_pgbouncer_token, plan, runtime_dir, run_as_user, pooler),
)
startup_cleanup.callback(_stop_managed_refresher, refresher)
failed: Final = refresher.start()
if failed is not None:
return failed
@ -683,11 +716,130 @@ def _only_in_this_process(action: Callable[[], None]) -> Callable[[], None]:
return run
def _raise_for_startup_signal(signum: int, _frame: FrameType | None) -> NoReturn:
raise SystemExit(128 + signum)
@contextmanager
def pgbouncer_startup_signals(enabled: bool) -> Generator[None, None, None]:
if not enabled:
yield
return
previous_term: Final = signal.getsignal(signal.SIGTERM)
previous_int: Final = signal.getsignal(signal.SIGINT)
signal.signal(signal.SIGTERM, _raise_for_startup_signal)
try:
signal.signal(signal.SIGINT, _raise_for_startup_signal)
try:
yield
finally:
signal.signal(signal.SIGINT, previous_int)
finally:
signal.signal(signal.SIGTERM, previous_term)
def _stop_managed_pooler(pooler: PgBouncerProcess) -> None:
pooler.stop(time.monotonic() + PGBOUNCER_STOP_GRACE_SECONDS)
def _stop_managed_refresher(refresher: PgBouncerTokenRefresher) -> None:
refresher.stop(time.monotonic() + PGBOUNCER_STOP_GRACE_SECONDS)
@dataclass(frozen=True, slots=True)
class ManagedPgBouncerRuntime:
pooled_url: str = field(repr=False)
pooler: PgBouncerProcess
refresher: PgBouncerTokenRefresher | None
runtime_dir: Path
@contextmanager
def owned_pid_guard(self) -> Generator[int | None, None, None]:
with self.pooler.owned_pid_guard() as pid:
yield pid
def stop(self, deadline: float | None = None) -> None:
if self.refresher is not None:
self.refresher.request_stop()
try:
self.pooler.stop(deadline)
finally:
try:
if self.refresher is not None:
self.refresher.stop(deadline)
finally:
shutil.rmtree(self.runtime_dir, ignore_errors=True)
def start_managed_pgbouncer(
settings: PgBouncerSettings,
upstream_url: str,
token_auth: DatabaseTokenAuth | None = None,
refresher_factory: Callable[
[PgBouncerTokenSource, Callable[[str], None]], PgBouncerTokenRefresher
] = PgBouncerTokenRefresher,
) -> ManagedPgBouncerRuntime | PgBouncerError:
version: Final = pgbouncer_version(settings.binary)
if isinstance(version, PgBouncerError):
return version
if version < PGBOUNCER_MIN_VERSION:
return PgBouncerError(
f"PgBouncer {version[0]}.{version[1]} keeps running after failing to bind its TCP port, so the proxy "
f"cannot tell it apart from another listener; {PGBOUNCER_MIN_VERSION[0]}.{PGBOUNCER_MIN_VERSION[1]} "
"or newer is required"
)
with ExitStack() as startup_cleanup:
runtime_dir: Final = Path(tempfile.mkdtemp(prefix="litellm-pgbouncer-"))
startup_cleanup.callback(shutil.rmtree, runtime_dir, ignore_errors=True)
run_as_user: Final = PGBOUNCER_UNPRIVILEGED_USER if os.geteuid() == 0 else None
plan: Final = plan_pgbouncer(upstream_url, settings, runtime_dir, run_as_user)
if isinstance(plan, PgBouncerError):
return plan
ini_path: Final = write_pgbouncer_ini(plan, runtime_dir, run_as_user)
if isinstance(ini_path, PgBouncerError):
return ini_path
pooler: Final = PgBouncerProcess(
argv=(settings.binary, str(ini_path)),
port=settings.port,
socket_path=unix_socket_path(runtime_dir, settings.port),
)
startup_cleanup.callback(_stop_managed_pooler, pooler)
refresher_result: Final = _install_upstream_password(
plan,
runtime_dir,
run_as_user,
pooler,
token_auth,
upstream_url,
startup_cleanup,
refresher_factory,
)
if isinstance(refresher_result, PgBouncerError):
return refresher_result
failed: Final = pooler.start(time.monotonic() + PGBOUNCER_STOP_GRACE_SECONDS)
if failed is not None:
return failed
runtime: Final = ManagedPgBouncerRuntime(
pooled_url=plan.pooled_url,
pooler=pooler,
refresher=refresher_result,
runtime_dir=runtime_dir,
)
startup_cleanup.pop_all()
return runtime
def _stop_managed_runtime_before_exit(runtime: ManagedPgBouncerRuntime, timeout_seconds: float) -> None:
runtime.stop(time.monotonic() + timeout_seconds)
def start_in_container_pgbouncer(
settings: PgBouncerSettings,
upstream_url: str,
token_auth: DatabaseTokenAuth | None = None,
register_exit_hook: Callable[[Callable[[], None]], object] = atexit.register,
*,
shutdown_timeout_seconds: float | None = None,
) -> str | PgBouncerError:
"""Start the pooler for ``upstream_url`` and return the loopback URL the workers must use.
@ -698,46 +850,21 @@ def start_in_container_pgbouncer(
``nobody``. With ``token_auth`` the password on ``upstream_url`` is ignored:
the pooler mints its own tokens and renews them for as long as it runs.
"""
version: Final = pgbouncer_version(settings.binary)
if isinstance(version, PgBouncerError):
return version
if version < PGBOUNCER_MIN_VERSION:
return PgBouncerError(
f"PgBouncer {version[0]}.{version[1]} keeps running after failing to bind its TCP port, so the proxy "
f"cannot tell it apart from another listener; {PGBOUNCER_MIN_VERSION[0]}.{PGBOUNCER_MIN_VERSION[1]} "
"or newer is required"
)
runtime_dir: Final = Path(tempfile.mkdtemp(prefix="litellm-pgbouncer-"))
register_exit_hook(_only_in_this_process(lambda: shutil.rmtree(runtime_dir, ignore_errors=True)))
run_as_user: Final = PGBOUNCER_UNPRIVILEGED_USER if os.geteuid() == 0 else None
plan: Final = plan_pgbouncer(upstream_url, settings, runtime_dir, run_as_user)
if isinstance(plan, PgBouncerError):
return plan
ini_path: Final = write_pgbouncer_ini(plan, runtime_dir, run_as_user)
if isinstance(ini_path, PgBouncerError):
return ini_path
pooler: Final = PgBouncerProcess(
argv=(settings.binary, str(ini_path)),
port=settings.port,
socket_path=unix_socket_path(runtime_dir, settings.port),
runtime: Final = start_managed_pgbouncer(settings, upstream_url, token_auth)
if isinstance(runtime, PgBouncerError):
return runtime
cleanup: Final = (
runtime.stop
if shutdown_timeout_seconds is None
else functools.partial(_stop_managed_runtime_before_exit, runtime, shutdown_timeout_seconds)
)
refresher: Final = _install_upstream_password(plan, runtime_dir, run_as_user, pooler, token_auth, upstream_url)
if isinstance(refresher, PgBouncerError):
return refresher
failed: Final = pooler.start()
if failed is not None:
if refresher is not None:
refresher.stop()
return failed
register_exit_hook(_only_in_this_process(pooler.stop))
if refresher is not None:
register_exit_hook(_only_in_this_process(refresher.stop))
register_exit_hook(_only_in_this_process(cleanup))
verbose_proxy_logger.info(
"In-container pgbouncer (pid %s) listening on %s:%s; capping this pod at %s upstream database connections%s.",
pooler.pid,
runtime.pooler.pid,
PGBOUNCER_LISTEN_ADDR,
settings.port,
settings.max_db_connections,
"" if token_auth is None else f" and renewing its {token_auth.label} before each one expires",
)
return plan.pooled_url
return runtime.pooled_url

View file

@ -0,0 +1,496 @@
import http.client
import json
import os
import signal
import subprocess
import sys
import textwrap
import threading
import time
from pathlib import Path
from typing import Final, cast
import pytest
from tests.integration._support.ports import allocate_port
from tests.integration._support.timing import assert_enforced_within
ROOT: Final = Path(os.getenv("LITELLM_TEST_ROOT", str(Path(__file__).resolve().parents[3])))
LAUNCHER_SOURCE: Final = """\
import ctypes
import json
import os
import shutil
import signal
import socket
import subprocess
import sys
import time
from pathlib import Path
PR_SET_CHILD_SUBREAPER = 36
if ctypes.CDLL(None, use_errno=True).prctl(PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0) != 0:
raise OSError(ctypes.get_errno(), "prctl(PR_SET_CHILD_SUBREAPER) failed")
root, launcher_pid_file, pooler_pid_file, config_file, port_file, result_file, *command = sys.argv[1:]
started = time.monotonic()
launcher = subprocess.Popen(command, cwd=root, env=os.environ.copy())
Path(launcher_pid_file).write_text(str(launcher.pid))
status = launcher.wait()
try:
pooler_pid = int(Path(pooler_pid_file).read_text())
except (OSError, ValueError):
pooler_pid = None
try:
config_path = Path(Path(config_file).read_text())
except OSError:
config_path = None
try:
port = int(Path(port_file).read_text())
except (OSError, ValueError):
port = None
pooler_present = pooler_pid is not None and Path(f"/proc/{pooler_pid}").exists()
listener_open = False
if port is not None:
try:
with socket.create_connection(("127.0.0.1", port), timeout=0.2):
listener_open = True
except OSError:
pass
result = {
"launcher_exit_code": status,
"elapsed": time.monotonic() - started,
"pooler_pid": pooler_pid,
"pooler_present": pooler_present,
"listener_open": listener_open,
"runtime_dir_present": config_path is not None and config_path.parent.exists(),
}
Path(result_file).write_text(json.dumps(result))
if pooler_pid is not None and Path(f"/proc/{pooler_pid}").exists():
try:
os.kill(pooler_pid, signal.SIGKILL)
except ProcessLookupError:
pass
try:
os.waitpid(pooler_pid, 0)
except ChildProcessError:
pass
if config_path is not None:
shutil.rmtree(config_path.parent, ignore_errors=True)
raise SystemExit(status)
"""
PGBOUNCER_SOURCE: Final = """\
import configparser
import os
import select
import signal
import socket
import sys
import time
from pathlib import Path
signal.signal(signal.SIGTERM, signal.SIG_IGN)
signal.signal(signal.SIGINT, signal.SIG_IGN)
signal.signal(signal.SIGHUP, signal.SIG_IGN)
if sys.argv[1:] == ["--version"]:
print("PgBouncer 1.26.0")
raise SystemExit(0)
config_path = Path(sys.argv[1])
parser = configparser.ConfigParser()
parser.read(config_path)
port = parser.getint("pgbouncer", "listen_port")
socket_path = Path(parser.get("pgbouncer", "unix_socket_dir")) / f".s.PGSQL.{port}"
Path(os.environ["LITELLM_TEST_POOLER_PID_FILE"]).write_text(str(os.getpid()))
Path(os.environ["LITELLM_TEST_POOLER_CONFIG_FILE"]).write_text(str(config_path))
Path(os.environ["LITELLM_TEST_POOLER_PORT_FILE"]).write_text(str(port))
if os.environ["LITELLM_TEST_POOLER_MODE"] == "slow":
while True:
signal.pause()
if os.environ["LITELLM_TEST_POOLER_MODE"] == "exit":
time.sleep(0.25)
raise SystemExit(3)
tcp = socket.socket()
tcp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
tcp.bind(("127.0.0.1", port))
tcp.listen()
unix = socket.socket(socket.AF_UNIX)
unix.bind(str(socket_path))
unix.listen()
while True:
for listener in select.select((tcp, unix), (), ())[0]:
listener.accept()[0].close()
"""
SERVE_SOURCE: Final = """\
import sys
import uvicorn
from gateway.launch import main
async def app(scope, receive, send):
if scope["type"] == "http":
await send({"type": "http.response.start", "status": 200, "headers": []})
await send({"type": "http.response.body", "body": b"ready"})
def serve(argv):
port = int(argv[argv.index("--port") + 1])
uvicorn.Server(
uvicorn.Config(app, host="127.0.0.1", port=port, lifespan="off", workers=1)
).run()
main(sys.argv[1:], serve=serve)
"""
def _write_script(path: Path, source: str) -> Path:
path.write_text(f"#!/usr/bin/env python3\n{textwrap.dedent(source)}")
path.chmod(0o700)
return path
def _wait_for_file(path: Path, timeout_seconds: float = 15.0) -> str:
deadline: Final = time.monotonic() + timeout_seconds
while time.monotonic() < deadline:
try:
return path.read_text()
except OSError:
time.sleep(0.05)
pytest.fail(f"timed out waiting for {path}")
def _wait_for_new_pid(path: Path, previous_pid: int, timeout_seconds: float = 5.0) -> int:
deadline: Final = time.monotonic() + timeout_seconds
while time.monotonic() < deadline:
try:
pid: Final = int(path.read_text())
if pid != previous_pid:
return pid
except (OSError, ValueError):
pass
time.sleep(0.05)
pytest.fail(f"timed out waiting for a replacement process in {path}")
def _http_health(port: int) -> bool:
try:
connection: Final = http.client.HTTPConnection("127.0.0.1", port, timeout=0.5)
connection.request("GET", "/health/liveliness")
response: Final = connection.getresponse()
response.read()
connection.close()
return response.status == 200
except (OSError, http.client.HTTPException):
return False
def _wait_for_health(port: int, timeout_seconds: float = 15.0) -> None:
deadline: Final = time.monotonic() + timeout_seconds
while time.monotonic() < deadline:
if _http_health(port):
return
time.sleep(0.05)
pytest.fail(f"gateway did not become healthy on port {port}")
def _run_black_box(
tmp_path: Path, pooler_mode: str, signal_number: signal.Signals, require_health: bool
) -> dict[str, object]:
launcher_pid_file: Final = tmp_path / "launcher.pid"
pooler_pid_file: Final = tmp_path / "pooler.pid"
config_file: Final = tmp_path / "pooler-config"
port_file: Final = tmp_path / "pooler-port"
result_file: Final = tmp_path / "result.json"
pooler_port: Final = allocate_port()
gateway_port: Final = allocate_port()
pooler_binary: Final = _write_script(tmp_path / "fake-pgbouncer", PGBOUNCER_SOURCE)
wrapper: Final = _write_script(tmp_path / "subreaper-wrapper.py", LAUNCHER_SOURCE)
environment: Final = {
**os.environ,
"PYTHONPATH": os.pathsep.join(filter(None, (str(ROOT), os.environ.get("PYTHONPATH", "")))),
"DATABASE_URL": "postgresql://app:password@127.0.0.1:5432/litellm",
"LITELLM_PGBOUNCER_ENABLED": "true",
"LITELLM_PGBOUNCER_BINARY": str(pooler_binary),
"LITELLM_PGBOUNCER_PORT": str(pooler_port),
"LITELLM_TEST_POOLER_MODE": pooler_mode,
"LITELLM_TEST_POOLER_PID_FILE": str(pooler_pid_file),
"LITELLM_TEST_POOLER_CONFIG_FILE": str(config_file),
"LITELLM_TEST_POOLER_PORT_FILE": str(port_file),
}
serve_script: Final = _write_script(tmp_path / "serve-gateway.py", SERVE_SOURCE)
command: Final = (
(
sys.executable,
str(serve_script),
"--host",
"127.0.0.1",
"--port",
str(gateway_port),
"--workers",
"1",
)
if require_health
else (
sys.executable,
"-m",
"gateway.launch",
"--host",
"127.0.0.1",
"--port",
str(gateway_port),
"--workers",
"1",
)
)
launch_log: Final = (tmp_path / "gateway.log").open("w")
wrapper_process: Final = subprocess.Popen(
(
sys.executable,
"-P",
str(wrapper),
str(ROOT),
str(launcher_pid_file),
str(pooler_pid_file),
str(config_file),
str(port_file),
str(result_file),
*command,
),
cwd=ROOT,
env=environment,
stdin=subprocess.DEVNULL,
stdout=launch_log,
stderr=subprocess.STDOUT,
start_new_session=True,
)
try:
launcher_pid: Final = int(_wait_for_file(launcher_pid_file))
_wait_for_file(pooler_pid_file)
if require_health:
try:
_wait_for_health(gateway_port)
except BaseException as error:
raise AssertionError((tmp_path / "gateway.log").read_text()) from error
os.kill(launcher_pid, signal_number) # test-quality-ok: signal starts the lifecycle under test
wrapper_process.wait(timeout=35)
return cast(dict[str, object], json.loads(result_file.read_text()))
finally:
if wrapper_process.poll() is None:
try:
launcher_pid = int(launcher_pid_file.read_text())
os.kill(launcher_pid, signal.SIGKILL) # test-quality-ok: bounded fixture cleanup
except (OSError, ValueError):
pass
try:
wrapper_process.wait(timeout=15)
except subprocess.TimeoutExpired:
os.kill(wrapper_process.pid, signal.SIGKILL) # test-quality-ok: bounded fixture cleanup
wrapper_process.wait()
launch_log.close()
@pytest.mark.parametrize(
("signal_number", "expected_exit"),
((signal.SIGTERM, 143), (signal.SIGINT, 130)),
)
def test_startup_signal_removes_slow_pooler_and_runtime(
tmp_path: Path, signal_number: signal.Signals, expected_exit: int
) -> None:
result: Final = _run_black_box(tmp_path, "slow", signal_number, require_health=False)
assert result["pooler_present"] is False, result
assert result["listener_open"] is False, result
assert result["runtime_dir_present"] is False, result
assert result["launcher_exit_code"] == expected_exit, result
assert cast(float, result["elapsed"]) <= 30
@pytest.mark.parametrize(
("signal_number", "expected_exit"),
((signal.SIGTERM, 143), (signal.SIGINT, 130)),
)
def test_running_gateway_signal_removes_pooler_and_runtime(
tmp_path: Path, signal_number: signal.Signals, expected_exit: int
) -> None:
result: Final = _run_black_box(tmp_path, "serve", signal_number, require_health=True)
assert result["pooler_present"] is False, result
assert result["listener_open"] is False, result
assert result["runtime_dir_present"] is False, result
assert result["launcher_exit_code"] == expected_exit, result
assert cast(float, result["elapsed"]) <= 30
def test_managed_pooler_stop_reaps_owned_pid_and_runtime(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
from litellm.proxy.db.pgbouncer import ManagedPgBouncerRuntime, PgBouncerSettings, start_managed_pgbouncer
pid_file: Final = tmp_path / "pooler.pid"
config_file: Final = tmp_path / "pooler-config"
port_file: Final = tmp_path / "pooler-port"
pooler_binary: Final = _write_script(tmp_path / "fake-pgbouncer", PGBOUNCER_SOURCE)
settings: Final = PgBouncerSettings(enabled=True, binary=str(pooler_binary), port=allocate_port())
environment: Final = {
"LITELLM_TEST_POOLER_MODE": "serve",
"LITELLM_TEST_POOLER_PID_FILE": str(pid_file),
"LITELLM_TEST_POOLER_CONFIG_FILE": str(config_file),
"LITELLM_TEST_POOLER_PORT_FILE": str(port_file),
}
for key, value in environment.items():
monkeypatch.setenv(key, value)
runtime_result: Final = start_managed_pgbouncer(settings, "postgresql://app:password@db/litellm")
assert isinstance(runtime_result, ManagedPgBouncerRuntime), runtime_result
runtime: Final = runtime_result
pid: Final = runtime.pooler.pid
try:
assert pid is not None
with runtime.owned_pid_guard() as guarded_pid:
assert guarded_pid == pid
finally:
runtime.stop(time.monotonic() + 15)
assert pid is not None
assert not Path(f"/proc/{pid}").exists()
assert not runtime.runtime_dir.exists()
def test_pooler_start_failure_stops_injected_fast_refresher(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
from collections.abc import Callable
from litellm.proxy.db.pgbouncer import (
PgBouncerError,
PgBouncerSettings,
PgBouncerTokenRefresher,
PgBouncerTokenSource,
start_managed_pgbouncer,
)
from litellm.proxy.db.token_auth import AzureEntraTokenAuth
pid_file: Final = tmp_path / "pooler.pid"
config_file: Final = tmp_path / "pooler-config"
port_file: Final = tmp_path / "pooler-port"
pooler_binary: Final = _write_script(tmp_path / "fake-pgbouncer", PGBOUNCER_SOURCE)
settings: Final = PgBouncerSettings(enabled=True, binary=str(pooler_binary), port=allocate_port())
monkeypatch.setenv("LITELLM_TEST_POOLER_MODE", "exit")
monkeypatch.setenv("LITELLM_TEST_POOLER_PID_FILE", str(pid_file))
monkeypatch.setenv("LITELLM_TEST_POOLER_CONFIG_FILE", str(config_file))
monkeypatch.setenv("LITELLM_TEST_POOLER_PORT_FILE", str(port_file))
first_mint: Final = threading.Event()
refreshed_during_start: Final = threading.Event()
start_failed: Final = threading.Event()
refreshed_after_failure: Final = threading.Event()
initial_install_done: Final = threading.Event()
def provide_token() -> str:
if first_mint.is_set():
refreshed_during_start.set()
if start_failed.is_set():
refreshed_after_failure.set()
else:
first_mint.set()
return "opaque-token"
def fast_refresher_factory(source: PgBouncerTokenSource, install: Callable[[str], None]) -> PgBouncerTokenRefresher:
def install_once(token: str) -> None:
if initial_install_done.is_set():
return
install(token)
initial_install_done.set()
return PgBouncerTokenRefresher(source, install_once, fallback_seconds=0.02, retry_seconds=0.02)
outcome: Final = start_managed_pgbouncer(
settings,
"postgresql://app@db/litellm",
token_auth=AzureEntraTokenAuth(token_provider=provide_token),
refresher_factory=fast_refresher_factory,
)
assert isinstance(outcome, PgBouncerError), outcome
assert pid_file.exists(), outcome
assert refreshed_during_start.wait(2)
start_failed.set()
time.sleep(0.1)
assert not refreshed_after_failure.is_set()
runtime_dir: Final = Path(config_file.read_text()).parent
assert not runtime_dir.exists()
def test_managed_stop_reaps_replacement_before_refresher_deadline_and_skips_late_install(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
from collections.abc import Callable
from litellm.proxy.db.pgbouncer import (
ManagedPgBouncerRuntime,
PgBouncerSettings,
PgBouncerTokenRefresher,
PgBouncerTokenSource,
start_managed_pgbouncer,
)
from litellm.proxy.db.token_auth import AzureEntraTokenAuth
pid_file: Final = tmp_path / "pooler.pid"
config_file: Final = tmp_path / "pooler-config"
port_file: Final = tmp_path / "pooler-port"
monkeypatch.setenv("LITELLM_TEST_POOLER_MODE", "serve")
monkeypatch.setenv("LITELLM_TEST_POOLER_PID_FILE", str(pid_file))
monkeypatch.setenv("LITELLM_TEST_POOLER_CONFIG_FILE", str(config_file))
monkeypatch.setenv("LITELLM_TEST_POOLER_PORT_FILE", str(port_file))
first_mint: Final = threading.Event()
mint_blocked: Final = threading.Event()
release_mint: Final = threading.Event()
first_install: Final = threading.Event()
late_install: Final = threading.Event()
def provide_token() -> str:
if not first_mint.is_set():
first_mint.set()
return "initial-token"
mint_blocked.set()
release_mint.wait(10)
return "late-token"
def refresher_factory(source: PgBouncerTokenSource, install: Callable[[str], None]) -> PgBouncerTokenRefresher:
def observe_install(token: str) -> None:
if first_install.is_set():
late_install.set()
return
first_install.set()
install(token)
return PgBouncerTokenRefresher(source, observe_install, fallback_seconds=0.01, retry_seconds=0.01)
pooler_binary: Final = _write_script(tmp_path / "fake-pgbouncer", PGBOUNCER_SOURCE)
settings: Final = PgBouncerSettings(enabled=True, binary=str(pooler_binary), port=allocate_port())
outcome: Final = start_managed_pgbouncer(
settings,
"postgresql://app@db/litellm",
token_auth=AzureEntraTokenAuth(token_provider=provide_token),
refresher_factory=refresher_factory,
)
assert isinstance(outcome, ManagedPgBouncerRuntime), outcome
runtime: Final = outcome
refresher: Final = runtime.refresher
assert refresher is not None
try:
assert first_install.is_set()
assert mint_blocked.wait(3)
initial_pid: Final = runtime.pooler.pid
assert initial_pid is not None
os.kill(initial_pid, signal.SIGKILL) # test-quality-ok: crash the owned child to exercise watchdog replacement
replacement_pid: Final = _wait_for_new_pid(pid_file, initial_pid)
assert runtime.pooler.pid == replacement_pid
started: Final = time.monotonic()
stop_budget_seconds: Final = 0.23
runtime.stop(started + stop_budget_seconds)
assert_enforced_within(
time.monotonic() - started,
configured=stop_budget_seconds,
slack=0.27,
)
assert not Path(f"/proc/{replacement_pid}").exists()
assert not runtime.runtime_dir.exists()
release_mint.set()
refresher.stop(time.monotonic() + 2)
assert not late_install.is_set()
finally:
release_mint.set()
runtime.stop(time.monotonic() + 5)

View file

@ -23,6 +23,7 @@ import pytest
from litellm._logging import verbose_proxy_logger
from litellm.proxy.db.pgbouncer import (
PGBOUNCER_POOLED_ENV_VAR,
ManagedPgBouncerRuntime,
PgBouncerError,
PgBouncerPlan,
PgBouncerProcess,
@ -32,6 +33,7 @@ from litellm.proxy.db.pgbouncer import (
database_url_is_pooled,
export_pooled_database_url,
install_pgbouncer_token,
pgbouncer_startup_signals,
pgbouncer_version,
plan_pgbouncer,
start_in_container_pgbouncer,
@ -603,6 +605,51 @@ def _token_source(*tokens: str | Exception) -> PgBouncerTokenSource:
return PgBouncerTokenSource(auth=AzureEntraTokenAuth(token_provider=provide), endpoint=ENDPOINT)
@pytest.mark.skipif(sys.platform == "win32", reason="Windows does not deliver POSIX process signals")
@pytest.mark.parametrize("signum", (signal.SIGTERM, signal.SIGINT))
@pytest.mark.parametrize("enabled", (False, True))
def test_startup_signals_preserve_the_application_handler_after_interruption(signum: int, enabled: bool) -> None:
previous: Final = signal.signal(signum, signal.default_int_handler)
expected_exception: Final = SystemExit if enabled else KeyboardInterrupt
try:
with pytest.raises((SystemExit, KeyboardInterrupt)) as interrupted:
with pgbouncer_startup_signals(enabled):
signal.raise_signal(signum)
assert type(interrupted.value) is expected_exception
assert interrupted.value.args == ((128 + signum,) if enabled else ())
with pytest.raises(KeyboardInterrupt):
signal.raise_signal(signum)
finally:
signal.signal(signum, previous)
def test_managed_shutdown_cancels_pending_token_install_and_restart(tmp_path: Path) -> None:
runtime_dir: Final = tmp_path / "runtime"
runtime_dir.mkdir()
auth_file: Final = runtime_dir / "userlist.txt"
def install(token: str) -> None:
auth_file.write_text(token)
refresher: Final = PgBouncerTokenRefresher(_token_source("before", "after"), install, fallback_seconds=600)
pooler: Final = PgBouncerProcess(argv=(), port=6543, socket_path=runtime_dir / "socket")
runtime: Final = ManagedPgBouncerRuntime(
pooled_url="postgresql://app:password@127.0.0.1:6543/litellm",
pooler=pooler,
refresher=refresher,
runtime_dir=runtime_dir,
)
assert refresher.refresh() == 600
assert auth_file.read_text() == "before"
with runtime.owned_pid_guard() as pid:
assert pid is None
runtime.stop(deadline=0.0)
assert not runtime_dir.exists()
assert refresher.refresh() == 600
assert not runtime_dir.exists()
assert pooler.start() == PgBouncerError("pgbouncer was stopped before it started")
class TestPgBouncerTokenRefresher:
def _refresher(
self,