mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge 14fab5ed41 into fd41b626b6
This commit is contained in:
commit
0c85038dc3
4 changed files with 737 additions and 58 deletions
|
|
@ -21,9 +21,11 @@ from uvicorn.main import main as uvicorn_main
|
|||
|
||||
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
|
||||
from litellm.proxy.db.pgbouncer import (
|
||||
PGBOUNCER_MANAGED_SHUTDOWN_SECONDS,
|
||||
PgBouncerError,
|
||||
PgBouncerSettings,
|
||||
export_pooled_database_url,
|
||||
pgbouncer_startup_signals,
|
||||
start_in_container_pgbouncer,
|
||||
)
|
||||
|
||||
|
|
@ -55,7 +57,12 @@ def pool_database_url(
|
|||
upstream_url: Final = environ.get("DATABASE_URL")
|
||||
if upstream_url is None:
|
||||
return PgBouncerError("LITELLM_PGBOUNCER_ENABLED is set but no DATABASE_URL could be assembled")
|
||||
return start_in_container_pgbouncer(pgbouncer, upstream_url, token_auth=settings.token_auth())
|
||||
return start_in_container_pgbouncer(
|
||||
pgbouncer,
|
||||
upstream_url,
|
||||
token_auth=settings.token_auth(),
|
||||
shutdown_timeout_seconds=PGBOUNCER_MANAGED_SHUTDOWN_SECONDS,
|
||||
)
|
||||
|
||||
|
||||
def _serve(argv: Sequence[str]) -> None:
|
||||
|
|
@ -63,14 +70,16 @@ def _serve(argv: Sequence[str]) -> None:
|
|||
|
||||
|
||||
def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None:
|
||||
settings: Final = DatabaseURLSettings.from_env()
|
||||
settings.apply_to_env()
|
||||
pooled_url: Final = pool_database_url(settings, PgBouncerSettings(), os.environ)
|
||||
if isinstance(pooled_url, PgBouncerError):
|
||||
sys.exit(f"LiteLLM gateway: in-container pgbouncer could not start: {pooled_url.reason}")
|
||||
if pooled_url is not None:
|
||||
export_pooled_database_url(pooled_url)
|
||||
serve(uvicorn_argv(argv, os.environ))
|
||||
pgbouncer: Final = PgBouncerSettings()
|
||||
with pgbouncer_startup_signals(pgbouncer.enabled):
|
||||
settings: Final = DatabaseURLSettings.from_env()
|
||||
settings.apply_to_env()
|
||||
pooled_url: Final = pool_database_url(settings, pgbouncer, os.environ)
|
||||
if isinstance(pooled_url, PgBouncerError):
|
||||
sys.exit(f"LiteLLM gateway: in-container pgbouncer could not start: {pooled_url.reason}")
|
||||
if pooled_url is not None:
|
||||
export_pooled_database_url(pooled_url)
|
||||
serve(uvicorn_argv(argv, os.environ))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -46,12 +46,13 @@ import tempfile
|
|||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from collections.abc import Callable, Generator, Mapping, Sequence
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from types import FrameType, MappingProxyType
|
||||
from typing import Final, NoReturn
|
||||
|
||||
from pydantic import Field
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
|
@ -75,6 +76,7 @@ PGBOUNCER_CA_NAME: Final = "server-ca.pem"
|
|||
PGBOUNCER_RESTART_DELAY_SECONDS: Final = 1.0
|
||||
PGBOUNCER_READY_TIMEOUT_SECONDS: Final = 15.0
|
||||
PGBOUNCER_STOP_GRACE_SECONDS: Final = 10.0
|
||||
PGBOUNCER_MANAGED_SHUTDOWN_SECONDS: Final = 30.0
|
||||
PGBOUNCER_UNPRIVILEGED_USER: Final = "nobody"
|
||||
PGBOUNCER_MIN_VERSION: Final = (1, 19)
|
||||
PGBOUNCER_MAX_PASSWORD_BYTES: Final = 2048
|
||||
|
|
@ -396,6 +398,8 @@ class PgBouncerTokenRefresher:
|
|||
token: Final = self._source.mint()
|
||||
except Exception as mint_error:
|
||||
return PgBouncerError(f"could not mint a {label} for the in-container pgbouncer: {mint_error!r}")
|
||||
if self._stopping.is_set():
|
||||
return self._fallback_seconds
|
||||
if len(token.encode()) >= PGBOUNCER_MAX_PASSWORD_BYTES:
|
||||
return PgBouncerError(
|
||||
f"the {label} is {len(token.encode())} bytes long, but PgBouncer's auth file holds passwords of at "
|
||||
|
|
@ -440,10 +444,16 @@ class PgBouncerTokenRefresher:
|
|||
)
|
||||
return outcome
|
||||
|
||||
def stop(self) -> None:
|
||||
def request_stop(self) -> None:
|
||||
self._stopping.set()
|
||||
|
||||
def stop(self, deadline: float | None = None) -> None:
|
||||
self.request_stop()
|
||||
if self._thread is not None:
|
||||
self._thread.join()
|
||||
if deadline is None:
|
||||
self._thread.join()
|
||||
else:
|
||||
self._thread.join(timeout=max(0.0, deadline - time.monotonic()))
|
||||
|
||||
|
||||
def _port_open(port: int) -> bool:
|
||||
|
|
@ -487,15 +497,27 @@ def pgbouncer_version(binary: str) -> tuple[int, int] | PgBouncerError:
|
|||
return int(found[1]), int(found[2])
|
||||
|
||||
|
||||
def _end(process: subprocess.Popen[bytes]) -> None:
|
||||
def _end(process: subprocess.Popen[bytes], deadline: float | None = None) -> None:
|
||||
if process.poll() is not None:
|
||||
return
|
||||
process.terminate()
|
||||
try:
|
||||
process.wait(timeout=PGBOUNCER_STOP_GRACE_SECONDS)
|
||||
remaining: Final = None if deadline is None else max(0.0, deadline - time.monotonic())
|
||||
timeout: Final = (
|
||||
PGBOUNCER_STOP_GRACE_SECONDS
|
||||
if remaining is None
|
||||
else max(0.0, min(PGBOUNCER_STOP_GRACE_SECONDS, remaining - 1.0))
|
||||
)
|
||||
process.wait(timeout=timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
process.wait()
|
||||
if deadline is None:
|
||||
process.wait()
|
||||
else:
|
||||
try:
|
||||
process.wait(timeout=max(0.0, deadline - time.monotonic()))
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
|
||||
|
||||
class PgBouncerProcess:
|
||||
|
|
@ -537,6 +559,12 @@ class PgBouncerProcess:
|
|||
with self._lock:
|
||||
return None if self._process is None else self._process.pid
|
||||
|
||||
@contextmanager
|
||||
def owned_pid_guard(self) -> Generator[int | None, None, None]:
|
||||
with self._lock:
|
||||
process: Final = self._process
|
||||
yield None if process is None or process.poll() is not None else process.pid
|
||||
|
||||
def _spawn(self) -> subprocess.Popen[bytes] | PgBouncerError | None:
|
||||
"""Start a child, or None once ``stop`` ran; both take the lock so no child can slip in after a stop.
|
||||
|
||||
|
|
@ -572,7 +600,7 @@ class PgBouncerProcess:
|
|||
f"within {self.ready_timeout_seconds:.0f}s"
|
||||
)
|
||||
|
||||
def start(self) -> PgBouncerError | None:
|
||||
def start(self, deadline: float | None = None) -> PgBouncerError | None:
|
||||
"""Spawn PgBouncer, wait until it listens on port and unix socket, then supervise it from a daemon thread."""
|
||||
process: Final = self._spawn()
|
||||
if process is None:
|
||||
|
|
@ -581,7 +609,7 @@ class PgBouncerProcess:
|
|||
return process
|
||||
not_ready: Final = self._wait_ready(process)
|
||||
if not_ready is not None:
|
||||
self.stop()
|
||||
self.stop(deadline)
|
||||
return not_ready
|
||||
self._watch(process)
|
||||
return None
|
||||
|
|
@ -631,12 +659,12 @@ class PgBouncerProcess:
|
|||
if self._process is not None:
|
||||
self._process.send_signal(signal.SIGHUP)
|
||||
|
||||
def stop(self) -> None:
|
||||
def stop(self, deadline: float | None = None) -> None:
|
||||
with self._lock:
|
||||
self._stopping.set()
|
||||
process: Final = self._process
|
||||
if process is not None:
|
||||
_end(process)
|
||||
_end(process, deadline)
|
||||
|
||||
|
||||
def install_pgbouncer_token(
|
||||
|
|
@ -653,6 +681,10 @@ def _install_upstream_password(
|
|||
pooler: PgBouncerProcess,
|
||||
token_auth: DatabaseTokenAuth | None,
|
||||
upstream_url: str,
|
||||
startup_cleanup: ExitStack,
|
||||
refresher_factory: Callable[
|
||||
[PgBouncerTokenSource, Callable[[str], None]], PgBouncerTokenRefresher
|
||||
] = PgBouncerTokenRefresher,
|
||||
) -> PgBouncerTokenRefresher | None | PgBouncerError:
|
||||
if token_auth is None:
|
||||
if plan.upstream_password is None:
|
||||
|
|
@ -662,10 +694,11 @@ def _install_upstream_password(
|
|||
)
|
||||
write_userlist(plan.userlist(plan.upstream_password), runtime_dir, run_as_user)
|
||||
return None
|
||||
refresher: Final = PgBouncerTokenRefresher(
|
||||
refresher: Final = refresher_factory(
|
||||
PgBouncerTokenSource(auth=token_auth, endpoint=parse_iam_endpoint_from_url(upstream_url)),
|
||||
functools.partial(install_pgbouncer_token, plan, runtime_dir, run_as_user, pooler),
|
||||
)
|
||||
startup_cleanup.callback(_stop_managed_refresher, refresher)
|
||||
failed: Final = refresher.start()
|
||||
if failed is not None:
|
||||
return failed
|
||||
|
|
@ -683,11 +716,130 @@ def _only_in_this_process(action: Callable[[], None]) -> Callable[[], None]:
|
|||
return run
|
||||
|
||||
|
||||
def _raise_for_startup_signal(signum: int, _frame: FrameType | None) -> NoReturn:
|
||||
raise SystemExit(128 + signum)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def pgbouncer_startup_signals(enabled: bool) -> Generator[None, None, None]:
|
||||
if not enabled:
|
||||
yield
|
||||
return
|
||||
previous_term: Final = signal.getsignal(signal.SIGTERM)
|
||||
previous_int: Final = signal.getsignal(signal.SIGINT)
|
||||
signal.signal(signal.SIGTERM, _raise_for_startup_signal)
|
||||
try:
|
||||
signal.signal(signal.SIGINT, _raise_for_startup_signal)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
signal.signal(signal.SIGINT, previous_int)
|
||||
finally:
|
||||
signal.signal(signal.SIGTERM, previous_term)
|
||||
|
||||
|
||||
def _stop_managed_pooler(pooler: PgBouncerProcess) -> None:
|
||||
pooler.stop(time.monotonic() + PGBOUNCER_STOP_GRACE_SECONDS)
|
||||
|
||||
|
||||
def _stop_managed_refresher(refresher: PgBouncerTokenRefresher) -> None:
|
||||
refresher.stop(time.monotonic() + PGBOUNCER_STOP_GRACE_SECONDS)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ManagedPgBouncerRuntime:
|
||||
pooled_url: str = field(repr=False)
|
||||
pooler: PgBouncerProcess
|
||||
refresher: PgBouncerTokenRefresher | None
|
||||
runtime_dir: Path
|
||||
|
||||
@contextmanager
|
||||
def owned_pid_guard(self) -> Generator[int | None, None, None]:
|
||||
with self.pooler.owned_pid_guard() as pid:
|
||||
yield pid
|
||||
|
||||
def stop(self, deadline: float | None = None) -> None:
|
||||
if self.refresher is not None:
|
||||
self.refresher.request_stop()
|
||||
try:
|
||||
self.pooler.stop(deadline)
|
||||
finally:
|
||||
try:
|
||||
if self.refresher is not None:
|
||||
self.refresher.stop(deadline)
|
||||
finally:
|
||||
shutil.rmtree(self.runtime_dir, ignore_errors=True)
|
||||
|
||||
|
||||
def start_managed_pgbouncer(
|
||||
settings: PgBouncerSettings,
|
||||
upstream_url: str,
|
||||
token_auth: DatabaseTokenAuth | None = None,
|
||||
refresher_factory: Callable[
|
||||
[PgBouncerTokenSource, Callable[[str], None]], PgBouncerTokenRefresher
|
||||
] = PgBouncerTokenRefresher,
|
||||
) -> ManagedPgBouncerRuntime | PgBouncerError:
|
||||
version: Final = pgbouncer_version(settings.binary)
|
||||
if isinstance(version, PgBouncerError):
|
||||
return version
|
||||
if version < PGBOUNCER_MIN_VERSION:
|
||||
return PgBouncerError(
|
||||
f"PgBouncer {version[0]}.{version[1]} keeps running after failing to bind its TCP port, so the proxy "
|
||||
f"cannot tell it apart from another listener; {PGBOUNCER_MIN_VERSION[0]}.{PGBOUNCER_MIN_VERSION[1]} "
|
||||
"or newer is required"
|
||||
)
|
||||
with ExitStack() as startup_cleanup:
|
||||
runtime_dir: Final = Path(tempfile.mkdtemp(prefix="litellm-pgbouncer-"))
|
||||
startup_cleanup.callback(shutil.rmtree, runtime_dir, ignore_errors=True)
|
||||
run_as_user: Final = PGBOUNCER_UNPRIVILEGED_USER if os.geteuid() == 0 else None
|
||||
plan: Final = plan_pgbouncer(upstream_url, settings, runtime_dir, run_as_user)
|
||||
if isinstance(plan, PgBouncerError):
|
||||
return plan
|
||||
ini_path: Final = write_pgbouncer_ini(plan, runtime_dir, run_as_user)
|
||||
if isinstance(ini_path, PgBouncerError):
|
||||
return ini_path
|
||||
pooler: Final = PgBouncerProcess(
|
||||
argv=(settings.binary, str(ini_path)),
|
||||
port=settings.port,
|
||||
socket_path=unix_socket_path(runtime_dir, settings.port),
|
||||
)
|
||||
startup_cleanup.callback(_stop_managed_pooler, pooler)
|
||||
refresher_result: Final = _install_upstream_password(
|
||||
plan,
|
||||
runtime_dir,
|
||||
run_as_user,
|
||||
pooler,
|
||||
token_auth,
|
||||
upstream_url,
|
||||
startup_cleanup,
|
||||
refresher_factory,
|
||||
)
|
||||
if isinstance(refresher_result, PgBouncerError):
|
||||
return refresher_result
|
||||
failed: Final = pooler.start(time.monotonic() + PGBOUNCER_STOP_GRACE_SECONDS)
|
||||
if failed is not None:
|
||||
return failed
|
||||
runtime: Final = ManagedPgBouncerRuntime(
|
||||
pooled_url=plan.pooled_url,
|
||||
pooler=pooler,
|
||||
refresher=refresher_result,
|
||||
runtime_dir=runtime_dir,
|
||||
)
|
||||
startup_cleanup.pop_all()
|
||||
return runtime
|
||||
|
||||
|
||||
def _stop_managed_runtime_before_exit(runtime: ManagedPgBouncerRuntime, timeout_seconds: float) -> None:
|
||||
runtime.stop(time.monotonic() + timeout_seconds)
|
||||
|
||||
|
||||
def start_in_container_pgbouncer(
|
||||
settings: PgBouncerSettings,
|
||||
upstream_url: str,
|
||||
token_auth: DatabaseTokenAuth | None = None,
|
||||
register_exit_hook: Callable[[Callable[[], None]], object] = atexit.register,
|
||||
*,
|
||||
shutdown_timeout_seconds: float | None = None,
|
||||
) -> str | PgBouncerError:
|
||||
"""Start the pooler for ``upstream_url`` and return the loopback URL the workers must use.
|
||||
|
||||
|
|
@ -698,46 +850,21 @@ def start_in_container_pgbouncer(
|
|||
``nobody``. With ``token_auth`` the password on ``upstream_url`` is ignored:
|
||||
the pooler mints its own tokens and renews them for as long as it runs.
|
||||
"""
|
||||
version: Final = pgbouncer_version(settings.binary)
|
||||
if isinstance(version, PgBouncerError):
|
||||
return version
|
||||
if version < PGBOUNCER_MIN_VERSION:
|
||||
return PgBouncerError(
|
||||
f"PgBouncer {version[0]}.{version[1]} keeps running after failing to bind its TCP port, so the proxy "
|
||||
f"cannot tell it apart from another listener; {PGBOUNCER_MIN_VERSION[0]}.{PGBOUNCER_MIN_VERSION[1]} "
|
||||
"or newer is required"
|
||||
)
|
||||
runtime_dir: Final = Path(tempfile.mkdtemp(prefix="litellm-pgbouncer-"))
|
||||
register_exit_hook(_only_in_this_process(lambda: shutil.rmtree(runtime_dir, ignore_errors=True)))
|
||||
run_as_user: Final = PGBOUNCER_UNPRIVILEGED_USER if os.geteuid() == 0 else None
|
||||
plan: Final = plan_pgbouncer(upstream_url, settings, runtime_dir, run_as_user)
|
||||
if isinstance(plan, PgBouncerError):
|
||||
return plan
|
||||
ini_path: Final = write_pgbouncer_ini(plan, runtime_dir, run_as_user)
|
||||
if isinstance(ini_path, PgBouncerError):
|
||||
return ini_path
|
||||
pooler: Final = PgBouncerProcess(
|
||||
argv=(settings.binary, str(ini_path)),
|
||||
port=settings.port,
|
||||
socket_path=unix_socket_path(runtime_dir, settings.port),
|
||||
runtime: Final = start_managed_pgbouncer(settings, upstream_url, token_auth)
|
||||
if isinstance(runtime, PgBouncerError):
|
||||
return runtime
|
||||
cleanup: Final = (
|
||||
runtime.stop
|
||||
if shutdown_timeout_seconds is None
|
||||
else functools.partial(_stop_managed_runtime_before_exit, runtime, shutdown_timeout_seconds)
|
||||
)
|
||||
refresher: Final = _install_upstream_password(plan, runtime_dir, run_as_user, pooler, token_auth, upstream_url)
|
||||
if isinstance(refresher, PgBouncerError):
|
||||
return refresher
|
||||
failed: Final = pooler.start()
|
||||
if failed is not None:
|
||||
if refresher is not None:
|
||||
refresher.stop()
|
||||
return failed
|
||||
register_exit_hook(_only_in_this_process(pooler.stop))
|
||||
if refresher is not None:
|
||||
register_exit_hook(_only_in_this_process(refresher.stop))
|
||||
register_exit_hook(_only_in_this_process(cleanup))
|
||||
verbose_proxy_logger.info(
|
||||
"In-container pgbouncer (pid %s) listening on %s:%s; capping this pod at %s upstream database connections%s.",
|
||||
pooler.pid,
|
||||
runtime.pooler.pid,
|
||||
PGBOUNCER_LISTEN_ADDR,
|
||||
settings.port,
|
||||
settings.max_db_connections,
|
||||
"" if token_auth is None else f" and renewing its {token_auth.label} before each one expires",
|
||||
)
|
||||
return plan.pooled_url
|
||||
return runtime.pooled_url
|
||||
|
|
|
|||
496
tests/integration/configuration/test_pgbouncer_lifecycle.py
Normal file
496
tests/integration/configuration/test_pgbouncer_lifecycle.py
Normal file
|
|
@ -0,0 +1,496 @@
|
|||
import http.client
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import textwrap
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Final, cast
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.integration._support.ports import allocate_port
|
||||
from tests.integration._support.timing import assert_enforced_within
|
||||
|
||||
ROOT: Final = Path(os.getenv("LITELLM_TEST_ROOT", str(Path(__file__).resolve().parents[3])))
|
||||
LAUNCHER_SOURCE: Final = """\
|
||||
import ctypes
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
PR_SET_CHILD_SUBREAPER = 36
|
||||
if ctypes.CDLL(None, use_errno=True).prctl(PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0) != 0:
|
||||
raise OSError(ctypes.get_errno(), "prctl(PR_SET_CHILD_SUBREAPER) failed")
|
||||
|
||||
root, launcher_pid_file, pooler_pid_file, config_file, port_file, result_file, *command = sys.argv[1:]
|
||||
started = time.monotonic()
|
||||
launcher = subprocess.Popen(command, cwd=root, env=os.environ.copy())
|
||||
Path(launcher_pid_file).write_text(str(launcher.pid))
|
||||
status = launcher.wait()
|
||||
try:
|
||||
pooler_pid = int(Path(pooler_pid_file).read_text())
|
||||
except (OSError, ValueError):
|
||||
pooler_pid = None
|
||||
try:
|
||||
config_path = Path(Path(config_file).read_text())
|
||||
except OSError:
|
||||
config_path = None
|
||||
try:
|
||||
port = int(Path(port_file).read_text())
|
||||
except (OSError, ValueError):
|
||||
port = None
|
||||
pooler_present = pooler_pid is not None and Path(f"/proc/{pooler_pid}").exists()
|
||||
listener_open = False
|
||||
if port is not None:
|
||||
try:
|
||||
with socket.create_connection(("127.0.0.1", port), timeout=0.2):
|
||||
listener_open = True
|
||||
except OSError:
|
||||
pass
|
||||
result = {
|
||||
"launcher_exit_code": status,
|
||||
"elapsed": time.monotonic() - started,
|
||||
"pooler_pid": pooler_pid,
|
||||
"pooler_present": pooler_present,
|
||||
"listener_open": listener_open,
|
||||
"runtime_dir_present": config_path is not None and config_path.parent.exists(),
|
||||
}
|
||||
Path(result_file).write_text(json.dumps(result))
|
||||
if pooler_pid is not None and Path(f"/proc/{pooler_pid}").exists():
|
||||
try:
|
||||
os.kill(pooler_pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
try:
|
||||
os.waitpid(pooler_pid, 0)
|
||||
except ChildProcessError:
|
||||
pass
|
||||
if config_path is not None:
|
||||
shutil.rmtree(config_path.parent, ignore_errors=True)
|
||||
raise SystemExit(status)
|
||||
"""
|
||||
|
||||
PGBOUNCER_SOURCE: Final = """\
|
||||
import configparser
|
||||
import os
|
||||
import select
|
||||
import signal
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
||||
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
||||
signal.signal(signal.SIGHUP, signal.SIG_IGN)
|
||||
|
||||
if sys.argv[1:] == ["--version"]:
|
||||
print("PgBouncer 1.26.0")
|
||||
raise SystemExit(0)
|
||||
|
||||
config_path = Path(sys.argv[1])
|
||||
parser = configparser.ConfigParser()
|
||||
parser.read(config_path)
|
||||
port = parser.getint("pgbouncer", "listen_port")
|
||||
socket_path = Path(parser.get("pgbouncer", "unix_socket_dir")) / f".s.PGSQL.{port}"
|
||||
Path(os.environ["LITELLM_TEST_POOLER_PID_FILE"]).write_text(str(os.getpid()))
|
||||
Path(os.environ["LITELLM_TEST_POOLER_CONFIG_FILE"]).write_text(str(config_path))
|
||||
Path(os.environ["LITELLM_TEST_POOLER_PORT_FILE"]).write_text(str(port))
|
||||
if os.environ["LITELLM_TEST_POOLER_MODE"] == "slow":
|
||||
while True:
|
||||
signal.pause()
|
||||
if os.environ["LITELLM_TEST_POOLER_MODE"] == "exit":
|
||||
time.sleep(0.25)
|
||||
raise SystemExit(3)
|
||||
|
||||
tcp = socket.socket()
|
||||
tcp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
tcp.bind(("127.0.0.1", port))
|
||||
tcp.listen()
|
||||
unix = socket.socket(socket.AF_UNIX)
|
||||
unix.bind(str(socket_path))
|
||||
unix.listen()
|
||||
while True:
|
||||
for listener in select.select((tcp, unix), (), ())[0]:
|
||||
listener.accept()[0].close()
|
||||
"""
|
||||
|
||||
SERVE_SOURCE: Final = """\
|
||||
import sys
|
||||
import uvicorn
|
||||
from gateway.launch import main
|
||||
|
||||
async def app(scope, receive, send):
|
||||
if scope["type"] == "http":
|
||||
await send({"type": "http.response.start", "status": 200, "headers": []})
|
||||
await send({"type": "http.response.body", "body": b"ready"})
|
||||
|
||||
def serve(argv):
|
||||
port = int(argv[argv.index("--port") + 1])
|
||||
uvicorn.Server(
|
||||
uvicorn.Config(app, host="127.0.0.1", port=port, lifespan="off", workers=1)
|
||||
).run()
|
||||
|
||||
main(sys.argv[1:], serve=serve)
|
||||
"""
|
||||
|
||||
|
||||
def _write_script(path: Path, source: str) -> Path:
|
||||
path.write_text(f"#!/usr/bin/env python3\n{textwrap.dedent(source)}")
|
||||
path.chmod(0o700)
|
||||
return path
|
||||
|
||||
|
||||
def _wait_for_file(path: Path, timeout_seconds: float = 15.0) -> str:
|
||||
deadline: Final = time.monotonic() + timeout_seconds
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
return path.read_text()
|
||||
except OSError:
|
||||
time.sleep(0.05)
|
||||
pytest.fail(f"timed out waiting for {path}")
|
||||
|
||||
|
||||
def _wait_for_new_pid(path: Path, previous_pid: int, timeout_seconds: float = 5.0) -> int:
|
||||
deadline: Final = time.monotonic() + timeout_seconds
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
pid: Final = int(path.read_text())
|
||||
if pid != previous_pid:
|
||||
return pid
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
time.sleep(0.05)
|
||||
pytest.fail(f"timed out waiting for a replacement process in {path}")
|
||||
|
||||
|
||||
def _http_health(port: int) -> bool:
|
||||
try:
|
||||
connection: Final = http.client.HTTPConnection("127.0.0.1", port, timeout=0.5)
|
||||
connection.request("GET", "/health/liveliness")
|
||||
response: Final = connection.getresponse()
|
||||
response.read()
|
||||
connection.close()
|
||||
return response.status == 200
|
||||
except (OSError, http.client.HTTPException):
|
||||
return False
|
||||
|
||||
|
||||
def _wait_for_health(port: int, timeout_seconds: float = 15.0) -> None:
|
||||
deadline: Final = time.monotonic() + timeout_seconds
|
||||
while time.monotonic() < deadline:
|
||||
if _http_health(port):
|
||||
return
|
||||
time.sleep(0.05)
|
||||
pytest.fail(f"gateway did not become healthy on port {port}")
|
||||
|
||||
|
||||
def _run_black_box(
|
||||
tmp_path: Path, pooler_mode: str, signal_number: signal.Signals, require_health: bool
|
||||
) -> dict[str, object]:
|
||||
launcher_pid_file: Final = tmp_path / "launcher.pid"
|
||||
pooler_pid_file: Final = tmp_path / "pooler.pid"
|
||||
config_file: Final = tmp_path / "pooler-config"
|
||||
port_file: Final = tmp_path / "pooler-port"
|
||||
result_file: Final = tmp_path / "result.json"
|
||||
pooler_port: Final = allocate_port()
|
||||
gateway_port: Final = allocate_port()
|
||||
pooler_binary: Final = _write_script(tmp_path / "fake-pgbouncer", PGBOUNCER_SOURCE)
|
||||
wrapper: Final = _write_script(tmp_path / "subreaper-wrapper.py", LAUNCHER_SOURCE)
|
||||
environment: Final = {
|
||||
**os.environ,
|
||||
"PYTHONPATH": os.pathsep.join(filter(None, (str(ROOT), os.environ.get("PYTHONPATH", "")))),
|
||||
"DATABASE_URL": "postgresql://app:password@127.0.0.1:5432/litellm",
|
||||
"LITELLM_PGBOUNCER_ENABLED": "true",
|
||||
"LITELLM_PGBOUNCER_BINARY": str(pooler_binary),
|
||||
"LITELLM_PGBOUNCER_PORT": str(pooler_port),
|
||||
"LITELLM_TEST_POOLER_MODE": pooler_mode,
|
||||
"LITELLM_TEST_POOLER_PID_FILE": str(pooler_pid_file),
|
||||
"LITELLM_TEST_POOLER_CONFIG_FILE": str(config_file),
|
||||
"LITELLM_TEST_POOLER_PORT_FILE": str(port_file),
|
||||
}
|
||||
serve_script: Final = _write_script(tmp_path / "serve-gateway.py", SERVE_SOURCE)
|
||||
command: Final = (
|
||||
(
|
||||
sys.executable,
|
||||
str(serve_script),
|
||||
"--host",
|
||||
"127.0.0.1",
|
||||
"--port",
|
||||
str(gateway_port),
|
||||
"--workers",
|
||||
"1",
|
||||
)
|
||||
if require_health
|
||||
else (
|
||||
sys.executable,
|
||||
"-m",
|
||||
"gateway.launch",
|
||||
"--host",
|
||||
"127.0.0.1",
|
||||
"--port",
|
||||
str(gateway_port),
|
||||
"--workers",
|
||||
"1",
|
||||
)
|
||||
)
|
||||
launch_log: Final = (tmp_path / "gateway.log").open("w")
|
||||
wrapper_process: Final = subprocess.Popen(
|
||||
(
|
||||
sys.executable,
|
||||
"-P",
|
||||
str(wrapper),
|
||||
str(ROOT),
|
||||
str(launcher_pid_file),
|
||||
str(pooler_pid_file),
|
||||
str(config_file),
|
||||
str(port_file),
|
||||
str(result_file),
|
||||
*command,
|
||||
),
|
||||
cwd=ROOT,
|
||||
env=environment,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=launch_log,
|
||||
stderr=subprocess.STDOUT,
|
||||
start_new_session=True,
|
||||
)
|
||||
try:
|
||||
launcher_pid: Final = int(_wait_for_file(launcher_pid_file))
|
||||
_wait_for_file(pooler_pid_file)
|
||||
if require_health:
|
||||
try:
|
||||
_wait_for_health(gateway_port)
|
||||
except BaseException as error:
|
||||
raise AssertionError((tmp_path / "gateway.log").read_text()) from error
|
||||
os.kill(launcher_pid, signal_number) # test-quality-ok: signal starts the lifecycle under test
|
||||
wrapper_process.wait(timeout=35)
|
||||
return cast(dict[str, object], json.loads(result_file.read_text()))
|
||||
finally:
|
||||
if wrapper_process.poll() is None:
|
||||
try:
|
||||
launcher_pid = int(launcher_pid_file.read_text())
|
||||
os.kill(launcher_pid, signal.SIGKILL) # test-quality-ok: bounded fixture cleanup
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
try:
|
||||
wrapper_process.wait(timeout=15)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.kill(wrapper_process.pid, signal.SIGKILL) # test-quality-ok: bounded fixture cleanup
|
||||
wrapper_process.wait()
|
||||
launch_log.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("signal_number", "expected_exit"),
|
||||
((signal.SIGTERM, 143), (signal.SIGINT, 130)),
|
||||
)
|
||||
def test_startup_signal_removes_slow_pooler_and_runtime(
|
||||
tmp_path: Path, signal_number: signal.Signals, expected_exit: int
|
||||
) -> None:
|
||||
result: Final = _run_black_box(tmp_path, "slow", signal_number, require_health=False)
|
||||
assert result["pooler_present"] is False, result
|
||||
assert result["listener_open"] is False, result
|
||||
assert result["runtime_dir_present"] is False, result
|
||||
assert result["launcher_exit_code"] == expected_exit, result
|
||||
assert cast(float, result["elapsed"]) <= 30
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("signal_number", "expected_exit"),
|
||||
((signal.SIGTERM, 143), (signal.SIGINT, 130)),
|
||||
)
|
||||
def test_running_gateway_signal_removes_pooler_and_runtime(
|
||||
tmp_path: Path, signal_number: signal.Signals, expected_exit: int
|
||||
) -> None:
|
||||
result: Final = _run_black_box(tmp_path, "serve", signal_number, require_health=True)
|
||||
assert result["pooler_present"] is False, result
|
||||
assert result["listener_open"] is False, result
|
||||
assert result["runtime_dir_present"] is False, result
|
||||
assert result["launcher_exit_code"] == expected_exit, result
|
||||
assert cast(float, result["elapsed"]) <= 30
|
||||
|
||||
|
||||
def test_managed_pooler_stop_reaps_owned_pid_and_runtime(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from litellm.proxy.db.pgbouncer import ManagedPgBouncerRuntime, PgBouncerSettings, start_managed_pgbouncer
|
||||
|
||||
pid_file: Final = tmp_path / "pooler.pid"
|
||||
config_file: Final = tmp_path / "pooler-config"
|
||||
port_file: Final = tmp_path / "pooler-port"
|
||||
pooler_binary: Final = _write_script(tmp_path / "fake-pgbouncer", PGBOUNCER_SOURCE)
|
||||
settings: Final = PgBouncerSettings(enabled=True, binary=str(pooler_binary), port=allocate_port())
|
||||
environment: Final = {
|
||||
"LITELLM_TEST_POOLER_MODE": "serve",
|
||||
"LITELLM_TEST_POOLER_PID_FILE": str(pid_file),
|
||||
"LITELLM_TEST_POOLER_CONFIG_FILE": str(config_file),
|
||||
"LITELLM_TEST_POOLER_PORT_FILE": str(port_file),
|
||||
}
|
||||
for key, value in environment.items():
|
||||
monkeypatch.setenv(key, value)
|
||||
runtime_result: Final = start_managed_pgbouncer(settings, "postgresql://app:password@db/litellm")
|
||||
assert isinstance(runtime_result, ManagedPgBouncerRuntime), runtime_result
|
||||
runtime: Final = runtime_result
|
||||
pid: Final = runtime.pooler.pid
|
||||
try:
|
||||
assert pid is not None
|
||||
with runtime.owned_pid_guard() as guarded_pid:
|
||||
assert guarded_pid == pid
|
||||
finally:
|
||||
runtime.stop(time.monotonic() + 15)
|
||||
assert pid is not None
|
||||
assert not Path(f"/proc/{pid}").exists()
|
||||
assert not runtime.runtime_dir.exists()
|
||||
|
||||
|
||||
def test_pooler_start_failure_stops_injected_fast_refresher(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from collections.abc import Callable
|
||||
|
||||
from litellm.proxy.db.pgbouncer import (
|
||||
PgBouncerError,
|
||||
PgBouncerSettings,
|
||||
PgBouncerTokenRefresher,
|
||||
PgBouncerTokenSource,
|
||||
start_managed_pgbouncer,
|
||||
)
|
||||
from litellm.proxy.db.token_auth import AzureEntraTokenAuth
|
||||
|
||||
pid_file: Final = tmp_path / "pooler.pid"
|
||||
config_file: Final = tmp_path / "pooler-config"
|
||||
port_file: Final = tmp_path / "pooler-port"
|
||||
pooler_binary: Final = _write_script(tmp_path / "fake-pgbouncer", PGBOUNCER_SOURCE)
|
||||
settings: Final = PgBouncerSettings(enabled=True, binary=str(pooler_binary), port=allocate_port())
|
||||
monkeypatch.setenv("LITELLM_TEST_POOLER_MODE", "exit")
|
||||
monkeypatch.setenv("LITELLM_TEST_POOLER_PID_FILE", str(pid_file))
|
||||
monkeypatch.setenv("LITELLM_TEST_POOLER_CONFIG_FILE", str(config_file))
|
||||
monkeypatch.setenv("LITELLM_TEST_POOLER_PORT_FILE", str(port_file))
|
||||
first_mint: Final = threading.Event()
|
||||
refreshed_during_start: Final = threading.Event()
|
||||
start_failed: Final = threading.Event()
|
||||
refreshed_after_failure: Final = threading.Event()
|
||||
initial_install_done: Final = threading.Event()
|
||||
|
||||
def provide_token() -> str:
|
||||
if first_mint.is_set():
|
||||
refreshed_during_start.set()
|
||||
if start_failed.is_set():
|
||||
refreshed_after_failure.set()
|
||||
else:
|
||||
first_mint.set()
|
||||
return "opaque-token"
|
||||
|
||||
def fast_refresher_factory(source: PgBouncerTokenSource, install: Callable[[str], None]) -> PgBouncerTokenRefresher:
|
||||
def install_once(token: str) -> None:
|
||||
if initial_install_done.is_set():
|
||||
return
|
||||
install(token)
|
||||
initial_install_done.set()
|
||||
|
||||
return PgBouncerTokenRefresher(source, install_once, fallback_seconds=0.02, retry_seconds=0.02)
|
||||
|
||||
outcome: Final = start_managed_pgbouncer(
|
||||
settings,
|
||||
"postgresql://app@db/litellm",
|
||||
token_auth=AzureEntraTokenAuth(token_provider=provide_token),
|
||||
refresher_factory=fast_refresher_factory,
|
||||
)
|
||||
assert isinstance(outcome, PgBouncerError), outcome
|
||||
assert pid_file.exists(), outcome
|
||||
assert refreshed_during_start.wait(2)
|
||||
start_failed.set()
|
||||
time.sleep(0.1)
|
||||
assert not refreshed_after_failure.is_set()
|
||||
runtime_dir: Final = Path(config_file.read_text()).parent
|
||||
assert not runtime_dir.exists()
|
||||
|
||||
|
||||
def test_managed_stop_reaps_replacement_before_refresher_deadline_and_skips_late_install(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
from collections.abc import Callable
|
||||
|
||||
from litellm.proxy.db.pgbouncer import (
|
||||
ManagedPgBouncerRuntime,
|
||||
PgBouncerSettings,
|
||||
PgBouncerTokenRefresher,
|
||||
PgBouncerTokenSource,
|
||||
start_managed_pgbouncer,
|
||||
)
|
||||
from litellm.proxy.db.token_auth import AzureEntraTokenAuth
|
||||
|
||||
pid_file: Final = tmp_path / "pooler.pid"
|
||||
config_file: Final = tmp_path / "pooler-config"
|
||||
port_file: Final = tmp_path / "pooler-port"
|
||||
monkeypatch.setenv("LITELLM_TEST_POOLER_MODE", "serve")
|
||||
monkeypatch.setenv("LITELLM_TEST_POOLER_PID_FILE", str(pid_file))
|
||||
monkeypatch.setenv("LITELLM_TEST_POOLER_CONFIG_FILE", str(config_file))
|
||||
monkeypatch.setenv("LITELLM_TEST_POOLER_PORT_FILE", str(port_file))
|
||||
first_mint: Final = threading.Event()
|
||||
mint_blocked: Final = threading.Event()
|
||||
release_mint: Final = threading.Event()
|
||||
first_install: Final = threading.Event()
|
||||
late_install: Final = threading.Event()
|
||||
|
||||
def provide_token() -> str:
|
||||
if not first_mint.is_set():
|
||||
first_mint.set()
|
||||
return "initial-token"
|
||||
mint_blocked.set()
|
||||
release_mint.wait(10)
|
||||
return "late-token"
|
||||
|
||||
def refresher_factory(source: PgBouncerTokenSource, install: Callable[[str], None]) -> PgBouncerTokenRefresher:
|
||||
def observe_install(token: str) -> None:
|
||||
if first_install.is_set():
|
||||
late_install.set()
|
||||
return
|
||||
first_install.set()
|
||||
install(token)
|
||||
|
||||
return PgBouncerTokenRefresher(source, observe_install, fallback_seconds=0.01, retry_seconds=0.01)
|
||||
|
||||
pooler_binary: Final = _write_script(tmp_path / "fake-pgbouncer", PGBOUNCER_SOURCE)
|
||||
settings: Final = PgBouncerSettings(enabled=True, binary=str(pooler_binary), port=allocate_port())
|
||||
outcome: Final = start_managed_pgbouncer(
|
||||
settings,
|
||||
"postgresql://app@db/litellm",
|
||||
token_auth=AzureEntraTokenAuth(token_provider=provide_token),
|
||||
refresher_factory=refresher_factory,
|
||||
)
|
||||
assert isinstance(outcome, ManagedPgBouncerRuntime), outcome
|
||||
runtime: Final = outcome
|
||||
refresher: Final = runtime.refresher
|
||||
assert refresher is not None
|
||||
try:
|
||||
assert first_install.is_set()
|
||||
assert mint_blocked.wait(3)
|
||||
initial_pid: Final = runtime.pooler.pid
|
||||
assert initial_pid is not None
|
||||
os.kill(initial_pid, signal.SIGKILL) # test-quality-ok: crash the owned child to exercise watchdog replacement
|
||||
replacement_pid: Final = _wait_for_new_pid(pid_file, initial_pid)
|
||||
assert runtime.pooler.pid == replacement_pid
|
||||
started: Final = time.monotonic()
|
||||
stop_budget_seconds: Final = 0.23
|
||||
runtime.stop(started + stop_budget_seconds)
|
||||
assert_enforced_within(
|
||||
time.monotonic() - started,
|
||||
configured=stop_budget_seconds,
|
||||
slack=0.27,
|
||||
)
|
||||
assert not Path(f"/proc/{replacement_pid}").exists()
|
||||
assert not runtime.runtime_dir.exists()
|
||||
release_mint.set()
|
||||
refresher.stop(time.monotonic() + 2)
|
||||
assert not late_install.is_set()
|
||||
finally:
|
||||
release_mint.set()
|
||||
runtime.stop(time.monotonic() + 5)
|
||||
|
|
@ -23,6 +23,7 @@ import pytest
|
|||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.proxy.db.pgbouncer import (
|
||||
PGBOUNCER_POOLED_ENV_VAR,
|
||||
ManagedPgBouncerRuntime,
|
||||
PgBouncerError,
|
||||
PgBouncerPlan,
|
||||
PgBouncerProcess,
|
||||
|
|
@ -32,6 +33,7 @@ from litellm.proxy.db.pgbouncer import (
|
|||
database_url_is_pooled,
|
||||
export_pooled_database_url,
|
||||
install_pgbouncer_token,
|
||||
pgbouncer_startup_signals,
|
||||
pgbouncer_version,
|
||||
plan_pgbouncer,
|
||||
start_in_container_pgbouncer,
|
||||
|
|
@ -603,6 +605,51 @@ def _token_source(*tokens: str | Exception) -> PgBouncerTokenSource:
|
|||
return PgBouncerTokenSource(auth=AzureEntraTokenAuth(token_provider=provide), endpoint=ENDPOINT)
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform == "win32", reason="Windows does not deliver POSIX process signals")
|
||||
@pytest.mark.parametrize("signum", (signal.SIGTERM, signal.SIGINT))
|
||||
@pytest.mark.parametrize("enabled", (False, True))
|
||||
def test_startup_signals_preserve_the_application_handler_after_interruption(signum: int, enabled: bool) -> None:
|
||||
previous: Final = signal.signal(signum, signal.default_int_handler)
|
||||
expected_exception: Final = SystemExit if enabled else KeyboardInterrupt
|
||||
try:
|
||||
with pytest.raises((SystemExit, KeyboardInterrupt)) as interrupted:
|
||||
with pgbouncer_startup_signals(enabled):
|
||||
signal.raise_signal(signum)
|
||||
assert type(interrupted.value) is expected_exception
|
||||
assert interrupted.value.args == ((128 + signum,) if enabled else ())
|
||||
with pytest.raises(KeyboardInterrupt):
|
||||
signal.raise_signal(signum)
|
||||
finally:
|
||||
signal.signal(signum, previous)
|
||||
|
||||
|
||||
def test_managed_shutdown_cancels_pending_token_install_and_restart(tmp_path: Path) -> None:
|
||||
runtime_dir: Final = tmp_path / "runtime"
|
||||
runtime_dir.mkdir()
|
||||
auth_file: Final = runtime_dir / "userlist.txt"
|
||||
|
||||
def install(token: str) -> None:
|
||||
auth_file.write_text(token)
|
||||
|
||||
refresher: Final = PgBouncerTokenRefresher(_token_source("before", "after"), install, fallback_seconds=600)
|
||||
pooler: Final = PgBouncerProcess(argv=(), port=6543, socket_path=runtime_dir / "socket")
|
||||
runtime: Final = ManagedPgBouncerRuntime(
|
||||
pooled_url="postgresql://app:password@127.0.0.1:6543/litellm",
|
||||
pooler=pooler,
|
||||
refresher=refresher,
|
||||
runtime_dir=runtime_dir,
|
||||
)
|
||||
assert refresher.refresh() == 600
|
||||
assert auth_file.read_text() == "before"
|
||||
with runtime.owned_pid_guard() as pid:
|
||||
assert pid is None
|
||||
runtime.stop(deadline=0.0)
|
||||
assert not runtime_dir.exists()
|
||||
assert refresher.refresh() == 600
|
||||
assert not runtime_dir.exists()
|
||||
assert pooler.start() == PgBouncerError("pgbouncer was stopped before it started")
|
||||
|
||||
|
||||
class TestPgBouncerTokenRefresher:
|
||||
def _refresher(
|
||||
self,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue