mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-01 07:03:02 +00:00
57f6e1b39f
* fix(proxy): default the uvicorn worker healthcheck timeout to 60 s so multi-worker boots survive a slow import * fix(proxy): warn on uvicorn below 0.37 when multi-worker boots cannot get the longer healthcheck uvicorn 0.33 to 0.36 has no timeout_worker_healthcheck, so the 60 s default never reaches it and a slow worker start still crash-loops with no hint. Print an upgrade warning when the proxy runs more than one uvicorn worker on such a version, keep granian out of the uvicorn-only path, and type the new tests. * fix(proxy): name --run_granian in the healthcheck help text and add per-worker grace guidance to the helm chart * fix(proxy): keep the litellm command light so spawned workers answer uvicorn's first health ping Each worker uvicorn spawns re-runs the parent's main script before it starts answering health pings. The litellm console script imported the whole SDK at the top, so on a busy or CPU-limited host a new worker missed uvicorn's 5 s ping and was killed in a loop. The script now runs litellm_cli.main:proxy_server, which imports the proxy only when called, and gateway.launch imports LiteLLM inside main the same way. This replaces the earlier 60 s healthcheck default, so uvicorn's 5 s healthcheck and SIGTERM timing stay as they are. * fix(proxy): load the worker app before its log config and without GC pauses * fix(gateway): ship litellm_cli in the gateway image gateway.launch imports without_gc_pauses from litellm_cli, which the gateway image never copied, so python -m gateway.launch failed at import. Also build the gateway image when the launcher or litellm_cli changes. * fix(proxy): apply the worker log config before the proxy app import, as earlier releases did --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
93 lines
3.8 KiB
Python
93 lines
3.8 KiB
Python
"""Gateway supervisor: assemble DATABASE_URL, start the in-container PgBouncer, then run uvicorn.
|
|
|
|
``gateway/main.py`` assembles ``DATABASE_URL`` inside every uvicorn worker, which
|
|
is fine for a plain Postgres URL but not for the pooler: PgBouncer must be
|
|
started exactly once per pod, before the workers fork, and the workers must be
|
|
handed the loopback URL it listens on. A pre-existing ``DATABASE_URL`` wins in
|
|
``DatabaseURLSettings.apply_to_env`` under password auth, and one marked pooled
|
|
wins under token auth too, so exporting it here is enough for every worker to
|
|
pick the pooled URL up unchanged.
|
|
|
|
LiteLLM imports wait for ``main``: every uvicorn worker imports this module again
|
|
before it can answer its first 5 s health ping, so a slow host would kill each one.
|
|
Workers then load the gateway app through ``worker_app`` without garbage collection
|
|
pauses, which hold the GIL long enough to miss a ping.
|
|
|
|
Run with:
|
|
python -m gateway.launch --workers 4 --host 0.0.0.0 --port 4000
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
from collections.abc import Callable, Mapping, Sequence
|
|
from typing import TYPE_CHECKING, Final
|
|
|
|
from uvicorn.main import main as uvicorn_main
|
|
|
|
from litellm_cli.main import without_gc_pauses
|
|
|
|
if TYPE_CHECKING:
|
|
from fastapi import FastAPI
|
|
|
|
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
|
|
from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings
|
|
|
|
GATEWAY_APP: Final = "gateway.launch:worker_app"
|
|
KEEPALIVE_FLAG: Final = "--timeout-keep-alive"
|
|
|
|
|
|
def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str]) -> tuple[str, ...]:
|
|
"""Honor ``KEEPALIVE_TIMEOUT`` like ``proxy_cli.py`` does, unless the flag was passed explicitly."""
|
|
keepalive: Final = environ.get("KEEPALIVE_TIMEOUT")
|
|
if keepalive is None or any(arg == KEEPALIVE_FLAG or arg.startswith(f"{KEEPALIVE_FLAG}=") for arg in argv):
|
|
return (GATEWAY_APP, "--factory", *argv)
|
|
return (GATEWAY_APP, "--factory", *argv, KEEPALIVE_FLAG, keepalive)
|
|
|
|
|
|
def pool_database_url(
|
|
settings: "DatabaseURLSettings",
|
|
pgbouncer: "PgBouncerSettings",
|
|
environ: Mapping[str, str],
|
|
) -> "str | PgBouncerError | None":
|
|
"""Start the in-container PgBouncer and return its loopback URL, or None when ``pgbouncer.enabled`` is off.
|
|
|
|
The upstream URL is whatever ``apply_to_env`` assembled from the discrete
|
|
``DATABASE_*`` vars (or an operator-pinned ``DATABASE_URL``). Under token
|
|
auth the pooler mints and renews the upstream token itself.
|
|
"""
|
|
from litellm.proxy.db.pgbouncer import PgBouncerError, start_in_container_pgbouncer
|
|
|
|
if not pgbouncer.enabled:
|
|
return None
|
|
upstream_url: Final = environ.get("DATABASE_URL")
|
|
if upstream_url is None:
|
|
return PgBouncerError("LITELLM_PGBOUNCER_ENABLED is set but no DATABASE_URL could be assembled")
|
|
return start_in_container_pgbouncer(pgbouncer, upstream_url, token_auth=settings.token_auth())
|
|
|
|
|
|
def worker_app() -> "FastAPI":
|
|
with without_gc_pauses():
|
|
from gateway.main import app
|
|
return app
|
|
|
|
|
|
def _serve(argv: Sequence[str]) -> None:
|
|
uvicorn_main(tuple(argv), prog_name="uvicorn")
|
|
|
|
|
|
def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None:
|
|
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
|
|
from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, export_pooled_database_url
|
|
|
|
settings: Final = DatabaseURLSettings.from_env()
|
|
settings.apply_to_env()
|
|
pooled_url: Final = pool_database_url(settings, PgBouncerSettings(), os.environ)
|
|
if isinstance(pooled_url, PgBouncerError):
|
|
sys.exit(f"LiteLLM gateway: in-container pgbouncer could not start: {pooled_url.reason}")
|
|
if pooled_url is not None:
|
|
export_pooled_database_url(pooled_url)
|
|
serve(uvicorn_argv(argv, os.environ))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main(sys.argv[1:])
|