mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
Merge 55dfc7208e into ce4ef3855a
This commit is contained in:
commit
afef065f7a
22 changed files with 1312 additions and 35 deletions
1
.github/workflows/test-unit.yml
vendored
1
.github/workflows/test-unit.yml
vendored
|
|
@ -583,6 +583,7 @@ jobs:
|
|||
tests/unit/proxy/credential_endpoints
|
||||
tests/unit/proxy/discovery_endpoints
|
||||
tests/unit/proxy/shutdown
|
||||
tests/unit/proxy/rust_forward
|
||||
tests/unit/proxy/public_endpoints
|
||||
tests/unit/proxy/prompts
|
||||
tests/unit/proxy/rag_endpoints
|
||||
|
|
|
|||
|
|
@ -19,29 +19,43 @@ Run with:
|
|||
|
||||
import os
|
||||
import sys
|
||||
import sysconfig
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from typing_extensions import assert_never
|
||||
from uvicorn.main import main as uvicorn_main
|
||||
|
||||
from litellm_cli.main import without_gc_pauses
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fastapi import FastAPI
|
||||
|
||||
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
|
||||
from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings
|
||||
from litellm.proxy.rust_forward.listener import PublicListener
|
||||
from litellm.proxy.rust_forward.mode import ServerOverrides
|
||||
from litellm.proxy.rust_forward.peer import ASGIApp
|
||||
|
||||
GATEWAY_APP: Final = "gateway.launch:worker_app"
|
||||
KEEPALIVE_FLAG: Final = "--timeout-keep-alive"
|
||||
|
||||
|
||||
def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str]) -> tuple[str, ...]:
|
||||
def uvicorn_argv(
|
||||
argv: Sequence[str], environ: Mapping[str, str], forwarded: "ServerOverrides | None"
|
||||
) -> tuple[str, ...]:
|
||||
"""Honor ``KEEPALIVE_TIMEOUT`` like ``proxy_cli.py`` does, unless the flag was passed explicitly."""
|
||||
keepalive: Final = environ.get("KEEPALIVE_TIMEOUT")
|
||||
inherited_socket: Final = (
|
||||
()
|
||||
if forwarded is None
|
||||
else (
|
||||
"--fd",
|
||||
str(forwarded.fd),
|
||||
"--proxy-headers" if forwarded.proxy_headers else "--no-proxy-headers",
|
||||
)
|
||||
)
|
||||
if keepalive is None or any(arg == KEEPALIVE_FLAG or arg.startswith(f"{KEEPALIVE_FLAG}=") for arg in argv):
|
||||
return (GATEWAY_APP, "--factory", *argv)
|
||||
return (GATEWAY_APP, "--factory", *argv, KEEPALIVE_FLAG, keepalive)
|
||||
return (GATEWAY_APP, "--factory", *argv, *inherited_socket)
|
||||
return (GATEWAY_APP, "--factory", *argv, KEEPALIVE_FLAG, keepalive, *inherited_socket)
|
||||
|
||||
|
||||
def pool_database_url(
|
||||
|
|
@ -65,17 +79,54 @@ def pool_database_url(
|
|||
return start_in_container_pgbouncer(pgbouncer, upstream_url, token_auth=settings.token_auth())
|
||||
|
||||
|
||||
def worker_app() -> "FastAPI":
|
||||
def worker_app() -> "ASGIApp":
|
||||
with without_gc_pauses():
|
||||
from gateway.main import app
|
||||
return app
|
||||
from litellm.proxy.rust_forward.mode import rust_forward_mode
|
||||
from litellm.proxy.rust_forward.peer import wrap
|
||||
return wrap(app, rust_forward_mode(os.environ))
|
||||
|
||||
|
||||
def _serve(argv: Sequence[str]) -> None:
|
||||
uvicorn_main(tuple(argv), prog_name="uvicorn")
|
||||
|
||||
|
||||
def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None:
|
||||
def public_listener(argv: Sequence[str], binary: str | None) -> "tuple[str, PublicListener]":
|
||||
"""Validate the uvicorn flags for Rust forwarding and return the binary and the public listener."""
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.proxy.rust_forward.listener import PublicListener, StartupError, shutdown_drain_seconds, startup_check
|
||||
from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager
|
||||
|
||||
params: Final[Mapping[str, object]] = uvicorn_main.make_context(
|
||||
"uvicorn", list(uvicorn_argv(argv, os.environ, None))
|
||||
).params
|
||||
checked: Final = startup_check(
|
||||
binary,
|
||||
{
|
||||
"--ssl-keyfile": params["ssl_keyfile"],
|
||||
"--ssl-certfile": params["ssl_certfile"],
|
||||
"--ssl-ciphers": any(arg.split("=")[0] == "--ssl-ciphers" for arg in argv),
|
||||
"--uds": params["uds"] is not None,
|
||||
"--fd": params["fd"] is not None,
|
||||
},
|
||||
)
|
||||
if isinstance(checked, StartupError):
|
||||
sys.exit(checked.reason)
|
||||
graceful_timeout: Final = TypeAdapter[int | None](int | None).validate_python(params["timeout_graceful_shutdown"])
|
||||
return checked, PublicListener(
|
||||
host=TypeAdapter(str).validate_python(params["host"]),
|
||||
port=TypeAdapter(int).validate_python(params["port"]),
|
||||
keepalive_seconds=TypeAdapter(int).validate_python(params["timeout_keep_alive"]),
|
||||
shutdown_drain_seconds=shutdown_drain_seconds(
|
||||
GracefulShutdownManager.get_timeout() if graceful_timeout is None else graceful_timeout
|
||||
),
|
||||
proxy_headers=TypeAdapter(bool).validate_python(params["proxy_headers"]),
|
||||
trusted_proxies=TypeAdapter[str | None](str | None).validate_python(params["forwarded_allow_ips"]),
|
||||
)
|
||||
|
||||
|
||||
def configure_database() -> None:
|
||||
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
|
||||
from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, export_pooled_database_url
|
||||
|
||||
|
|
@ -86,7 +137,26 @@ def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -
|
|||
sys.exit(f"LiteLLM gateway: in-container pgbouncer could not start: {pooled_url.reason}")
|
||||
if pooled_url is not None:
|
||||
export_pooled_database_url(pooled_url)
|
||||
serve(uvicorn_argv(argv, os.environ))
|
||||
|
||||
|
||||
def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None:
|
||||
from litellm.proxy.rust_forward.mode import Disabled, Supervised, Supervisor, rust_forward_mode, server_overrides
|
||||
from litellm.proxy.rust_forward.supervisor import gateway_binary, supervise
|
||||
|
||||
mode: Final = rust_forward_mode(os.environ)
|
||||
match mode:
|
||||
case Disabled():
|
||||
configure_database()
|
||||
serve(uvicorn_argv(argv, os.environ, None))
|
||||
case Supervisor():
|
||||
binary, listener = public_listener(argv, gateway_binary(sysconfig.get_path("scripts")))
|
||||
configure_database()
|
||||
command: Final = (sys.executable, "-m", "gateway.launch", *argv)
|
||||
sys.exit(supervise(binary, command, listener, os.environ))
|
||||
case Supervised():
|
||||
serve(uvicorn_argv(argv, os.environ, server_overrides(mode)))
|
||||
case _:
|
||||
assert_never(mode)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ from socket import socket
|
|||
from typing import Final, Literal
|
||||
|
||||
from pydantic import ConfigDict, ImportString, JsonValue, TypeAdapter, with_config
|
||||
from typing_extensions import ReadOnly, Required, TypedDict
|
||||
from typing_extensions import NotRequired, ReadOnly, Required, TypedDict
|
||||
from uvicorn import Config, Server
|
||||
from uvicorn import config as uvicorn_config
|
||||
from uvicorn.supervisors import ChangeReload, Multiprocess
|
||||
|
|
@ -42,6 +42,7 @@ class ServerOptions(TypedDict, total=False):
|
|||
@with_config(ConfigDict(extra="allow"))
|
||||
class ReloadOptions(ServerOptions):
|
||||
app: Required[ReadOnly[str]]
|
||||
factory: NotRequired[ReadOnly[bool]]
|
||||
reload: ReadOnly[bool]
|
||||
reload_dirs: ReadOnly[tuple[str, ...]]
|
||||
reload_includes: ReadOnly[tuple[str, ...]]
|
||||
|
|
@ -92,7 +93,9 @@ def load_worker_app(import_path: str, worker_app_factory: bool) -> Callable[...,
|
|||
def pool_config(options: ReloadOptions, workers: int, worker_overrides: Mapping[str, object] | None = None) -> Config:
|
||||
overrides: Final = worker_overrides or {}
|
||||
worker_app: Final = TypeAdapter(str).validate_python(overrides.get("app", options["app"]))
|
||||
worker_app_factory: Final = TypeAdapter(bool).validate_python(overrides.get("factory", False))
|
||||
worker_app_factory: Final = TypeAdapter(bool).validate_python(
|
||||
overrides.get("factory", options.get("factory", False))
|
||||
)
|
||||
return Config(
|
||||
partial(load_worker_app, worker_app, worker_app_factory),
|
||||
factory=True,
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import random
|
|||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import sysconfig
|
||||
import urllib.parse as urlparse
|
||||
from collections.abc import Callable, Iterable, Mapping, Sequence
|
||||
from pathlib import Path
|
||||
|
|
@ -35,6 +36,10 @@ from litellm.types.llms.base import LiteLLMBaseModel
|
|||
|
||||
if TYPE_CHECKING:
|
||||
from fastapi import FastAPI
|
||||
|
||||
from litellm.proxy.rust_forward.listener import PublicListener
|
||||
from litellm.proxy.rust_forward.mode import ServerOverrides
|
||||
from litellm.proxy.rust_forward.peer import ASGIApp
|
||||
else:
|
||||
FastAPI = Any
|
||||
|
||||
|
|
@ -261,6 +266,7 @@ def _apply_config_logging_settings(litellm_settings: StartupOutputSettings | Non
|
|||
|
||||
|
||||
WORKER_APP: Final = "litellm_cli.main:proxy_worker_app"
|
||||
FORWARD_APP: Final = "litellm.proxy.rust_forward.asgi:app"
|
||||
WORKER_LOG_CONFIG_ENV: Final = "LITELLM_WORKER_LOG_CONFIG"
|
||||
_LOG_CONFIG: Final = TypeAdapter[str | Mapping[str, object]](str | Mapping[str, object])
|
||||
_LOG_CONFIG_FILE: Final = TypeAdapter[Mapping[str, object]](Mapping[str, object])
|
||||
|
|
@ -389,12 +395,12 @@ class ProxyInitializationHelpers:
|
|||
logging.config.fileConfig(log_config, disable_existing_loggers=False)
|
||||
|
||||
@staticmethod
|
||||
def child_process_uvicorn_overrides(log_config: object) -> Mapping[str, object]:
|
||||
def child_process_uvicorn_overrides(log_config: object, app: str = WORKER_APP) -> Mapping[str, object]:
|
||||
if log_config is None:
|
||||
return MappingProxyType({"app": WORKER_APP, "factory": True})
|
||||
return MappingProxyType({"app": app, "factory": True})
|
||||
ProxyInitializationHelpers.apply_log_config(_LOG_CONFIG.validate_python(log_config))
|
||||
os.environ[WORKER_LOG_CONFIG_ENV] = json.dumps(log_config)
|
||||
return MappingProxyType({"app": WORKER_APP, "factory": True, "log_config": None})
|
||||
return MappingProxyType({"app": app, "factory": True, "log_config": None})
|
||||
|
||||
@staticmethod
|
||||
def apply_worker_log_config(environ: Mapping[str, str]) -> None:
|
||||
|
|
@ -409,6 +415,7 @@ class ProxyInitializationHelpers:
|
|||
log_config: str | None = None,
|
||||
keepalive_timeout: int | None = None,
|
||||
timeout_worker_healthcheck: int | None = None,
|
||||
forwarded: "ServerOverrides | None" = None,
|
||||
) -> dict:
|
||||
"""
|
||||
Get the arguments for `uvicorn` worker
|
||||
|
|
@ -427,6 +434,16 @@ class ProxyInitializationHelpers:
|
|||
"port": port,
|
||||
"server_header": False,
|
||||
"timeout_graceful_shutdown": GracefulShutdownManager.get_timeout(),
|
||||
**(
|
||||
{}
|
||||
if forwarded is None
|
||||
else {
|
||||
"app": FORWARD_APP,
|
||||
"factory": True,
|
||||
"fd": forwarded.fd,
|
||||
"proxy_headers": forwarded.proxy_headers,
|
||||
}
|
||||
),
|
||||
}
|
||||
if log_config is not None:
|
||||
print_startup_message(f"Using log_config: {log_config}")
|
||||
|
|
@ -674,12 +691,13 @@ class ProxyInitializationHelpers:
|
|||
def run_gunicorn_server(
|
||||
host: str,
|
||||
port: int,
|
||||
app: FastAPI,
|
||||
app: "ASGIApp",
|
||||
num_workers: int,
|
||||
ssl_certfile_path: str,
|
||||
ssl_keyfile_path: str,
|
||||
max_requests_before_restart: int | None = None,
|
||||
max_requests_before_restart_jitter: int | None = None,
|
||||
forwarded: "ServerOverrides | None" = None,
|
||||
):
|
||||
"""
|
||||
Run litellm with `gunicorn`
|
||||
|
|
@ -752,9 +770,11 @@ class ProxyInitializationHelpers:
|
|||
)
|
||||
graceful_shutdown_timeout: Final = GracefulShutdownManager.get_timeout()
|
||||
gunicorn_options: Final = {
|
||||
"bind": f"{host}:{port}",
|
||||
"bind": f"{host}:{port}" if forwarded is None else f"fd://{forwarded.fd}",
|
||||
"workers": num_workers, # default is 1
|
||||
"worker_class": build_graceful_shutdown_uvicorn_worker(graceful_shutdown_timeout),
|
||||
"worker_class": build_graceful_shutdown_uvicorn_worker(
|
||||
graceful_shutdown_timeout, proxy_headers=True if forwarded is None else forwarded.proxy_headers
|
||||
),
|
||||
"graceful_timeout": math.ceil(graceful_shutdown_timeout) + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS,
|
||||
"preload": True, # Add the preload flag,
|
||||
"accesslog": "-", # Log to stdout
|
||||
|
|
@ -815,7 +835,7 @@ class ProxyInitializationHelpers:
|
|||
_run_ollama_serve = run_ollama_serve
|
||||
|
||||
@staticmethod
|
||||
def is_port_in_use(port):
|
||||
def is_port_in_use(port: int):
|
||||
import socket
|
||||
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
|
|
@ -823,6 +843,34 @@ class ProxyInitializationHelpers:
|
|||
|
||||
_is_port_in_use = is_port_in_use
|
||||
|
||||
@staticmethod
|
||||
def public_listener(host: str, port: int, keepalive_timeout: int | None, run_gunicorn: bool) -> "PublicListener":
|
||||
"""``run_gunicorn_server`` never hands ``--keepalive_timeout`` to gunicorn, so its workers keep gunicorn's."""
|
||||
from gunicorn.config import Config as GunicornConfig
|
||||
|
||||
from litellm.proxy.rust_forward.listener import (
|
||||
PublicListener,
|
||||
shutdown_drain_seconds,
|
||||
uvicorn_default_keepalive,
|
||||
)
|
||||
from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager
|
||||
|
||||
keepalive: Final = (
|
||||
TypeAdapter(int).validate_python(GunicornConfig().keepalive)
|
||||
if run_gunicorn
|
||||
else (uvicorn_default_keepalive() if keepalive_timeout is None else keepalive_timeout)
|
||||
)
|
||||
return PublicListener(
|
||||
host=host,
|
||||
port=random.randint(1024, 49152)
|
||||
if port == 4000 and ProxyInitializationHelpers.is_port_in_use(port)
|
||||
else port,
|
||||
keepalive_seconds=keepalive,
|
||||
shutdown_drain_seconds=shutdown_drain_seconds(GracefulShutdownManager.get_timeout()),
|
||||
proxy_headers=True,
|
||||
trusted_proxies=None,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_loop_type():
|
||||
"""Helper function to determine the event loop type based on platform"""
|
||||
|
|
@ -1210,8 +1258,8 @@ class ProxyInitializationHelpers:
|
|||
)
|
||||
def run_server(
|
||||
cli_args,
|
||||
host,
|
||||
port,
|
||||
host: str,
|
||||
port: int,
|
||||
api_base,
|
||||
api_version,
|
||||
model,
|
||||
|
|
@ -1280,6 +1328,31 @@ def run_server(
|
|||
run_setup_wizard()
|
||||
return
|
||||
|
||||
from litellm.proxy.rust_forward.mode import Supervised, Supervisor, rust_forward_mode, server_overrides
|
||||
|
||||
forward_mode: Final = rust_forward_mode(os.environ)
|
||||
if isinstance(forward_mode, Supervisor) and not any(
|
||||
(version, validate_config, health, test, skip_server_startup, check_migrations_against_existing_data)
|
||||
):
|
||||
from litellm.proxy.rust_forward.listener import StartupError, startup_check
|
||||
from litellm.proxy.rust_forward.supervisor import cli_command, gateway_binary, supervise
|
||||
|
||||
binary: Final = startup_check(
|
||||
gateway_binary(sysconfig.get_path("scripts")),
|
||||
{
|
||||
"--ssl_keyfile_path": ssl_keyfile_path,
|
||||
"--ssl_certfile_path": ssl_certfile_path,
|
||||
"--ciphers": ciphers,
|
||||
"--run_hypercorn": run_hypercorn,
|
||||
"--run_granian": run_granian,
|
||||
},
|
||||
)
|
||||
if isinstance(binary, StartupError):
|
||||
raise click.ClickException(binary.reason)
|
||||
listener: Final = ProxyInitializationHelpers.public_listener(host, port, keepalive_timeout, run_gunicorn)
|
||||
raise SystemExit(supervise(binary, cli_command(sys.argv[1:]), listener, os.environ))
|
||||
forwarded: Final = server_overrides(forward_mode) if isinstance(forward_mode, Supervised) else None
|
||||
|
||||
args: Final = locals()
|
||||
try:
|
||||
from litellm.proxy.proxy_server import (
|
||||
|
|
@ -1729,14 +1802,15 @@ def run_server(
|
|||
if prometheus_metrics_port is not None and prometheus_multiproc_dir is not None:
|
||||
from litellm.proxy.prometheus_metrics_server import MetricsServerStartupError, start_metrics_server_process
|
||||
|
||||
metrics_bind_host: Final = host if forwarded is None else forwarded.metrics_host
|
||||
try:
|
||||
metrics_process: Final = start_metrics_server_process(
|
||||
host=host, port=prometheus_metrics_port, multiproc_dir=prometheus_multiproc_dir
|
||||
host=metrics_bind_host, port=prometheus_metrics_port, multiproc_dir=prometheus_multiproc_dir
|
||||
)
|
||||
except MetricsServerStartupError as error:
|
||||
raise click.ClickException(str(error)) from error
|
||||
print_startup_message(
|
||||
f"\033[1;32mLiteLLM: Serving Prometheus metrics on {host}:{prometheus_metrics_port}/metrics "
|
||||
f"\033[1;32mLiteLLM: Serving Prometheus metrics on {metrics_bind_host}:{prometheus_metrics_port}/metrics "
|
||||
f"(pid {metrics_process.pid})\033[0m"
|
||||
)
|
||||
|
||||
|
|
@ -1747,6 +1821,7 @@ def run_server(
|
|||
log_config=log_config,
|
||||
keepalive_timeout=keepalive_timeout,
|
||||
timeout_worker_healthcheck=(timeout_worker_healthcheck if running_uvicorn else None),
|
||||
forwarded=forwarded,
|
||||
)
|
||||
# Optional: recycle uvicorn workers after N requests
|
||||
if max_requests_before_restart is not None:
|
||||
|
|
@ -1786,7 +1861,7 @@ def run_server(
|
|||
start_query_engine_reaper()
|
||||
if num_workers > 1 or reload:
|
||||
worker_overrides: Final = ProxyInitializationHelpers.child_process_uvicorn_overrides(
|
||||
uvicorn_args.get("log_config")
|
||||
uvicorn_args.get("log_config"), WORKER_APP if forwarded is None else FORWARD_APP
|
||||
)
|
||||
if reload and num_workers > 1:
|
||||
from litellm.proxy.dev_reload import (
|
||||
|
|
@ -1812,15 +1887,18 @@ def run_server(
|
|||
workers=num_workers,
|
||||
)
|
||||
elif run_gunicorn is True:
|
||||
from litellm.proxy.rust_forward.peer import wrap
|
||||
|
||||
ProxyInitializationHelpers.run_gunicorn_server(
|
||||
host=host,
|
||||
port=port,
|
||||
app=app,
|
||||
app=wrap(app, forward_mode),
|
||||
num_workers=num_workers,
|
||||
ssl_certfile_path=ssl_certfile_path,
|
||||
ssl_keyfile_path=ssl_keyfile_path,
|
||||
max_requests_before_restart=max_requests_before_restart,
|
||||
max_requests_before_restart_jitter=max_requests_before_restart_jitter,
|
||||
forwarded=forwarded,
|
||||
)
|
||||
elif run_hypercorn is True:
|
||||
ProxyInitializationHelpers.init_hypercorn_server(
|
||||
|
|
|
|||
8
litellm/proxy/rust_forward/AGENTS.md
Normal file
8
litellm/proxy/rust_forward/AGENTS.md
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
# Rust forwarding, Python half
|
||||
|
||||
With `LITELLM_RUST` on, the supervisor here runs the Rust gateway on the public port and Python on a private loopback listener. This package is the Python half of forwarding and pairs with the crate `litellm-rust/crates/gateway-forward`. Once Rust serves every route, both get deleted together, along with one dispatch branch in each entry point
|
||||
|
||||
- Nothing outside this package imports Rust forwarding concepts, except the dispatch, the flag adapter and the app wrap in `litellm/proxy/proxy_cli.py` and `gateway/launch.py`. The proxy app itself never knows Rust is in front of it
|
||||
- Python must see each request as if the client had connected directly. `x-litellm-rust-peer` is the only Rust-specific input, and `peer.py` honors it only from the loopback hop
|
||||
- Python owns every setting Rust needs. The supervisor resolves the host, port, keepalive and shutdown drain Python actually runs with and hands them to Rust as the `LITELLM_FORWARD_*` variables in `rust_spec`. Rust never reads Python's own settings, so a new value Rust needs gets a new variable there
|
||||
- Entry points turn their own flags into a `PublicListener`, apply `server_overrides` in the supervised child, and wrap the app with `peer.wrap` where they load it
|
||||
0
litellm/proxy/rust_forward/__init__.py
Normal file
0
litellm/proxy/rust_forward/__init__.py
Normal file
10
litellm/proxy/rust_forward/asgi.py
Normal file
10
litellm/proxy/rust_forward/asgi.py
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
import os
|
||||
|
||||
from litellm.proxy.rust_forward.mode import rust_forward_mode
|
||||
from litellm.proxy.rust_forward.peer import ASGIApp, wrap
|
||||
|
||||
|
||||
def app() -> ASGIApp:
|
||||
from litellm_cli.main import proxy_worker_app
|
||||
|
||||
return wrap(proxy_worker_app(), rust_forward_mode(os.environ))
|
||||
46
litellm/proxy/rust_forward/listener.py
Normal file
46
litellm/proxy/rust_forward/listener.py
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
import os
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
from litellm.rust_bridge.configuration import ENV_NAME
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PublicListener:
|
||||
host: str
|
||||
port: int
|
||||
keepalive_seconds: int
|
||||
shutdown_drain_seconds: float
|
||||
proxy_headers: bool
|
||||
trusted_proxies: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class StartupError:
|
||||
reason: str
|
||||
|
||||
|
||||
def startup_check(binary: str | None, unsupported: Mapping[str, object]) -> str | StartupError:
|
||||
flag: Final = next((name for name, value in unsupported.items() if value), None)
|
||||
if flag is not None:
|
||||
return StartupError(f"{ENV_NAME} does not support {flag}")
|
||||
if os.name != "posix":
|
||||
return StartupError(f"{ENV_NAME} requires POSIX inherited sockets and process signals")
|
||||
if binary is None:
|
||||
return StartupError(
|
||||
f"{ENV_NAME} requires the litellm-gateway binary. Install litellm[proxy] in this environment"
|
||||
)
|
||||
return binary
|
||||
|
||||
|
||||
def uvicorn_default_keepalive() -> int:
|
||||
import uvicorn
|
||||
|
||||
return uvicorn.Config(app="", log_config=None).timeout_keep_alive
|
||||
|
||||
|
||||
def shutdown_drain_seconds(graceful_timeout: float) -> float:
|
||||
from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS
|
||||
|
||||
return graceful_timeout + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS
|
||||
68
litellm/proxy/rust_forward/mode.py
Normal file
68
litellm/proxy/rust_forward/mode.py
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.rust_bridge.configuration import environment_switch
|
||||
|
||||
GATEWAY_FD: Final = "_LITELLM_GATEWAY_FD"
|
||||
METRICS_HOST: Final = "_LITELLM_GATEWAY_METRICS_HOST"
|
||||
PROXY_HEADERS: Final = "_LITELLM_GATEWAY_PROXY_HEADERS"
|
||||
TRUSTED_PROXIES: Final = "FORWARDED_ALLOW_IPS"
|
||||
DEFAULT_TRUSTED_PROXIES: Final = "127.0.0.1"
|
||||
RUST_HOP: Final = "127.0.0.1"
|
||||
|
||||
_FD: Final = TypeAdapter(int)
|
||||
_ENABLED: Final = TypeAdapter(bool)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Disabled:
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Supervisor:
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Supervised:
|
||||
listener_fd: int
|
||||
metrics_host: str
|
||||
proxy_headers: bool
|
||||
trusted_proxies: str
|
||||
|
||||
|
||||
RustForwardMode: TypeAlias = Disabled | Supervisor | Supervised
|
||||
|
||||
|
||||
def rust_forward_mode(environ: Mapping[str, str]) -> RustForwardMode:
|
||||
fd: Final = environ.get(GATEWAY_FD)
|
||||
if fd is not None:
|
||||
return Supervised(
|
||||
_FD.validate_python(fd),
|
||||
environ[METRICS_HOST],
|
||||
_ENABLED.validate_python(environ.get(PROXY_HEADERS, "true")),
|
||||
environ.get(TRUSTED_PROXIES, DEFAULT_TRUSTED_PROXIES),
|
||||
)
|
||||
if environment_switch(environ):
|
||||
return Supervisor()
|
||||
return Disabled()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ServerOverrides:
|
||||
fd: int
|
||||
proxy_headers: bool
|
||||
metrics_host: str
|
||||
|
||||
|
||||
def server_overrides(mode: Supervised) -> ServerOverrides:
|
||||
"""The private listener the supervised server binds instead of its own flags.
|
||||
|
||||
Proxy headers stay off at the server, since it would judge Rust's loopback address instead of the real peer.
|
||||
``peer.wrap`` applies the operator's trust settings to the restored peer.
|
||||
"""
|
||||
return ServerOverrides(fd=mode.listener_fd, proxy_headers=False, metrics_host=mode.metrics_host)
|
||||
57
litellm/proxy/rust_forward/peer.py
Normal file
57
litellm/proxy/rust_forward/peer.py
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
from collections.abc import Awaitable, Callable
|
||||
from typing import TYPE_CHECKING, Final, TypeAlias
|
||||
|
||||
from typing_extensions import assert_never
|
||||
|
||||
from litellm.proxy.rust_forward.mode import RUST_HOP, Disabled, RustForwardMode, Supervised, Supervisor
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from uvicorn._types import ASGIReceiveCallable, ASGISendCallable, Scope, WWWScope
|
||||
|
||||
PEER_HEADER: Final = b"x-litellm-rust-peer"
|
||||
|
||||
ASGIApp: TypeAlias = Callable[..., Awaitable[None]]
|
||||
|
||||
|
||||
def _restore_peer(scope: "WWWScope") -> None:
|
||||
headers: Final = tuple(scope["headers"])
|
||||
peer: Final = next((value for name, value in headers if name == PEER_HEADER), None)
|
||||
if peer is None:
|
||||
return
|
||||
remaining: Final = tuple((name, value) for name, value in headers if name != PEER_HEADER)
|
||||
scope["headers"] = remaining # rebind-ok: the peer header is private to the Rust hop
|
||||
client: Final = scope["client"]
|
||||
if client is None or client[0] != RUST_HOP:
|
||||
return
|
||||
host, _, port = peer.decode("latin-1").rpartition(":")
|
||||
scope["client"] = (host.strip("[]"), int(port)) # rebind-ok: uvicorn's access log reads this same scope
|
||||
|
||||
|
||||
class RustPeerMiddleware:
|
||||
"""Make Python see each request as if the client had connected to it directly.
|
||||
|
||||
Rust owns the public port, so Python's peer is always Rust on loopback. Rust passes the real
|
||||
TCP peer in ``x-litellm-rust-peer`` and leaves every forwarding header untouched. This puts the
|
||||
real peer back into the scope, then runs uvicorn's own ``ProxyHeadersMiddleware`` with the trust
|
||||
settings Python would have used on the public port.
|
||||
"""
|
||||
|
||||
def __init__(self, app: ASGIApp, mode: Supervised) -> None:
|
||||
from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware
|
||||
|
||||
self.app: Final = ProxyHeadersMiddleware(app, mode.trusted_proxies) if mode.proxy_headers else app
|
||||
|
||||
async def __call__(self, scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None:
|
||||
if scope["type"] != "lifespan":
|
||||
_restore_peer(scope)
|
||||
await self.app(scope, receive, send)
|
||||
|
||||
|
||||
def wrap(app: ASGIApp, mode: RustForwardMode) -> ASGIApp:
|
||||
match mode:
|
||||
case Supervised():
|
||||
return RustPeerMiddleware(app, mode)
|
||||
case Disabled() | Supervisor():
|
||||
return app
|
||||
case _:
|
||||
assert_never(mode)
|
||||
219
litellm/proxy/rust_forward/supervisor.py
Normal file
219
litellm/proxy/rust_forward/supervisor.py
Normal file
|
|
@ -0,0 +1,219 @@
|
|||
import asyncio
|
||||
import contextlib
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
import socket
|
||||
import sys
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, IntEnum
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from typing_extensions import assert_never
|
||||
|
||||
from litellm.proxy.rust_forward.listener import PublicListener
|
||||
from litellm.proxy.rust_forward.mode import GATEWAY_FD, METRICS_HOST, PROXY_HEADERS, RUST_HOP, TRUSTED_PROXIES
|
||||
|
||||
_ADDRESS: Final = TypeAdapter(tuple[str, int])
|
||||
|
||||
|
||||
def gateway_binary(scripts_dir: str) -> str | None:
|
||||
name: Final = "litellm-gateway.exe" if os.name == "nt" else "litellm-gateway"
|
||||
return shutil.which(name, path=scripts_dir)
|
||||
|
||||
|
||||
class Child(Enum):
|
||||
RUST = "rust"
|
||||
PYTHON = "python"
|
||||
|
||||
|
||||
DRAIN_ORDER: Final = (Child.RUST, Child.PYTHON)
|
||||
|
||||
|
||||
class ExitCode(IntEnum):
|
||||
CLEAN = 0
|
||||
CHILD_FAILED = 1
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Stop:
|
||||
signum: signal.Signals
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Exited:
|
||||
child: Child
|
||||
code: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Running:
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Draining:
|
||||
signum: signal.Signals
|
||||
signalled: Child
|
||||
remaining: tuple[Child, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Finished:
|
||||
code: ExitCode
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Signal:
|
||||
child: Child
|
||||
signum: signal.Signals
|
||||
|
||||
|
||||
Event: TypeAlias = Stop | Exited
|
||||
State: TypeAlias = Running | Draining | Finished
|
||||
|
||||
|
||||
def _signal_next(signum: signal.Signals, order: tuple[Child, ...]) -> tuple[State, Signal | None]:
|
||||
if not order:
|
||||
return Finished(ExitCode.CLEAN), None
|
||||
return Draining(signum, order[0], order[1:]), Signal(order[0], signum)
|
||||
|
||||
|
||||
def step(state: State, event: Event) -> tuple[State, Signal | None]:
|
||||
match state:
|
||||
case Running():
|
||||
match event:
|
||||
case Stop(signum):
|
||||
return _signal_next(signum, DRAIN_ORDER)
|
||||
case Exited():
|
||||
return Finished(ExitCode.CHILD_FAILED), None
|
||||
case _:
|
||||
assert_never(event)
|
||||
case Draining(signum, signalled, remaining):
|
||||
match event:
|
||||
case Stop():
|
||||
return state, None
|
||||
case Exited(child, code) if child is signalled and code == 0:
|
||||
return _signal_next(signum, remaining)
|
||||
case Exited():
|
||||
return Finished(ExitCode.CHILD_FAILED), None
|
||||
case _:
|
||||
assert_never(event)
|
||||
case Finished():
|
||||
return state, None
|
||||
case _:
|
||||
assert_never(state)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ChildSpec:
|
||||
argv: tuple[str, ...]
|
||||
env: Mapping[str, str]
|
||||
pass_fds: tuple[int, ...]
|
||||
|
||||
|
||||
def python_spec(
|
||||
command: Sequence[str], listener_fd: int, upstream_port: int, listener: PublicListener, environ: Mapping[str, str]
|
||||
) -> ChildSpec:
|
||||
return ChildSpec(
|
||||
argv=(*command, "--host", RUST_HOP, "--port", str(upstream_port)),
|
||||
env=MappingProxyType(
|
||||
{
|
||||
**environ,
|
||||
GATEWAY_FD: str(listener_fd),
|
||||
METRICS_HOST: listener.host,
|
||||
PROXY_HEADERS: str(listener.proxy_headers).lower(),
|
||||
**({} if listener.trusted_proxies is None else {TRUSTED_PROXIES: listener.trusted_proxies}),
|
||||
}
|
||||
),
|
||||
pass_fds=(listener_fd,),
|
||||
)
|
||||
|
||||
|
||||
def rust_spec(binary: str, upstream_port: int, listener: PublicListener, environ: Mapping[str, str]) -> ChildSpec:
|
||||
return ChildSpec(
|
||||
argv=(binary,),
|
||||
env=MappingProxyType(
|
||||
{
|
||||
**environ,
|
||||
"LITELLM_FORWARD_UPSTREAM": f"http://{RUST_HOP}:{upstream_port}",
|
||||
"LITELLM_FORWARD_HOST": listener.host,
|
||||
"LITELLM_FORWARD_PORT": str(listener.port),
|
||||
"LITELLM_FORWARD_UPSTREAM_KEEPALIVE": str(listener.keepalive_seconds),
|
||||
"LITELLM_FORWARD_SHUTDOWN_DRAIN": str(listener.shutdown_drain_seconds),
|
||||
}
|
||||
),
|
||||
pass_fds=(),
|
||||
)
|
||||
|
||||
|
||||
async def _spawn(spec: ChildSpec) -> asyncio.subprocess.Process:
|
||||
return await asyncio.create_subprocess_exec(
|
||||
*spec.argv, env=dict(spec.env), pass_fds=spec.pass_fds, start_new_session=True
|
||||
)
|
||||
|
||||
|
||||
def _terminate_group(process: asyncio.subprocess.Process) -> None:
|
||||
with contextlib.suppress(ProcessLookupError):
|
||||
os.killpg(process.pid, signal.SIGTERM)
|
||||
|
||||
|
||||
async def _report_exit(child: Child, process: asyncio.subprocess.Process, events: asyncio.Queue[Event]) -> None:
|
||||
events.put_nowait(Exited(child, await process.wait()))
|
||||
|
||||
|
||||
async def _drive(
|
||||
state: State, events: asyncio.Queue[Event], children: Mapping[Child, asyncio.subprocess.Process]
|
||||
) -> ExitCode:
|
||||
if isinstance(state, Finished):
|
||||
return state.code
|
||||
following, action = step(state, await events.get())
|
||||
if action is not None:
|
||||
with contextlib.suppress(ProcessLookupError):
|
||||
children[action.child].send_signal(action.signum)
|
||||
return await _drive(following, events, children)
|
||||
|
||||
|
||||
async def _supervise(
|
||||
binary: str, command: Sequence[str], listener: PublicListener, environ: Mapping[str, str]
|
||||
) -> ExitCode:
|
||||
loop: Final = asyncio.get_running_loop()
|
||||
events: Final[asyncio.Queue[Event]] = asyncio.Queue()
|
||||
for signum in (signal.SIGTERM, signal.SIGINT):
|
||||
loop.add_signal_handler(signum, events.put_nowait, Stop(signum))
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as private:
|
||||
private.bind((RUST_HOP, 0))
|
||||
private.listen()
|
||||
_, upstream_port = _ADDRESS.validate_python(private.getsockname())
|
||||
python: Final = await _spawn(python_spec(command, private.fileno(), upstream_port, listener, environ))
|
||||
try:
|
||||
rust: Final = await _spawn(rust_spec(binary, upstream_port, listener, environ))
|
||||
except OSError:
|
||||
_terminate_group(python)
|
||||
await python.wait()
|
||||
raise
|
||||
children: Final = MappingProxyType({Child.RUST: rust, Child.PYTHON: python})
|
||||
waiters: Final = tuple(
|
||||
asyncio.create_task(_report_exit(child, process, events)) for child, process in children.items()
|
||||
)
|
||||
try:
|
||||
return await _drive(Running(), events, children)
|
||||
finally:
|
||||
for process in children.values():
|
||||
_terminate_group(process)
|
||||
await asyncio.gather(*waiters)
|
||||
|
||||
|
||||
def supervise(binary: str, command: Sequence[str], listener: PublicListener, environ: Mapping[str, str]) -> int:
|
||||
try:
|
||||
return asyncio.run(_supervise(binary, command, listener, environ))
|
||||
except OSError as error:
|
||||
sys.stderr.write(f"Rust gateway startup failed: {error}\n")
|
||||
return ExitCode.CHILD_FAILED
|
||||
|
||||
|
||||
def cli_command(argv: Sequence[str]) -> tuple[str, ...]:
|
||||
return (sys.executable, "-m", "litellm.proxy.proxy_cli", *argv)
|
||||
|
|
@ -1,10 +1,16 @@
|
|||
from uvicorn.workers import UvicornWorker
|
||||
|
||||
|
||||
def build_graceful_shutdown_uvicorn_worker(timeout_graceful_shutdown: float) -> type[UvicornWorker]:
|
||||
def build_graceful_shutdown_uvicorn_worker(
|
||||
timeout_graceful_shutdown: float, *, proxy_headers: bool = True
|
||||
) -> type[UvicornWorker]:
|
||||
class GracefulShutdownUvicornWorker(UvicornWorker):
|
||||
def __init__(self, *args: object) -> None:
|
||||
self.CONFIG_KWARGS = {**UvicornWorker.CONFIG_KWARGS, "timeout_graceful_shutdown": timeout_graceful_shutdown}
|
||||
self.CONFIG_KWARGS = {
|
||||
**UvicornWorker.CONFIG_KWARGS,
|
||||
"timeout_graceful_shutdown": timeout_graceful_shutdown,
|
||||
"proxy_headers": proxy_headers,
|
||||
}
|
||||
super().__init__(*args)
|
||||
|
||||
return GracefulShutdownUvicornWorker
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import Mapping
|
||||
from enum import Enum, auto
|
||||
from functools import lru_cache
|
||||
from typing import Final
|
||||
|
|
@ -8,7 +9,7 @@ from typing import Final
|
|||
from pydantic import TypeAdapter, ValidationError
|
||||
from typing_extensions import assert_never
|
||||
|
||||
_GLOBAL_ENV_NAME: Final = "LITELLM_RUST"
|
||||
ENV_NAME: Final = "LITELLM_RUST"
|
||||
_ENV_BOOL: Final = TypeAdapter(bool)
|
||||
|
||||
|
||||
|
|
@ -44,6 +45,10 @@ def _parse_env_bool(value: str | None) -> bool | None:
|
|||
return None
|
||||
|
||||
|
||||
def environment_switch(environ: Mapping[str, str]) -> bool | None:
|
||||
return _parse_env_bool(environ.get(ENV_NAME))
|
||||
|
||||
|
||||
def decide(
|
||||
rollout: Rollout,
|
||||
*,
|
||||
|
|
@ -72,7 +77,7 @@ def decision(rollout: Rollout) -> Decision:
|
|||
return decide(
|
||||
rollout,
|
||||
process_override=_CONFIGURATION.override,
|
||||
environment_override=_parse_env_bool(os.getenv(_GLOBAL_ENV_NAME)),
|
||||
environment_override=environment_switch(os.environ),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
180
tests/integration/configuration/test_rust_forward.py
Normal file
180
tests/integration/configuration/test_rust_forward.py
Normal file
|
|
@ -0,0 +1,180 @@
|
|||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from tests.integration._support.client import Gateway, eventually
|
||||
from tests.integration._support.ports import allocate_port
|
||||
from tests.integration._support.process import stop_root_process
|
||||
|
||||
TRUSTED_BALANCER: Final = "127.0.0.3"
|
||||
|
||||
|
||||
def ready(client: httpx.Client) -> bool:
|
||||
try:
|
||||
return client.get("/health/liveliness").status_code == 200
|
||||
except httpx.TransportError:
|
||||
return False
|
||||
|
||||
|
||||
def client_ip(base_url: str, local_address: str, key: str, forwarded_for: str) -> httpx.Response:
|
||||
with httpx.Client(
|
||||
base_url=base_url,
|
||||
transport=httpx.HTTPTransport(local_address=local_address),
|
||||
trust_env=False,
|
||||
timeout=5,
|
||||
) as client:
|
||||
return client.get(
|
||||
"/v1/mcp/network/client-ip",
|
||||
headers={"Authorization": f"Bearer {key}", "X-Forwarded-For": forwarded_for},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("server_arguments", [(), ("--run_gunicorn",)])
|
||||
def test_gateway_preserves_metrics_bind_and_mcp_client_ip(
|
||||
gateway: Gateway, tmp_path: Path, server_arguments: tuple[str, ...]
|
||||
) -> None:
|
||||
config: Final = tmp_path / "config.yaml"
|
||||
config.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"model_list": [],
|
||||
"general_settings": {"master_key": "os.environ/LITELLM_MASTER_KEY", "use_x_forwarded_for": False},
|
||||
}
|
||||
)
|
||||
)
|
||||
metrics_port: Final = allocate_port()
|
||||
public_port: Final = allocate_port()
|
||||
environment: Final = {
|
||||
**{key: value for key, value in os.environ.items() if key not in ("DATABASE_URL", "DATABASE_URL_READ_REPLICA")},
|
||||
"LITELLM_MASTER_KEY": gateway.key,
|
||||
"STORE_MODEL_IN_DB": "False",
|
||||
"PYTHON_DOTENV_DISABLED": "1",
|
||||
"LITELLM_RUST": "true",
|
||||
"FORWARDED_ALLOW_IPS": TRUSTED_BALANCER,
|
||||
}
|
||||
with (tmp_path / "proxy.log").open("wb") as output:
|
||||
with subprocess.Popen(
|
||||
(
|
||||
sys.executable,
|
||||
"-I",
|
||||
"-m",
|
||||
"litellm.proxy.proxy_cli",
|
||||
"--config",
|
||||
str(config),
|
||||
"--host",
|
||||
"0.0.0.0",
|
||||
"--port",
|
||||
str(public_port),
|
||||
"--num_workers",
|
||||
"2",
|
||||
"--timeout_worker_healthcheck",
|
||||
"120",
|
||||
"--prometheus_metrics_port",
|
||||
str(metrics_port),
|
||||
*server_arguments,
|
||||
),
|
||||
env=environment,
|
||||
stdout=output,
|
||||
stderr=output,
|
||||
) as supervisor:
|
||||
try:
|
||||
with httpx.Client(
|
||||
base_url=f"http://127.0.0.1:{public_port}",
|
||||
transport=httpx.HTTPTransport(local_address="127.0.0.2"),
|
||||
trust_env=False,
|
||||
follow_redirects=True,
|
||||
timeout=5,
|
||||
) as external:
|
||||
assert eventually(lambda: ready(external), bool, seconds=180), (tmp_path / "proxy.log").read_text()
|
||||
response: Final = external.get(f"http://127.0.0.2:{metrics_port}/metrics")
|
||||
assert response.status_code == 200, response.text
|
||||
public: Final = f"http://127.0.0.1:{public_port}"
|
||||
spoofed: Final = client_ip(public, "127.0.0.2", gateway.key, "1.2.3.4")
|
||||
assert spoofed.status_code == 200, spoofed.text
|
||||
assert spoofed.json() == {"ip": "127.0.0.2"}
|
||||
behind_balancer: Final = client_ip(public, TRUSTED_BALANCER, gateway.key, "203.0.113.9")
|
||||
assert behind_balancer.status_code == 200, behind_balancer.text
|
||||
assert behind_balancer.json() == {"ip": "203.0.113.9"}
|
||||
borrowed: Final = client_ip(public, "127.0.0.2", gateway.key, f"203.0.113.9, {TRUSTED_BALANCER}")
|
||||
assert borrowed.status_code == 200, borrowed.text
|
||||
assert borrowed.json() == {"ip": "127.0.0.2"}
|
||||
finally:
|
||||
assert stop_root_process(supervisor), (tmp_path / "proxy.log").read_text()
|
||||
assert supervisor.returncode == 0, (tmp_path / "proxy.log").read_text()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"server_arguments",
|
||||
[("--no-proxy-headers",), ("--forwarded-allow-ips", "127.0.0.1")],
|
||||
)
|
||||
def test_gateway_launcher_private_hop_preserves_client_address(
|
||||
tmp_path: Path, server_arguments: tuple[str, ...]
|
||||
) -> None:
|
||||
config: Final = tmp_path / "config.yaml"
|
||||
config.write_text(
|
||||
json.dumps({"model_list": [], "general_settings": {"master_key": "os.environ/LITELLM_MASTER_KEY"}})
|
||||
)
|
||||
public_port: Final = allocate_port()
|
||||
environment: Final = {
|
||||
**{key: value for key, value in os.environ.items() if not key.startswith("DATABASE_") and key != "DIRECT_URL"},
|
||||
"LITELLM_MASTER_KEY": "sk-rust-forwarding-integration-key-not-a-secret",
|
||||
"LITELLM_RUST": "true",
|
||||
"LITELLM_PGBOUNCER_ENABLED": "false",
|
||||
"CONFIG_FILE_PATH": str(config),
|
||||
"STORE_MODEL_IN_DB": "False",
|
||||
"PYTHON_DOTENV_DISABLED": "1",
|
||||
}
|
||||
log: Final = tmp_path / "gateway.log"
|
||||
with log.open("wb") as output:
|
||||
with subprocess.Popen(
|
||||
(
|
||||
sys.executable,
|
||||
"-I",
|
||||
"-m",
|
||||
"gateway.launch",
|
||||
"--host",
|
||||
"0.0.0.0",
|
||||
"--port",
|
||||
str(public_port),
|
||||
"--workers",
|
||||
"2",
|
||||
"--timeout-worker-healthcheck",
|
||||
"120",
|
||||
*server_arguments,
|
||||
),
|
||||
env=environment,
|
||||
stdout=output,
|
||||
stderr=output,
|
||||
) as supervisor:
|
||||
try:
|
||||
with httpx.Client(
|
||||
base_url=f"http://127.0.0.1:{public_port}",
|
||||
transport=httpx.HTTPTransport(local_address="127.0.0.2"),
|
||||
trust_env=False,
|
||||
timeout=5,
|
||||
) as external:
|
||||
assert eventually(lambda: ready(external), bool, seconds=180), log.read_text()
|
||||
response: Final = external.get(
|
||||
"/health/liveliness?client-ip-proof", headers={"X-Forwarded-For": "198.51.100.10"}
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
events: Final = eventually(
|
||||
lambda: tuple(
|
||||
line
|
||||
for line in log.read_text().splitlines()
|
||||
if "GET /health/liveliness?client-ip-proof " in line
|
||||
),
|
||||
bool,
|
||||
seconds=10,
|
||||
)
|
||||
assert len(events) == 1, log.read_text()
|
||||
assert "127.0.0.2:" in events[0], events
|
||||
finally:
|
||||
assert stop_root_process(supervisor), log.read_text()
|
||||
assert supervisor.returncode == 0, log.read_text()
|
||||
|
|
@ -12,13 +12,18 @@ from typing import Final, cast
|
|||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import uvicorn
|
||||
from uvicorn.importer import import_from_string
|
||||
from uvicorn.main import main as uvicorn_main
|
||||
|
||||
from gateway.launch import GATEWAY_APP, main, pool_database_url, uvicorn_argv
|
||||
from gateway.launch import GATEWAY_APP, main, pool_database_url, public_listener, uvicorn_argv
|
||||
from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS
|
||||
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
|
||||
from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR, PgBouncerError, PgBouncerSettings
|
||||
from litellm.proxy.proxy_server import app as proxy_app
|
||||
from litellm.proxy.rust_forward.mode import Supervised, server_overrides
|
||||
from litellm.proxy.rust_forward.supervisor import rust_spec
|
||||
from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager
|
||||
|
||||
DB_ENV: Final = {
|
||||
"DATABASE_HOST": "db.internal",
|
||||
|
|
@ -114,17 +119,89 @@ def _uvicorn_params(argv: tuple[str, ...]) -> dict[str, object]:
|
|||
|
||||
|
||||
class TestUvicornArgv:
|
||||
@pytest.mark.parametrize(
|
||||
"arguments", [("--uds", "/tmp/gateway.sock"), ("--fd", "0"), ("--fd=17",), ("--ssl-keyfile", "unused")]
|
||||
)
|
||||
def test_forwarding_rejects_unsupported_listeners_before_starting_processes(
|
||||
self, arguments: tuple[str, ...]
|
||||
) -> None:
|
||||
with pytest.raises(SystemExit, match=f"does not support {arguments[0].split('=')[0]}"):
|
||||
public_listener(arguments, "gateway")
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("argv", "trust"),
|
||||
[
|
||||
((), (True, None)),
|
||||
(("--forwarded-allow-ips", "10.0.0.5"), (True, "10.0.0.5")),
|
||||
(("--forwarded-allow-ips=*",), (True, "*")),
|
||||
(("--no-proxy-headers",), (False, None)),
|
||||
],
|
||||
)
|
||||
def test_forwarding_carries_operator_proxy_header_options_to_the_child(
|
||||
self, argv: tuple[str, ...], trust: tuple[bool, str | None], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.delenv("FORWARDED_ALLOW_IPS", raising=False)
|
||||
binary, listener = public_listener(("--host", "0.0.0.0", "--port", "4000", *argv), "gateway")
|
||||
assert (binary, listener.host, listener.port) == ("gateway", "0.0.0.0", 4000)
|
||||
assert (listener.proxy_headers, listener.trusted_proxies) == trust
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("argv", "environ"),
|
||||
[(("--timeout-keep-alive", "2"), {}), ((), {"KEEPALIVE_TIMEOUT": "7"}), ((), {})],
|
||||
)
|
||||
def test_rust_retires_connections_by_the_keepalive_uvicorn_runs_with(
|
||||
self, argv: tuple[str, ...], environ: dict[str, str], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False)
|
||||
for name, value in environ.items():
|
||||
monkeypatch.setenv(name, value)
|
||||
_, listener = public_listener(argv, "gateway")
|
||||
sent: Final = rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_UPSTREAM_KEEPALIVE"]
|
||||
assert sent == str(_uvicorn_params(uvicorn_argv(argv, os.environ, None))["timeout_keep_alive"])
|
||||
|
||||
def test_without_a_keepalive_rust_follows_uvicorns_own_default(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False)
|
||||
_, listener = public_listener((), "gateway")
|
||||
assert listener.keepalive_seconds == uvicorn.Config(app="", log_config=None).timeout_keep_alive
|
||||
|
||||
@pytest.mark.parametrize("argv", [(), ("--timeout-graceful-shutdown", "12")])
|
||||
def test_rust_drains_for_pythons_graceful_timeout_plus_teardown(
|
||||
self, argv: tuple[str, ...], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("GRACEFUL_SHUTDOWN_TIMEOUT", "7.5")
|
||||
_, listener = public_listener(argv, "gateway")
|
||||
flag: Final = _uvicorn_params(uvicorn_argv(argv, {}, None))["timeout_graceful_shutdown"]
|
||||
graceful: Final = GracefulShutdownManager.get_timeout() if flag is None else float(str(flag))
|
||||
sent: Final = rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_SHUTDOWN_DRAIN"]
|
||||
assert float(sent) == graceful + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS
|
||||
|
||||
def test_supervised_listener_uses_the_inherited_loopback_socket(self):
|
||||
supervised: Final = server_overrides(Supervised(17, "0.0.0.0", True, "127.0.0.1"))
|
||||
params: Final = _uvicorn_params(uvicorn_argv(("--workers", "2"), {}, supervised))
|
||||
assert params["fd"] == 17
|
||||
assert params["workers"] == 2
|
||||
|
||||
@pytest.mark.parametrize("argv", [(), ("--proxy-headers",), ("--forwarded-allow-ips=*",)])
|
||||
def test_private_hop_leaves_proxy_headers_to_the_app(self, argv: tuple[str, ...]):
|
||||
params: Final = _uvicorn_params(uvicorn_argv(argv, {}, server_overrides(Supervised(17, "0.0.0.0", True, "*"))))
|
||||
assert params["proxy_headers"] is False
|
||||
|
||||
def test_direct_listener_keeps_explicit_proxy_header_options(self):
|
||||
params: Final = _uvicorn_params(uvicorn_argv(("--no-proxy-headers", "--forwarded-allow-ips=*"), {}, None))
|
||||
assert params["proxy_headers"] is False
|
||||
assert params["forwarded_allow_ips"] == "*"
|
||||
|
||||
def test_keepalive_env_reaches_uvicorn(self):
|
||||
params: Final = _uvicorn_params(uvicorn_argv(("--workers", "4"), {"KEEPALIVE_TIMEOUT": "75"}))
|
||||
params: Final = _uvicorn_params(uvicorn_argv(("--workers", "4"), {"KEEPALIVE_TIMEOUT": "75"}, None))
|
||||
assert params["app"] == GATEWAY_APP
|
||||
assert params["workers"] == 4
|
||||
assert params["timeout_keep_alive"] == 75
|
||||
|
||||
def test_unset_env_keeps_the_uvicorn_default(self):
|
||||
assert _uvicorn_params(uvicorn_argv(("--workers", "4"), {}))["timeout_keep_alive"] == 5
|
||||
assert _uvicorn_params(uvicorn_argv(("--workers", "4"), {}, None))["timeout_keep_alive"] == 5
|
||||
|
||||
def test_an_explicit_flag_wins_over_the_env(self):
|
||||
argv: Final = uvicorn_argv(("--timeout-keep-alive", "30"), {"KEEPALIVE_TIMEOUT": "75"})
|
||||
argv: Final = uvicorn_argv(("--timeout-keep-alive", "30"), {"KEEPALIVE_TIMEOUT": "75"}, None)
|
||||
assert _uvicorn_params(argv)["timeout_keep_alive"] == 30
|
||||
|
||||
def test_the_app_uvicorn_is_told_to_serve_is_the_trimmed_gateway(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
|
|
@ -141,7 +218,7 @@ class TestUvicornArgv:
|
|||
):
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
|
||||
params: Final = _uvicorn_params(uvicorn_argv((), {}))
|
||||
params: Final = _uvicorn_params(uvicorn_argv((), {}, None))
|
||||
assert params["factory"] is True
|
||||
try:
|
||||
served: Final = import_from_string(cast(str, params["app"]))()
|
||||
|
|
@ -229,6 +306,16 @@ class TestMain:
|
|||
DatabaseURLSettings.from_env().apply_to_env()
|
||||
assert os.environ["DATABASE_URL"] == pooled
|
||||
|
||||
def test_forwarding_disabled_keeps_uvicorn_socket_options(
|
||||
self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("LITELLM_RUST", "false")
|
||||
monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False)
|
||||
monkeypatch.delenv("LITELLM_PGBOUNCER_ENABLED", raising=False)
|
||||
served: Final[list[tuple[str, ...]]] = []
|
||||
main(("--uds", "/tmp/gateway.sock"), serve=lambda argv: served.append(tuple(argv)))
|
||||
assert _uvicorn_params(served[0])["uds"] == "/tmp/gateway.sock"
|
||||
|
||||
def test_a_pooler_that_cannot_start_stops_the_gateway_before_uvicorn(
|
||||
self, password_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
):
|
||||
|
|
|
|||
0
tests/unit/proxy/rust_forward/__init__.py
Normal file
0
tests/unit/proxy/rust_forward/__init__.py
Normal file
74
tests/unit/proxy/rust_forward/test_listener.py
Normal file
74
tests/unit/proxy/rust_forward/test_listener.py
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
import uvicorn
|
||||
from click.testing import CliRunner
|
||||
from gunicorn.config import Config as GunicornConfig
|
||||
|
||||
from litellm.constants import GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS
|
||||
from litellm.proxy.proxy_cli import ProxyInitializationHelpers, run_server
|
||||
from litellm.proxy.rust_forward.listener import PublicListener, StartupError, startup_check
|
||||
from litellm.proxy.rust_forward.supervisor import rust_spec
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"flag", ["--ssl_keyfile_path", "--ssl_certfile_path", "--ciphers", "--run_hypercorn", "--run_granian"]
|
||||
)
|
||||
def test_unsupported_modes_fail_before_starting_children(flag: str, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("LITELLM_RUST", "true")
|
||||
monkeypatch.delenv("_LITELLM_GATEWAY_FD", raising=False)
|
||||
values: Final = ["unused"] if flag.startswith("--ssl_") or flag == "--ciphers" else []
|
||||
result: Final = CliRunner().invoke(run_server, [flag, *values])
|
||||
assert result.exit_code != 0
|
||||
assert f"LITELLM_RUST does not support {flag}" in result.output
|
||||
|
||||
|
||||
def test_missing_binary_gives_install_instruction() -> None:
|
||||
assert startup_check(None, {}) == StartupError(
|
||||
"LITELLM_RUST requires the litellm-gateway binary. Install litellm[proxy] in this environment"
|
||||
)
|
||||
|
||||
|
||||
def _cli_listener(argv: list[str]) -> tuple[Mapping[str, object], PublicListener]:
|
||||
params: Final = run_server.make_context("litellm", ["--port", "45999", *argv]).params
|
||||
return params, ProxyInitializationHelpers.public_listener(
|
||||
str(params["host"]), int(str(params["port"])), params["keepalive_timeout"], params["run_gunicorn"] is True
|
||||
)
|
||||
|
||||
|
||||
def _uvicorn_keepalive(options: Mapping[str, object]) -> int:
|
||||
configured: Final = {"timeout_keep_alive": options["timeout_keep_alive"]} if "timeout_keep_alive" in options else {}
|
||||
return uvicorn.Config(app="", log_config=None, **configured).timeout_keep_alive
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("argv", "environ"),
|
||||
[(["--keepalive_timeout", "2"], {}), ([], {"KEEPALIVE_TIMEOUT": "7"}), ([], {})],
|
||||
)
|
||||
def test_rust_retires_connections_by_the_keepalive_uvicorn_runs_with(
|
||||
argv: list[str], environ: dict[str, str], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False)
|
||||
for name, value in environ.items():
|
||||
monkeypatch.setenv(name, value)
|
||||
params, listener = _cli_listener(argv)
|
||||
uvicorn_options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args(
|
||||
"127.0.0.1", 45555, keepalive_timeout=params["keepalive_timeout"]
|
||||
)
|
||||
sent: Final = rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_UPSTREAM_KEEPALIVE"]
|
||||
assert sent == str(_uvicorn_keepalive(uvicorn_options))
|
||||
|
||||
|
||||
def test_gunicorn_workers_keep_gunicorns_keepalive_and_rust_follows(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("KEEPALIVE_TIMEOUT", raising=False)
|
||||
_, listener = _cli_listener(["--run_gunicorn", "--keepalive_timeout", "9"])
|
||||
assert listener.keepalive_seconds == GunicornConfig().keepalive
|
||||
|
||||
|
||||
def test_rust_drains_for_pythons_graceful_timeout_plus_teardown(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("GRACEFUL_SHUTDOWN_TIMEOUT", "12.5")
|
||||
_, listener = _cli_listener([])
|
||||
uvicorn_options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args("127.0.0.1", 45555)
|
||||
sent: Final = float(rust_spec("gateway", 45555, listener, {}).env["LITELLM_FORWARD_SHUTDOWN_DRAIN"])
|
||||
assert sent == uvicorn_options["timeout_graceful_shutdown"] + GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS
|
||||
43
tests/unit/proxy/rust_forward/test_mode.py
Normal file
43
tests/unit/proxy/rust_forward/test_mode.py
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.proxy.proxy_cli import ProxyInitializationHelpers
|
||||
from litellm.proxy.rust_forward.mode import Disabled, Supervised, Supervisor, rust_forward_mode, server_overrides
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("environ", "expected"),
|
||||
[
|
||||
({}, Disabled()),
|
||||
({"LITELLM_RUST": "false"}, Disabled()),
|
||||
({"LITELLM_RUST": "off"}, Disabled()),
|
||||
({"LITELLM_RUST": "0"}, Disabled()),
|
||||
({"LITELLM_RUST": "true"}, Supervisor()),
|
||||
({"LITELLM_RUST": "1"}, Supervisor()),
|
||||
({"LITELLM_RUST": "yes"}, Supervisor()),
|
||||
({"LITELLM_RUST": "on"}, Supervisor()),
|
||||
({"LITELLM_RUST": " 1 "}, Supervisor()),
|
||||
({"LITELLM_RUST": "enabled"}, Disabled()),
|
||||
(
|
||||
{
|
||||
"LITELLM_RUST": "true",
|
||||
"_LITELLM_GATEWAY_FD": "17",
|
||||
"_LITELLM_GATEWAY_METRICS_HOST": "0.0.0.0",
|
||||
"FORWARDED_ALLOW_IPS": "127.0.0.1,10.0.0.5",
|
||||
},
|
||||
Supervised(17, "0.0.0.0", True, "127.0.0.1,10.0.0.5"),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_mode_is_resolved_from_environment(environ: dict[str, str], expected: object) -> None:
|
||||
assert rust_forward_mode(environ) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", [None, Supervised(17, "0.0.0.0", True, "127.0.0.1")])
|
||||
def test_inherited_socket_reaches_uvicorn(mode: Supervised | None) -> None:
|
||||
options: Final = ProxyInitializationHelpers.get_default_unvicorn_init_args(
|
||||
"127.0.0.1", 45555, forwarded=None if mode is None else server_overrides(mode)
|
||||
)
|
||||
assert options.get("fd") == (None if mode is None else mode.listener_fd)
|
||||
assert options.get("proxy_headers", True) is (mode is None)
|
||||
144
tests/unit/proxy/rust_forward/test_peer.py
Normal file
144
tests/unit/proxy/rust_forward/test_peer.py
Normal file
|
|
@ -0,0 +1,144 @@
|
|||
import asyncio
|
||||
import gc
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi import Request
|
||||
from fastapi.routing import APIRoute
|
||||
from uvicorn.importer import import_from_string
|
||||
from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from uvicorn._types import (
|
||||
ASGIReceiveCallable,
|
||||
ASGISendCallable,
|
||||
ASGISendEvent,
|
||||
HTTPDisconnectEvent,
|
||||
HTTPScope,
|
||||
Scope,
|
||||
)
|
||||
|
||||
from litellm.proxy.proxy_server import app as proxy_app
|
||||
from litellm.proxy.rust_forward.mode import Supervised
|
||||
from litellm.proxy.rust_forward.peer import wrap
|
||||
|
||||
|
||||
Headers = tuple[tuple[bytes, bytes], ...]
|
||||
|
||||
|
||||
async def _receive() -> "HTTPDisconnectEvent":
|
||||
return {"type": "http.disconnect"}
|
||||
|
||||
|
||||
async def _send(message: "ASGISendEvent") -> None:
|
||||
return None
|
||||
|
||||
|
||||
async def _seen_by_app(
|
||||
behind_rust: bool, client: tuple[str, int], headers: Headers, mode: Supervised
|
||||
) -> tuple[tuple[str, int] | None, str, Headers]:
|
||||
seen: Final[asyncio.Future[tuple[tuple[str, int] | None, str, Headers]]] = (
|
||||
asyncio.get_running_loop().create_future()
|
||||
)
|
||||
|
||||
async def probe(scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None:
|
||||
assert scope["type"] == "http"
|
||||
seen.set_result((scope["client"], scope["scheme"], tuple(scope["headers"])))
|
||||
|
||||
direct: Final = ProxyHeadersMiddleware(probe, mode.trusted_proxies) if mode.proxy_headers else probe
|
||||
app: Final = wrap(probe, mode) if behind_rust else direct
|
||||
scope: Final[HTTPScope] = {
|
||||
"type": "http",
|
||||
"asgi": {"version": "3.0", "spec_version": "2.3"},
|
||||
"http_version": "1.1",
|
||||
"method": "GET",
|
||||
"scheme": "http",
|
||||
"path": "/",
|
||||
"raw_path": b"/",
|
||||
"query_string": b"",
|
||||
"root_path": "",
|
||||
"headers": headers,
|
||||
"client": client,
|
||||
"server": ("127.0.0.1", 4000),
|
||||
}
|
||||
await app(scope, _receive, _send)
|
||||
return seen.result()
|
||||
|
||||
|
||||
PEER: Final = ("203.0.113.7", 51234)
|
||||
TRUSTED_PEER: Final = ("10.0.12.34", 40000)
|
||||
FORWARDED: Final[Headers] = (
|
||||
(b"x-forwarded-for", b"198.51.100.1, 203.0.113.7"),
|
||||
(b"x-forwarded-proto", b"https"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("peer", [PEER, TRUSTED_PEER, ("2001:db8::1", 443)])
|
||||
@pytest.mark.parametrize("headers", [(), FORWARDED, ((b"x-forwarded-for", b"10.1.2.3"), (b"host", b"public.example"))])
|
||||
@pytest.mark.parametrize(
|
||||
"mode",
|
||||
[
|
||||
Supervised(17, "0.0.0.0", True, "127.0.0.1"),
|
||||
Supervised(17, "0.0.0.0", True, "10.0.0.0/8"),
|
||||
Supervised(17, "0.0.0.0", True, "*"),
|
||||
Supervised(17, "0.0.0.0", False, "10.0.0.0/8"),
|
||||
],
|
||||
)
|
||||
async def test_python_behind_rust_sees_what_a_direct_connection_would(
|
||||
peer: tuple[str, int], headers: Headers, mode: Supervised
|
||||
) -> None:
|
||||
rendered: Final = f"[{peer[0]}]:{peer[1]}" if ":" in peer[0] else f"{peer[0]}:{peer[1]}"
|
||||
behind_rust: Final = await _seen_by_app(
|
||||
True, ("127.0.0.1", 60000), (*headers, (b"x-litellm-rust-peer", rendered.encode())), mode
|
||||
)
|
||||
direct: Final = await _seen_by_app(False, peer, headers, mode)
|
||||
assert behind_rust == direct
|
||||
|
||||
|
||||
async def test_untrusted_peer_cannot_set_the_scheme_behind_rust() -> None:
|
||||
client, scheme, _ = await _seen_by_app(
|
||||
True,
|
||||
("127.0.0.1", 60000),
|
||||
((b"x-forwarded-proto", b"https"), (b"x-litellm-rust-peer", b"203.0.113.7:51234")),
|
||||
Supervised(17, "0.0.0.0", True, "127.0.0.1"),
|
||||
)
|
||||
assert (client, scheme) == (("203.0.113.7", 51234), "http")
|
||||
|
||||
|
||||
async def test_peer_header_from_anyone_but_the_rust_hop_is_dropped_and_ignored() -> None:
|
||||
client, _, headers = await _seen_by_app(
|
||||
True,
|
||||
("192.0.2.9", 1000),
|
||||
((b"x-litellm-rust-peer", b"10.0.12.34:1"),),
|
||||
Supervised(17, "0.0.0.0", False, "127.0.0.1"),
|
||||
)
|
||||
assert (client, headers) == (("192.0.2.9", 1000), ())
|
||||
|
||||
|
||||
async def _peer_probe(request: Request) -> dict[str, object]:
|
||||
assert request.client is not None
|
||||
return {"client": [request.client.host, request.client.port], "peer": request.headers.get("x-litellm-rust-peer")}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("entry_point", ["litellm.proxy.rust_forward.asgi:app", "gateway.launch:worker_app"])
|
||||
async def test_every_entry_point_hands_routes_the_peer_rust_saw(
|
||||
entry_point: str, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("_LITELLM_GATEWAY_FD", "17")
|
||||
monkeypatch.setenv("_LITELLM_GATEWAY_METRICS_HOST", "0.0.0.0")
|
||||
monkeypatch.setattr(proxy_app.router, "lifespan_context", proxy_app.router.lifespan_context)
|
||||
monkeypatch.setattr(
|
||||
proxy_app.router, "routes", [APIRoute("/__rust_forward_peer_probe", _peer_probe), *proxy_app.router.routes]
|
||||
)
|
||||
try:
|
||||
served: Final = import_from_string(entry_point)()
|
||||
finally:
|
||||
gc.unfreeze()
|
||||
transport: Final = httpx.ASGITransport(app=served, client=("127.0.0.1", 60000))
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://proxy") as client:
|
||||
response: Final = await client.get(
|
||||
"/__rust_forward_peer_probe", headers={"x-litellm-rust-peer": "203.0.113.7:51234"}
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == {"client": ["203.0.113.7", 51234], "peer": None}
|
||||
139
tests/unit/proxy/rust_forward/test_supervisor.py
Normal file
139
tests/unit/proxy/rust_forward/test_supervisor.py
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
import functools
|
||||
import signal
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.proxy.proxy_cli import run_server
|
||||
from litellm.proxy.rust_forward.listener import PublicListener
|
||||
from litellm.proxy.rust_forward.mode import Supervised, rust_forward_mode
|
||||
from litellm.proxy.rust_forward.supervisor import (
|
||||
Child,
|
||||
Event,
|
||||
ExitCode,
|
||||
Exited,
|
||||
Finished,
|
||||
Running,
|
||||
Signal,
|
||||
State,
|
||||
Stop,
|
||||
cli_command,
|
||||
gateway_binary,
|
||||
python_spec,
|
||||
rust_spec,
|
||||
step,
|
||||
)
|
||||
|
||||
TERM: Final = signal.SIGTERM
|
||||
INT: Final = signal.SIGINT
|
||||
|
||||
|
||||
def _listener(proxy_headers: bool = True, trusted_proxies: str | None = None) -> PublicListener:
|
||||
return PublicListener("0.0.0.0", 4000, 5, 40.0, proxy_headers, trusted_proxies)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("environ", "listener", "expected"),
|
||||
[
|
||||
({}, _listener(), Supervised(17, "0.0.0.0", True, "127.0.0.1")),
|
||||
({"FORWARDED_ALLOW_IPS": "10.0.0.5"}, _listener(), Supervised(17, "0.0.0.0", True, "10.0.0.5")),
|
||||
({"FORWARDED_ALLOW_IPS": "10.0.0.5"}, _listener(True, "*"), Supervised(17, "0.0.0.0", True, "*")),
|
||||
({}, _listener(False), Supervised(17, "0.0.0.0", False, "127.0.0.1")),
|
||||
],
|
||||
)
|
||||
def test_python_child_keeps_the_trust_python_alone_would_use(
|
||||
environ: dict[str, str], listener: PublicListener, expected: Supervised
|
||||
) -> None:
|
||||
spec: Final = python_spec(("python",), 17, 45555, listener, environ)
|
||||
assert rust_forward_mode(spec.env) == expected
|
||||
|
||||
|
||||
def test_python_child_listens_privately_and_keeps_options() -> None:
|
||||
arguments: Final = ("--host", "0.0.0.0", "--port", "4000", "--num_workers", "2", "--reload")
|
||||
spec: Final = python_spec(arguments, 17, 45555, _listener(), {})
|
||||
params: Final = run_server.make_context("litellm", list(spec.argv)).params
|
||||
assert params["host"] == "127.0.0.1"
|
||||
assert params["port"] == 45555
|
||||
assert params["num_workers"] == 2
|
||||
assert params["reload"] is True
|
||||
assert spec.pass_fds == (17,)
|
||||
assert cli_command(()) == (sys.executable, "-m", "litellm.proxy.proxy_cli")
|
||||
|
||||
|
||||
def test_rust_child_serves_public_port_and_forwards_to_private_listener() -> None:
|
||||
spec: Final = rust_spec(
|
||||
"gateway", 45555, PublicListener("0.0.0.0", 4000, 2, 12.5, True, None), {"RUST_LOG": "info"}
|
||||
)
|
||||
assert spec.argv == ("gateway",)
|
||||
assert spec.pass_fds == ()
|
||||
assert dict(spec.env) == {
|
||||
"RUST_LOG": "info",
|
||||
"LITELLM_FORWARD_UPSTREAM": "http://127.0.0.1:45555",
|
||||
"LITELLM_FORWARD_HOST": "0.0.0.0",
|
||||
"LITELLM_FORWARD_PORT": "4000",
|
||||
"LITELLM_FORWARD_UPSTREAM_KEEPALIVE": "2",
|
||||
"LITELLM_FORWARD_SHUTDOWN_DRAIN": "12.5",
|
||||
}
|
||||
|
||||
|
||||
def test_binary_is_found_only_in_the_environment_scripts_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
scripts: Final = tmp_path / "scripts"
|
||||
elsewhere: Final = tmp_path / "elsewhere"
|
||||
scripts.mkdir()
|
||||
elsewhere.mkdir()
|
||||
binary: Final = elsewhere / "litellm-gateway"
|
||||
binary.write_text("#!/bin/sh\nexit 0\n")
|
||||
binary.chmod(0o755)
|
||||
monkeypatch.setenv("PATH", str(elsewhere))
|
||||
assert gateway_binary(str(scripts)) is None
|
||||
assert gateway_binary(str(elsewhere)) == str(binary)
|
||||
|
||||
|
||||
def _fold(events: tuple[Event, ...]) -> tuple[State, tuple[Signal, ...]]:
|
||||
def advance(acc: tuple[State, tuple[Signal, ...]], event: Event) -> tuple[State, tuple[Signal, ...]]:
|
||||
state, sent = acc
|
||||
following, action = step(state, event)
|
||||
return following, sent if action is None else (*sent, action)
|
||||
|
||||
return functools.reduce(advance, events, (Running(), ()))
|
||||
|
||||
|
||||
RUST: Final = Child.RUST
|
||||
PYTHON: Final = Child.PYTHON
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("events", "signals", "code"),
|
||||
[
|
||||
(
|
||||
(Stop(TERM), Exited(RUST, 0), Exited(PYTHON, 0)),
|
||||
(Signal(RUST, TERM), Signal(PYTHON, TERM)),
|
||||
ExitCode.CLEAN,
|
||||
),
|
||||
((Exited(PYTHON, 0),), (), ExitCode.CHILD_FAILED),
|
||||
((Exited(RUST, 1),), (), ExitCode.CHILD_FAILED),
|
||||
((Stop(TERM), Exited(PYTHON, 0)), (Signal(RUST, TERM),), ExitCode.CHILD_FAILED),
|
||||
((Stop(TERM), Exited(RUST, 1)), (Signal(RUST, TERM),), ExitCode.CHILD_FAILED),
|
||||
(
|
||||
(Stop(TERM), Exited(RUST, 0), Exited(PYTHON, 1)),
|
||||
(Signal(RUST, TERM), Signal(PYTHON, TERM)),
|
||||
ExitCode.CHILD_FAILED,
|
||||
),
|
||||
(
|
||||
(Stop(INT), Stop(INT), Exited(RUST, 0), Exited(PYTHON, 0)),
|
||||
(Signal(RUST, INT), Signal(PYTHON, INT)),
|
||||
ExitCode.CLEAN,
|
||||
),
|
||||
(
|
||||
(Exited(RUST, 1), Stop(TERM), Exited(PYTHON, 0)),
|
||||
(),
|
||||
ExitCode.CHILD_FAILED,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_supervisor_drains_rust_before_python_and_fails_on_unasked_exit(
|
||||
events: tuple[Event, ...], signals: tuple[Signal, ...], code: ExitCode
|
||||
) -> None:
|
||||
assert _fold(events) == (Finished(code), signals)
|
||||
|
|
@ -86,6 +86,39 @@ def test_a_broken_factory_exits_the_worker_as_a_startup_failure(
|
|||
assert exit_info.value.code == uvicorn_config.STARTUP_FAILURE
|
||||
|
||||
|
||||
@pytest.mark.parametrize("worker_overrides", [None, {"factory": True}])
|
||||
def test_a_factory_flag_in_the_options_reaches_the_workers(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, worker_overrides: dict[str, object] | None
|
||||
) -> None:
|
||||
(tmp_path / "reload_factory_flag_app.py").write_text(
|
||||
"async def served(scope, receive, send):\n"
|
||||
" await send({'type': 'http.response.start', 'status': 204, 'headers': []})\n"
|
||||
" await send({'type': 'http.response.body', 'body': b'', 'more_body': False})\n"
|
||||
"\n"
|
||||
"def factory():\n"
|
||||
" return served\n"
|
||||
)
|
||||
monkeypatch.syspath_prepend(str(tmp_path))
|
||||
config: Final = pool_config(
|
||||
RELOAD_OPTIONS.validate_python({**_options("reload_factory_flag_app:factory"), "factory": True}),
|
||||
workers=POOL_WORKERS,
|
||||
worker_overrides=worker_overrides,
|
||||
)
|
||||
config.load()
|
||||
loaded_app: Final = config.loaded_app
|
||||
assert loaded_app is not None
|
||||
sent: Final[list[Message]] = []
|
||||
|
||||
async def receive() -> Message:
|
||||
return {"type": "http.request", "body": b"", "more_body": False}
|
||||
|
||||
async def send(message: Message) -> None:
|
||||
sent.append(message)
|
||||
|
||||
asyncio.run(loaded_app({"type": "http", "method": "GET", "path": "/", "headers": []}, receive, send))
|
||||
assert sent[0] == {"type": "http.response.start", "status": 204, "headers": []}
|
||||
|
||||
|
||||
@REQUIRES_UVICORN_STARTUP_FAILURE
|
||||
@pytest.mark.parametrize(
|
||||
("module", "source"),
|
||||
|
|
@ -118,7 +151,9 @@ def test_the_reload_pool_defaults_the_worker_healthcheck_and_keeps_an_explicit_o
|
|||
explicit: Final = RELOAD_WORKER_HEALTHCHECK_SECONDS + 7
|
||||
assert pool_config(_options("app:app"), workers=2).timeout_worker_healthcheck == RELOAD_WORKER_HEALTHCHECK_SECONDS
|
||||
assert (
|
||||
pool_config({**_options("app:app"), "timeout_worker_healthcheck": explicit}, workers=2).timeout_worker_healthcheck
|
||||
pool_config(
|
||||
{**_options("app:app"), "timeout_worker_healthcheck": explicit}, workers=2
|
||||
).timeout_worker_healthcheck
|
||||
== explicit
|
||||
)
|
||||
|
||||
|
|
@ -142,7 +177,9 @@ def test_the_reload_pool_accepts_and_keeps_every_uvicorn_arg_the_proxy_cli_build
|
|||
|
||||
|
||||
def test_the_reload_pool_passes_a_uvicorn_arg_it_does_not_list_through_to_the_workers() -> None:
|
||||
config: Final = pool_config(RELOAD_OPTIONS.validate_python({**_options("app:app"), "backlog": 123}), workers=POOL_WORKERS)
|
||||
config: Final = pool_config(
|
||||
RELOAD_OPTIONS.validate_python({**_options("app:app"), "backlog": 123}), workers=POOL_WORKERS
|
||||
)
|
||||
assert (config.backlog, config.reload) == (123, False)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1550,6 +1550,7 @@ class TestProxyInitializationHelpers:
|
|||
log_config=None,
|
||||
keepalive_timeout=30,
|
||||
timeout_worker_healthcheck=None,
|
||||
forwarded=None,
|
||||
)
|
||||
mock_uvicorn_run.assert_called_once()
|
||||
|
||||
|
|
@ -1626,6 +1627,7 @@ class TestProxyInitializationHelpers:
|
|||
log_config=None,
|
||||
keepalive_timeout=None,
|
||||
timeout_worker_healthcheck=15,
|
||||
forwarded=None,
|
||||
)
|
||||
|
||||
@patch("uvicorn.run")
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue