From dc111da09bb6948d3aa43584c3b618858f398677 Mon Sep 17 00:00:00 2001 From: myu1d157h0u54nd Date: Wed, 2 Sep 2026 18:05:55 -0300 Subject: [PATCH] LiteLLM Hardened Production Stack Documentation --- docker-compose.hardened.yml | 46 ---- docker-compose.yml | 73 ------ docker-compose/README.md | 245 +++++++++++++++++++++ docker-compose/compose.yaml.template | 93 ++++++++ docker-compose/deploy.sh | 318 +++++++++++++++++++++++++++ 5 files changed, 656 insertions(+), 119 deletions(-) delete mode 100644 docker-compose.hardened.yml delete mode 100644 docker-compose.yml create mode 100644 docker-compose/README.md create mode 100644 docker-compose/compose.yaml.template create mode 100644 docker-compose/deploy.sh diff --git a/docker-compose.hardened.yml b/docker-compose.hardened.yml deleted file mode 100644 index 31d0c2e9ef2..00000000000 --- a/docker-compose.hardened.yml +++ /dev/null @@ -1,46 +0,0 @@ -services: - # Hardened stack: for testing the proxy under non-root, read-only, proxy-enforced constraints. - # Keep this file focused on hardening/QA scenarios; leave the main docker-compose.yml for default dev usage. - litellm: - build: - context: . - dockerfile: docker/Dockerfile.non_root - target: runtime - args: - PROXY_EXTRAS_SOURCE: "local" - depends_on: - - squid - user: "101:101" - group_add: - - "2345" - read_only: true - cap_drop: - - ALL - security_opt: - - no-new-privileges:true - tmpfs: - - /app/cache:rw,noexec,nosuid,nodev,size=128m,uid=101,gid=101,mode=1777 - - /app/migrations:rw,noexec,nosuid,nodev,size=64m,uid=101,gid=101,mode=1777 - volumes: - - ./proxy_server_config.yaml:/app/config.yaml:ro - environment: - LITELLM_NON_ROOT: "true" - PRISMA_BINARY_CACHE_DIR: "/app/cache/prisma-python/binaries" - XDG_CACHE_HOME: "/app/cache" - LITELLM_MIGRATION_DIR: "/app/migrations" - HTTP_PROXY: "http://squid:3128" - HTTPS_PROXY: "http://squid:3128" - NO_PROXY: "localhost,127.0.0.1,db" - command: - - "--port" - - "4000" - - "--config" - - "/app/config.yaml" - squid: - image: sameersbn/squid:3.5.27-2 - restart: unless-stopped - ports: - - "3128:3128" - tmpfs: - - /var/spool/squid:rw,noexec,nosuid,nodev,size=64m - - /var/log/squid:rw,noexec,nosuid,nodev,size=16m diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 80e1f289aad..00000000000 --- a/docker-compose.yml +++ /dev/null @@ -1,73 +0,0 @@ -services: - litellm: - build: - context: . - args: - target: runtime - image: docker.litellm.ai/berriai/litellm:main-stable - ######################################### - ## Uncomment these lines to start proxy with a config.yaml file ## - # volumes: - # - ./config.yaml:/app/config.yaml - # command: - # - "--config=/app/config.yaml" - ############################################## - ports: - - "4000:4000" # Map the container port to the host, change the host port if necessary - environment: - DATABASE_URL: "postgresql://llmproxy:dbpassword9090@db:5432/litellm" - # Optional: route read-only queries (find_*, count, group_by, query_raw/_first) - # to a separate reader endpoint, e.g. an Aurora reader. Leave unset for - # single-DB deployments. With IAM_TOKEN_DB_AUTH enabled, the reader URL - # is auto-refreshed alongside the writer. - # DATABASE_URL_READ_REPLICA: "postgresql://llmproxy:dbpassword9090@db-reader:5432/litellm" - STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI - env_file: - - .env # Load local .env file - depends_on: - - db # Indicates that this service depends on the 'db' service, ensuring 'db' starts first - healthcheck: # Defines the health check configuration for the container - test: - - CMD-SHELL - - python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveliness')" # Command to execute for health check - interval: 30s # Perform health check every 30 seconds - timeout: 10s # Health check command times out after 10 seconds - retries: 3 # Retry up to 3 times if health check fails - start_period: 40s # Wait 40 seconds after container start before beginning health checks - - db: - image: postgres:16 - restart: always - container_name: litellm_db - environment: - POSTGRES_DB: litellm - POSTGRES_USER: llmproxy - POSTGRES_PASSWORD: dbpassword9090 - ports: - - "5432:5432" - volumes: - - postgres_data:/var/lib/postgresql/data # Persists Postgres data across container restarts - healthcheck: - test: ["CMD-SHELL", "pg_isready -d litellm -U llmproxy"] - interval: 1s - timeout: 5s - retries: 10 - - prometheus: - image: prom/prometheus - volumes: - - prometheus_data:/prometheus - - ./prometheus.yml:/etc/prometheus/prometheus.yml - ports: - - "9090:9090" - command: - - "--config.file=/etc/prometheus/prometheus.yml" - - "--storage.tsdb.path=/prometheus" - - "--storage.tsdb.retention.time=15d" - restart: always - -volumes: - prometheus_data: - driver: local - postgres_data: - name: litellm_postgres_data # Named volume for Postgres data persistence diff --git a/docker-compose/README.md b/docker-compose/README.md new file mode 100644 index 00000000000..cb78311e4a0 --- /dev/null +++ b/docker-compose/README.md @@ -0,0 +1,245 @@ +# LiteLLM Hardened Production Stack Documentation + +This document outlines the production-grade deployment guidelines for the LiteLLM Proxy infrastructure. The system is engineered as a single, unified architecture offering two configurable egress topologies: **Direct Egress** and **Proxy-Filtered Egress (Squid)**. The underlying design emphasizes container isolation, OWASP-compliant secrets management, and strict network segmentation through a Zero-Trust internal model. + +--- + +## Network Architecture and Traffic Flows + +In both execution modes, the architecture enforces strict isolation for sensitive infrastructure components. The PostgreSQL database is placed within a dedicated internal network lacking a default gateway (`backend-internal`), permanently preventing outbound connection attempts to the external internet. + +### Topology Option 1: Direct Egress + +This configuration optimizes for minimal latency. LiteLLM establishes outbound connections directly to external AI provider endpoints via the host's bridge interface. + +``` + [ Host Firewall / UFW ] + │ + (127.0.0.1:4000) + │ +┌───────────────────────────┼───────────────────────────────────────────┐ +│ DOCKER HOST ▼ │ +│ │ +│ [ public-egress Network ] │ +│ ┌─────────────────────────────────┐ │ +│ │ litellm Proxy │ │ +│ │ (Reads Secrets via tmpfs) │ │ +│ └────────────────┬────────────────┘ │ +│ │ │ +│ [ backend-internal Network (NO GATEWAY) ] │ +│ ┌────────────────┴────────────────┐ │ +│ │ litellm_db │ │ +│ │ (PostgreSQL - Isolated) │ │ +│ └─────────────────────────────────┘ │ +└───────────────────┬───────────────────────────────────────────────────┘ + │ Direct egress via host bridge + ▼ + [ Internet / Cloud APIs ] + ├── api.openai.com + ├── api.anthropic.com + └── bedrock.us-east-1.amazonaws.com +``` + +### Topology Option 2: Proxy-Filtered Egress (Squid) + +This configuration introduces outbound traffic control to mitigate data exfiltration risks. All egress requests initiated by LiteLLM are forcibly directed through an internal Squid proxy acting as an egress gateway. + +``` + [ Host Firewall / UFW ] + │ + (127.0.0.1:4000) + │ +┌───────────────────────────┼───────────────────────────────────────────┐ +│ DOCKER HOST ▼ │ +│ │ +│ [ public-egress Network ] │ +│ ┌─────────────────────────────────┐ │ +│ │ litellm Proxy │ │ +│ │ HTTP_PROXY=http://squid:3128 ├────────┐ │ +│ └────────────────┬────────────────┘ │ HTTP/HTTPS via Proxy │ +│ │ ▼ │ +│ │ ┌────────────────────┐ │ +│ │ │ litellm_squid │ │ +│ │ │ (Egress Proxy) │ │ +│ │ └──────────┬─────────┘ │ +│ [ backend-internal Network ] │ │ +│ ┌────────────────┴────────────────┐ │ │ +│ │ litellm_db │ │ Evaluates Egress Rules │ +│ │ (PostgreSQL - Isolated) │ │ │ +│ └─────────────────────────────────┘ │ │ +└─────────────────────────────────────────────┼─────────────────────────┘ + │ + ▼ + [ Internet / Cloud APIs ] +``` + +--- + +## System Prerequisites + +- **Operating System:** Linux (Ubuntu LTS or Debian recommended). +- **Access Privileges:** Administrative rights (`root` or `sudo` access) required for systemd unit installation, filesystem permission management, and firewall rule modification. +- **Software Dependencies:** + - `git` (for repository cloning and sparse-checkout) + - `docker` (Engine 20.10+) + - `docker compose` (V2 plugin) + - `openssl` + - `envsubst` (provided by `gettext-base`) + +--- + +## Deployment Procedures + +### Step 1: Prepare Destination Directory in `/opt/` + +Create the production installation path under `/opt/litellm` and navigate into it: + +```bash +sudo mkdir -p /opt/litellm +cd /opt/litellm +``` + +### Step 2: Download Deployment Files via Git Sparse-Checkout + +Fetch strictly the target deployment directory without cloning the entire repository history: + +```bash +sudo git clone --no-checkout --depth 1 git@github.com:BerriAI/litellm.git src +cd src +sudo git sparse-checkout init --cone +sudo git sparse-checkout set docker-compose +sudo git checkout main +cd docker-compose +``` + +### Step 3: Validate Provisioned Deployment Files + +Verify that the sparse checkout successfully populated the working directory with the required artifacts: + +- `deploy.sh` +- `compose.yaml.template` + +### Step 4: Set Execution Permissions + +Grant execution rights to the deployment script: + +```bash +sudo chmod +x deploy.sh +``` + +### Step 5: Execute Deployment Script + +Select the desired topology option using the runtime flags. Append `--install-systemd` if system boot persistence is required: + +#### Option 1: Direct Egress Deployment + +```bash +# Standard container execution +sudo ./deploy.sh --force-open-firewall + +# With Systemd boot persistence +sudo ./deploy.sh --force-open-firewall --install-systemd +``` + +#### Option 2: Proxy-Filtered Egress Deployment + +```bash +# Standard container execution +sudo ./deploy.sh --force-open-firewall --enable-egress-proxy + +# With Systemd boot persistence +sudo ./deploy.sh --force-open-firewall --enable-egress-proxy --install-systemd +``` + +### Automated Execution Lifecycle + +Upon invocation, the deployment script executes the following operations: + +1. Validates host dependencies and configures system firewall rules (UFW). +2. Generates cryptographically secure secrets in `./secrets/` with restricted `600` permissions mapped to UID/GID `101:101`. +3. Renders the final `compose.yaml` configuration using `compose.yaml.template`. +4. Registers and enables the `litellm.service` systemd unit **only if** `--install-systemd` is passed. +5. Initializes the container stack via `docker compose up -d` (or `systemctl restart litellm.service` when systemd is enabled). + +--- + +## Post-Installation Operations + +### Environment Tuning & Operational Flags + +To maintain a clean Admin UI and prevent false-positive operational warnings in single-worker deployments, the stack sets specific environment flags: + +- **`LITELLM_DISABLE_NO_REDIS_WARNING=true`**: Suppresses the persistent Redis configuration warning banner in the Admin UI. Since this deployment runs as a single, isolated container instance, an in-memory/PostgreSQL setup is sufficient for rate limiting and state persistence without introducing additional Redis infrastructure overhead. + - _Official Reference:_ [LiteLLM Redis Requirements & Single-Worker Configuration](https://docs.litellm.ai/docs/proxy/redis_requirements) + +- **Secrets Ingestion Wrapper (`entrypoint`)**: To comply with Docker Secrets security standards while working around LiteLLM's lack of native `*_FILE` variable parsing, secrets are injected into environment variables at container startup using an inline shell wrapper: + ```bash + export DATABASE_URL=$(cat /run/secrets/db_url) + export LITELLM_MASTER_KEY=$(cat /run/secrets/master_key) + export LITELLM_SALT_KEY=$(cat /run/secrets/salt_key) + exec docker/prod_entrypoint.sh + ``` + +```` + +### Service Endpoints + +- **Service Base URL:** `http://127.0.0.1:4000` +- **Health Verification Endpoint:** `http://127.0.0.1:4000/health` + +### Secrets and Credential Retrieval + +Generated credentials are stored in the `./secrets/` directory on the host and are readable exclusively by the `root` user. + +#### Retrieve Master API Key (Admin / UI Access) + +```bash +sudo cat ./secrets/master_key.txt +```` + +#### Retrieve Application Salt Key + +```bash +sudo cat ./secrets/salt_key.txt +``` + +#### Retrieve Database Connection String + +```bash +sudo cat ./secrets/db_url.txt +``` + +#### Retrieve PostgreSQL Database Password + +```bash +sudo cat ./secrets/db_password.txt +``` + +--- + +## Teardown and Cleanup Procedures + +To completely remove the container stack, associated systemd units, networks, persistent data volumes, and installation files: + +### Step 1: Stop and Remove Systemd Service (If Installed) + +```bash +sudo systemctl stop litellm.service 2>/dev/null || true +sudo systemctl disable litellm.service 2>/dev/null || true +sudo rm -f /etc/systemd/system/litellm.service +sudo systemctl daemon-reload +``` + +### Step 2: Purge Docker Resources and Volumes + +Run from `/opt/litellm/src/docker-compose`: + +```bash +sudo docker compose down -v --remove-orphans +``` + +### Step 3: Remove Production Installation Directory + +```bash +sudo rm -rf /opt/litellm +``` diff --git a/docker-compose/compose.yaml.template b/docker-compose/compose.yaml.template new file mode 100644 index 00000000000..3c1e096d298 --- /dev/null +++ b/docker-compose/compose.yaml.template @@ -0,0 +1,93 @@ +services: + litellm: + image: docker.litellm.ai/berriai/litellm:main-stable + container_name: litellm + restart: unless-stopped + user: "101:101" + read_only: true + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + tmpfs: + - /app/cache:rw,noexec,nosuid,nodev,size=256m,uid=101,gid=101,mode=1777 + - /app/migrations:rw,noexec,nosuid,nodev,size=64m,uid=101,gid=101,mode=1777 + ports: + - "127.0.0.1:4000:4000" + environment: + DATABASE_URL_FILE: /run/secrets/db_url + LITELLM_NON_ROOT: "true" + LITELLM_DISABLE_NO_REDIS_WARNING: "true" + PRISMA_BINARY_CACHE_DIR: "/app/cache/prisma-python/binaries" + XDG_CACHE_HOME: "/app/cache" + LITELLM_MIGRATION_DIR: "/app/migrations" + ${PROXY_ENV_MAPPING} + entrypoint: > + /bin/sh -c " + export DATABASE_URL=$$(cat /run/secrets/db_url) && + export LITELLM_MASTER_KEY=$$(cat /run/secrets/master_key) && + export LITELLM_SALT_KEY=$$(cat /run/secrets/salt_key) && + exec docker/prod_entrypoint.sh + " + env_file: + - .env + secrets: + - db_url + - master_key + - salt_key + networks: + - public-egress + - backend-internal + depends_on: + db: + condition: service_healthy + ${LITELLM_SQUID_DEPENDENCY} + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:4000/health/liveliness')"] + interval: 5s + timeout: 5s + retries: 5 + start_period: 60s + + db: + image: postgres:16-alpine + restart: unless-stopped + container_name: litellm_db + environment: + POSTGRES_DB: litellm + POSTGRES_USER: llmproxy + POSTGRES_PASSWORD_FILE: /run/secrets/db_password + secrets: + - db_password + volumes: + - postgres_data:/var/lib/postgresql/data + networks: + - backend-internal + healthcheck: + test: ["CMD-SHELL", "pg_isready -d litellm -U llmproxy"] + interval: 5s + timeout: 5s + retries: 5 + + ${SQUID_SERVICE_BLOCK} + +secrets: + db_password: + file: ./secrets/db_password.txt + master_key: + file: ./secrets/master_key.txt + salt_key: + file: ./secrets/salt_key.txt + db_url: + file: ./secrets/db_url.txt + +volumes: + postgres_data: + name: litellm_postgres_data + +networks: + public-egress: + driver: bridge + backend-internal: + driver: bridge + internal: true \ No newline at end of file diff --git a/docker-compose/deploy.sh b/docker-compose/deploy.sh new file mode 100644 index 00000000000..be2906cbf65 --- /dev/null +++ b/docker-compose/deploy.sh @@ -0,0 +1,318 @@ +#!/usr/bin/env bash +set -euo pipefail + +ulimit -c 0 +export HISTIGNORE="*openssl*:*sk-*:*PASSWORD*:*MASTER_KEY*" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SECRETS_DIR="${SCRIPT_DIR}/secrets" +FORCE_FIREWALL=false +ENABLE_PROXY=false +INSTALL_SYSTEMD=false +TARGET_PORT="4000" + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +BOLD='\033[1m' +NC='\033[0m' + +log_info() { echo -e "${BLUE}[INFO]${NC} $(date +'%Y-%m-%dT%H:%M:%S%z') - $1"; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $(date +'%Y-%m-%dT%H:%M:%S%z') - $1"; } +log_error() { echo -e "${RED}[ERROR]${NC} $(date +'%Y-%m-%dT%H:%M:%S%z') - $1" >&2; } + +cleanup_on_error() { + local exit_code=$? + if [ $exit_code -ne 0 ]; then + log_error "Deployment failed unexpectedly with exit code $exit_code." + fi +} +trap cleanup_on_error EXIT + +cd "${SCRIPT_DIR}" + +for arg in "$@"; do + case $arg in + --force-open-firewall) + FORCE_FIREWALL=true + shift + ;; + --enable-egress-proxy) + ENABLE_PROXY=true + shift + ;; + --install-systemd) + INSTALL_SYSTEMD=true + shift + ;; + *) + ;; + esac +done + +log_info "Starting Production LiteLLM Infrastructure Deployment..." + +if [ "$(id -u)" -ne 0 ]; then + log_error "Root privileges are required to setup secrets, systemd, and UFW rules." + exit 1 +fi + +REQUIRED_TOOLS=("docker" "openssl" "envsubst") +for tool in "${REQUIRED_TOOLS[@]}"; do + if ! command -v "$tool" >/dev/null 2>&1; then + log_error "Required dependency '$tool' is missing." + exit 1 + fi +done + +if ! docker compose version >/dev/null 2>&1; then + log_error "Docker Compose V2 is missing." + exit 1 +fi + +# Firewall Check +if command -v ufw >/dev/null 2>&1; then + UFW_STATUS=$(ufw status | grep -i "status: active" || true) + if [ -n "${UFW_STATUS}" ]; then + PORT_ALLOWED=$(ufw status | grep -E "${TARGET_PORT}(/tcp)?.*ALLOW" || true) + if [ -z "${PORT_ALLOWED}" ]; then + if [ "${FORCE_FIREWALL}" = true ]; then + log_info "Force flag enabled. Granting rule for port ${TARGET_PORT}/tcp..." + ufw allow "${TARGET_PORT}/tcp" comment 'LiteLLM Proxy' + else + log_error "Port ${TARGET_PORT} is blocked by UFW firewall." + log_error "Re-run with '--force-open-firewall' or run: ufw allow ${TARGET_PORT}/tcp" + exit 1 + fi + fi + fi +fi + +# Provision Secrets +mkdir -p -m 700 "${SECRETS_DIR}" + +generate_secret_file() { + local file_path="$1" + local secret_content="$2" + if [ ! -f "${file_path}" ]; then + printf "%s" "${secret_content}" > "${file_path}" + chmod 600 "${file_path}" + chown 101:101 "${file_path}" 2>/dev/null || true + fi +} + +DB_PASS_RAW="$(openssl rand -hex 20)" +generate_secret_file "${SECRETS_DIR}/db_password.txt" "${DB_PASS_RAW}" +DB_PASS_STORED="$(cat "${SECRETS_DIR}/db_password.txt")" +DB_CONNECTION_URL="postgresql://llmproxy:${DB_PASS_STORED}@db:5432/litellm" + +generate_secret_file "${SECRETS_DIR}/db_url.txt" "${DB_CONNECTION_URL}" +generate_secret_file "${SECRETS_DIR}/master_key.txt" "sk-$(openssl rand -hex 16)" +generate_secret_file "${SECRETS_DIR}/salt_key.txt" "sk-$(openssl rand -hex 16)" +unset DB_PASS_RAW DB_PASS_STORED DB_CONNECTION_URL + +# Environment File +if [ ! -f ".env" ]; then + cat > .env < /etc/squid/squid.conf + acl SSL_ports port 443 + acl Safe_ports port 80 + acl Safe_ports port 443 + acl CONNECT method CONNECT + http_access deny !Safe_ports + http_access deny CONNECT !SSL_ports + http_access allow all + http_port 3128 + SQUIDCONF + chown -R proxy:proxy /var/spool/squid /var/log/squid /etc/squid + exec /sbin/entrypoint.sh + networks: + public-egress: + aliases: + - squid + tmpfs: + - /var/spool/squid:rw,noexec,nosuid,nodev,size=64m + - /var/log/squid:rw,noexec,nosuid,nodev,size=16m + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/3128"] + interval: 5s + timeout: 3s + retries: 5 + start_period: 5s +EOF +) + + export PROXY_ENV_MAPPING + export LITELLM_SQUID_DEPENDENCY + export SQUID_SERVICE_BLOCK +else + log_info "Mode: Option 1 (Direct Egress)" + export PROXY_ENV_MAPPING="# egress proxy disabled" + export LITELLM_SQUID_DEPENDENCY="# no squid dependency" + export SQUID_SERVICE_BLOCK="# no squid service" +fi + +# Inject variables into template +envsubst '${PROXY_ENV_MAPPING} ${LITELLM_SQUID_DEPENDENCY} ${SQUID_SERVICE_BLOCK}' < compose.yaml.template > compose.yaml + +# Systemd Integration +SERVICE_FILE="/etc/systemd/system/litellm.service" +DOCKER_BIN="$(command -v docker)" + +if [ "${INSTALL_SYSTEMD}" = true ]; then + log_info "Installing and enabling Systemd service unit..." + cat > "${SERVICE_FILE}" <