scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-28 01:32:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 43

LiteLLM Hardened Production Stack Documentation

Browse source
This commit is contained in:
myu1d157h0u54nd 2026-09-02 18:05:55 -03:00
parent 7a1afa1c40
commit dc111da09b
5 changed files with 656 additions and 119 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
docker-compose.hardened.yml
View file

@ -1,46 +0,0 @@
services:
# Hardened stack: for testing the proxy under non-root, read-only, proxy-enforced constraints.
# Keep this file focused on hardening/QA scenarios; leave the main docker-compose.yml for default dev usage.
litellm:
build:
context: .
dockerfile: docker/Dockerfile.non_root
target: runtime
args:
PROXY_EXTRAS_SOURCE: "local"
depends_on:
- squid
user: "101:101"
group_add:
- "2345"
read_only: true
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
tmpfs:
- /app/cache:rw,noexec,nosuid,nodev,size=128m,uid=101,gid=101,mode=1777
- /app/migrations:rw,noexec,nosuid,nodev,size=64m,uid=101,gid=101,mode=1777
volumes:
- ./proxy_server_config.yaml:/app/config.yaml:ro
environment:
LITELLM_NON_ROOT: "true"
PRISMA_BINARY_CACHE_DIR: "/app/cache/prisma-python/binaries"
XDG_CACHE_HOME: "/app/cache"
LITELLM_MIGRATION_DIR: "/app/migrations"
HTTP_PROXY: "http://squid:3128"
HTTPS_PROXY: "http://squid:3128"
NO_PROXY: "localhost,127.0.0.1,db"
command:
- "--port"
- "4000"
- "--config"
- "/app/config.yaml"
squid:
image: sameersbn/squid:3.5.27-2
restart: unless-stopped
ports:
- "3128:3128"
tmpfs:
- /var/spool/squid:rw,noexec,nosuid,nodev,size=64m
- /var/log/squid:rw,noexec,nosuid,nodev,size=16m

73
docker-compose.yml
View file

@ -1,73 +0,0 @@
services:
litellm:
build:
context: .
args:
target: runtime
image: docker.litellm.ai/berriai/litellm:main-stable
#########################################
## Uncomment these lines to start proxy with a config.yaml file ##
# volumes:
# - ./config.yaml:/app/config.yaml
# command:
# - "--config=/app/config.yaml"
##############################################
ports:
- "4000:4000" # Map the container port to the host, change the host port if necessary
environment:
DATABASE_URL: "postgresql://llmproxy:dbpassword9090@db:5432/litellm"
# Optional: route read-only queries (find_*, count, group_by, query_raw/_first)
# to a separate reader endpoint, e.g. an Aurora reader. Leave unset for
# single-DB deployments. With IAM_TOKEN_DB_AUTH enabled, the reader URL
# is auto-refreshed alongside the writer.
# DATABASE_URL_READ_REPLICA: "postgresql://llmproxy:dbpassword9090@db-reader:5432/litellm"
STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI
env_file:
- .env # Load local .env file
depends_on:
- db # Indicates that this service depends on the 'db' service, ensuring 'db' starts first
healthcheck: # Defines the health check configuration for the container
test:
- CMD-SHELL
- python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveliness')" # Command to execute for health check
interval: 30s # Perform health check every 30 seconds
timeout: 10s # Health check command times out after 10 seconds
retries: 3 # Retry up to 3 times if health check fails
start_period: 40s # Wait 40 seconds after container start before beginning health checks
db:
image: postgres:16
restart: always
container_name: litellm_db
environment:
POSTGRES_DB: litellm
POSTGRES_USER: llmproxy
POSTGRES_PASSWORD: dbpassword9090
ports:
- "5432:5432"
volumes:
- postgres_data:/var/lib/postgresql/data # Persists Postgres data across container restarts
healthcheck:
test: ["CMD-SHELL", "pg_isready -d litellm -U llmproxy"]
interval: 1s
timeout: 5s
retries: 10
prometheus:
image: prom/prometheus
volumes:
- prometheus_data:/prometheus
- ./prometheus.yml:/etc/prometheus/prometheus.yml
ports:
- "9090:9090"
command:
- "--config.file=/etc/prometheus/prometheus.yml"
- "--storage.tsdb.path=/prometheus"
- "--storage.tsdb.retention.time=15d"
restart: always
volumes:
prometheus_data:
driver: local
postgres_data:
name: litellm_postgres_data # Named volume for Postgres data persistence

245
docker-compose/README.md Normal file
View file

@ -0,0 +1,245 @@
# LiteLLM Hardened Production Stack Documentation
This document outlines the production-grade deployment guidelines for the LiteLLM Proxy infrastructure. The system is engineered as a single, unified architecture offering two configurable egress topologies: **Direct Egress** and **Proxy-Filtered Egress (Squid)**. The underlying design emphasizes container isolation, OWASP-compliant secrets management, and strict network segmentation through a Zero-Trust internal model.
---
## Network Architecture and Traffic Flows
In both execution modes, the architecture enforces strict isolation for sensitive infrastructure components. The PostgreSQL database is placed within a dedicated internal network lacking a default gateway (`backend-internal`), permanently preventing outbound connection attempts to the external internet.
### Topology Option 1: Direct Egress
This configuration optimizes for minimal latency. LiteLLM establishes outbound connections directly to external AI provider endpoints via the host's bridge interface.
```
[ Host Firewall / UFW ]
│
(127.0.0.1:4000)
│
┌───────────────────────────┼───────────────────────────────────────────┐
│ DOCKER HOST ▼ │
│ │
│ [ public-egress Network ] │
│ ┌─────────────────────────────────┐ │
│ │ litellm Proxy │ │
│ │ (Reads Secrets via tmpfs) │ │
│ └────────────────┬────────────────┘ │
│ │ │
│ [ backend-internal Network (NO GATEWAY) ] │
│ ┌────────────────┴────────────────┐ │
│ │ litellm_db │ │
│ │ (PostgreSQL - Isolated) │ │
│ └─────────────────────────────────┘ │
└───────────────────┬───────────────────────────────────────────────────┘
│ Direct egress via host bridge
▼
[ Internet / Cloud APIs ]
├── api.openai.com
├── api.anthropic.com
└── bedrock.us-east-1.amazonaws.com
```
### Topology Option 2: Proxy-Filtered Egress (Squid)
This configuration introduces outbound traffic control to mitigate data exfiltration risks. All egress requests initiated by LiteLLM are forcibly directed through an internal Squid proxy acting as an egress gateway.
```
[ Host Firewall / UFW ]
│
(127.0.0.1:4000)
│
┌───────────────────────────┼───────────────────────────────────────────┐
│ DOCKER HOST ▼ │
│ │
│ [ public-egress Network ] │
│ ┌─────────────────────────────────┐ │
│ │ litellm Proxy │ │
│ │ HTTP_PROXY=http://squid:3128 ├────────┐ │
│ └────────────────┬────────────────┘ │ HTTP/HTTPS via Proxy │
│ │ ▼ │
│ │ ┌────────────────────┐ │
│ │ │ litellm_squid │ │
│ │ │ (Egress Proxy) │ │
│ │ └──────────┬─────────┘ │
│ [ backend-internal Network ] │ │
│ ┌────────────────┴────────────────┐ │ │
│ │ litellm_db │ │ Evaluates Egress Rules │
│ │ (PostgreSQL - Isolated) │ │ │
│ └─────────────────────────────────┘ │ │
└─────────────────────────────────────────────┼─────────────────────────┘
│
▼
[ Internet / Cloud APIs ]
```
---
## System Prerequisites
- **Operating System:** Linux (Ubuntu LTS or Debian recommended).
- **Access Privileges:** Administrative rights (`root` or `sudo` access) required for systemd unit installation, filesystem permission management, and firewall rule modification.
- **Software Dependencies:**
- `git` (for repository cloning and sparse-checkout)
- `docker` (Engine 20.10+)
- `docker compose` (V2 plugin)
- `openssl`
- `envsubst` (provided by `gettext-base`)
---
## Deployment Procedures
### Step 1: Prepare Destination Directory in `/opt/`
Create the production installation path under `/opt/litellm` and navigate into it:
```bash
sudo mkdir -p /opt/litellm
cd /opt/litellm
```
### Step 2: Download Deployment Files via Git Sparse-Checkout
Fetch strictly the target deployment directory without cloning the entire repository history:
```bash
sudo git clone --no-checkout --depth 1 git@github.com:BerriAI/litellm.git src
cd src
sudo git sparse-checkout init --cone
sudo git sparse-checkout set docker-compose
sudo git checkout main
cd docker-compose
```
### Step 3: Validate Provisioned Deployment Files
Verify that the sparse checkout successfully populated the working directory with the required artifacts:
- `deploy.sh`
- `compose.yaml.template`
### Step 4: Set Execution Permissions
Grant execution rights to the deployment script:
```bash
sudo chmod +x deploy.sh
```
### Step 5: Execute Deployment Script
Select the desired topology option using the runtime flags. Append `--install-systemd` if system boot persistence is required:
#### Option 1: Direct Egress Deployment
```bash
# Standard container execution
sudo ./deploy.sh --force-open-firewall
# With Systemd boot persistence
sudo ./deploy.sh --force-open-firewall --install-systemd
```
#### Option 2: Proxy-Filtered Egress Deployment
```bash
# Standard container execution
sudo ./deploy.sh --force-open-firewall --enable-egress-proxy
# With Systemd boot persistence
sudo ./deploy.sh --force-open-firewall --enable-egress-proxy --install-systemd
```
### Automated Execution Lifecycle
Upon invocation, the deployment script executes the following operations:
1. Validates host dependencies and configures system firewall rules (UFW).
2. Generates cryptographically secure secrets in `./secrets/` with restricted `600` permissions mapped to UID/GID `101:101`.
3. Renders the final `compose.yaml` configuration using `compose.yaml.template`.
4. Registers and enables the `litellm.service` systemd unit **only if** `--install-systemd` is passed.
5. Initializes the container stack via `docker compose up -d` (or `systemctl restart litellm.service` when systemd is enabled).
---
## Post-Installation Operations
### Environment Tuning & Operational Flags
To maintain a clean Admin UI and prevent false-positive operational warnings in single-worker deployments, the stack sets specific environment flags:
- **`LITELLM_DISABLE_NO_REDIS_WARNING=true`**: Suppresses the persistent Redis configuration warning banner in the Admin UI. Since this deployment runs as a single, isolated container instance, an in-memory/PostgreSQL setup is sufficient for rate limiting and state persistence without introducing additional Redis infrastructure overhead.
- _Official Reference:_ [LiteLLM Redis Requirements & Single-Worker Configuration](https://docs.litellm.ai/docs/proxy/redis_requirements)
- **Secrets Ingestion Wrapper (`entrypoint`)**: To comply with Docker Secrets security standards while working around LiteLLM's lack of native `*_FILE` variable parsing, secrets are injected into environment variables at container startup using an inline shell wrapper:
```bash
export DATABASE_URL=$(cat /run/secrets/db_url)
export LITELLM_MASTER_KEY=$(cat /run/secrets/master_key)
export LITELLM_SALT_KEY=$(cat /run/secrets/salt_key)
exec docker/prod_entrypoint.sh
```
````
### Service Endpoints
- **Service Base URL:** `http://127.0.0.1:4000`
- **Health Verification Endpoint:** `http://127.0.0.1:4000/health`
### Secrets and Credential Retrieval
Generated credentials are stored in the `./secrets/` directory on the host and are readable exclusively by the `root` user.
#### Retrieve Master API Key (Admin / UI Access)
```bash
sudo cat ./secrets/master_key.txt
````
#### Retrieve Application Salt Key
```bash
sudo cat ./secrets/salt_key.txt
```
#### Retrieve Database Connection String
```bash
sudo cat ./secrets/db_url.txt
```
#### Retrieve PostgreSQL Database Password
```bash
sudo cat ./secrets/db_password.txt
```
---
## Teardown and Cleanup Procedures
To completely remove the container stack, associated systemd units, networks, persistent data volumes, and installation files:
### Step 1: Stop and Remove Systemd Service (If Installed)
```bash
sudo systemctl stop litellm.service 2>/dev/null || true
sudo systemctl disable litellm.service 2>/dev/null || true
sudo rm -f /etc/systemd/system/litellm.service
sudo systemctl daemon-reload
```
### Step 2: Purge Docker Resources and Volumes
Run from `/opt/litellm/src/docker-compose`:
```bash
sudo docker compose down -v --remove-orphans
```
### Step 3: Remove Production Installation Directory
```bash
sudo rm -rf /opt/litellm
```

93
docker-compose/compose.yaml.template Normal file
View file

@ -0,0 +1,93 @@
services:
litellm:
image: docker.litellm.ai/berriai/litellm:main-stable
container_name: litellm
restart: unless-stopped
user: "101:101"
read_only: true
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
tmpfs:
- /app/cache:rw,noexec,nosuid,nodev,size=256m,uid=101,gid=101,mode=1777
- /app/migrations:rw,noexec,nosuid,nodev,size=64m,uid=101,gid=101,mode=1777
ports:
- "127.0.0.1:4000:4000"
environment:
DATABASE_URL_FILE: /run/secrets/db_url
LITELLM_NON_ROOT: "true"
LITELLM_DISABLE_NO_REDIS_WARNING: "true"
PRISMA_BINARY_CACHE_DIR: "/app/cache/prisma-python/binaries"
XDG_CACHE_HOME: "/app/cache"
LITELLM_MIGRATION_DIR: "/app/migrations"
${PROXY_ENV_MAPPING}
entrypoint: >
/bin/sh -c "
export DATABASE_URL=$$(cat /run/secrets/db_url) &&
export LITELLM_MASTER_KEY=$$(cat /run/secrets/master_key) &&
export LITELLM_SALT_KEY=$$(cat /run/secrets/salt_key) &&
exec docker/prod_entrypoint.sh
"
env_file:
- .env
secrets:
- db_url
- master_key
- salt_key
networks:
- public-egress
- backend-internal
depends_on:
db:
condition: service_healthy
${LITELLM_SQUID_DEPENDENCY}
healthcheck:
test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:4000/health/liveliness')"]
interval: 5s
timeout: 5s
retries: 5
start_period: 60s
db:
image: postgres:16-alpine
restart: unless-stopped
container_name: litellm_db
environment:
POSTGRES_DB: litellm
POSTGRES_USER: llmproxy
POSTGRES_PASSWORD_FILE: /run/secrets/db_password
secrets:
- db_password
volumes:
- postgres_data:/var/lib/postgresql/data
networks:
- backend-internal
healthcheck:
test: ["CMD-SHELL", "pg_isready -d litellm -U llmproxy"]
interval: 5s
timeout: 5s
retries: 5
${SQUID_SERVICE_BLOCK}
secrets:
db_password:
file: ./secrets/db_password.txt
master_key:
file: ./secrets/master_key.txt
salt_key:
file: ./secrets/salt_key.txt
db_url:
file: ./secrets/db_url.txt
volumes:
postgres_data:
name: litellm_postgres_data
networks:
public-egress:
driver: bridge
backend-internal:
driver: bridge
internal: true

318
docker-compose/deploy.sh Normal file
View file

@ -0,0 +1,318 @@
#!/usr/bin/env bash
set -euo pipefail
ulimit -c 0
export HISTIGNORE="*openssl*:*sk-*:*PASSWORD*:*MASTER_KEY*"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SECRETS_DIR="${SCRIPT_DIR}/secrets"
FORCE_FIREWALL=false
ENABLE_PROXY=false
INSTALL_SYSTEMD=false
TARGET_PORT="4000"
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
BOLD='\033[1m'
NC='\033[0m'
log_info() { echo -e "${BLUE}[INFO]${NC} $(date +'%Y-%m-%dT%H:%M:%S%z') - $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $(date +'%Y-%m-%dT%H:%M:%S%z') - $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $(date +'%Y-%m-%dT%H:%M:%S%z') - $1" >&2; }
cleanup_on_error() {
local exit_code=$?
if [ $exit_code -ne 0 ]; then
log_error "Deployment failed unexpectedly with exit code $exit_code."
fi
}
trap cleanup_on_error EXIT
cd "${SCRIPT_DIR}"
for arg in "$@"; do
case $arg in
--force-open-firewall)
FORCE_FIREWALL=true
shift
;;
--enable-egress-proxy)
ENABLE_PROXY=true
shift
;;
--install-systemd)
INSTALL_SYSTEMD=true
shift
;;
*)
;;
esac
done
log_info "Starting Production LiteLLM Infrastructure Deployment..."
if [ "$(id -u)" -ne 0 ]; then
log_error "Root privileges are required to setup secrets, systemd, and UFW rules."
exit 1
fi
REQUIRED_TOOLS=("docker" "openssl" "envsubst")
for tool in "${REQUIRED_TOOLS[@]}"; do
if ! command -v "$tool" >/dev/null 2>&1; then
log_error "Required dependency '$tool' is missing."
exit 1
fi
done
if ! docker compose version >/dev/null 2>&1; then
log_error "Docker Compose V2 is missing."
exit 1
fi
# Firewall Check
if command -v ufw >/dev/null 2>&1; then
UFW_STATUS=$(ufw status | grep -i "status: active" || true)
if [ -n "${UFW_STATUS}" ]; then
PORT_ALLOWED=$(ufw status | grep -E "${TARGET_PORT}(/tcp)?.*ALLOW" || true)
if [ -z "${PORT_ALLOWED}" ]; then
if [ "${FORCE_FIREWALL}" = true ]; then
log_info "Force flag enabled. Granting rule for port ${TARGET_PORT}/tcp..."
ufw allow "${TARGET_PORT}/tcp" comment 'LiteLLM Proxy'
else
log_error "Port ${TARGET_PORT} is blocked by UFW firewall."
log_error "Re-run with '--force-open-firewall' or run: ufw allow ${TARGET_PORT}/tcp"
exit 1
fi
fi
fi
fi
# Provision Secrets
mkdir -p -m 700 "${SECRETS_DIR}"
generate_secret_file() {
local file_path="$1"
local secret_content="$2"
if [ ! -f "${file_path}" ]; then
printf "%s" "${secret_content}" > "${file_path}"
chmod 600 "${file_path}"
chown 101:101 "${file_path}" 2>/dev/null || true
fi
}
DB_PASS_RAW="$(openssl rand -hex 20)"
generate_secret_file "${SECRETS_DIR}/db_password.txt" "${DB_PASS_RAW}"
DB_PASS_STORED="$(cat "${SECRETS_DIR}/db_password.txt")"
DB_CONNECTION_URL="postgresql://llmproxy:${DB_PASS_STORED}@db:5432/litellm"
generate_secret_file "${SECRETS_DIR}/db_url.txt" "${DB_CONNECTION_URL}"
generate_secret_file "${SECRETS_DIR}/master_key.txt" "sk-$(openssl rand -hex 16)"
generate_secret_file "${SECRETS_DIR}/salt_key.txt" "sk-$(openssl rand -hex 16)"
unset DB_PASS_RAW DB_PASS_STORED DB_CONNECTION_URL
# Environment File
if [ ! -f ".env" ]; then
cat > .env <<EOF
STORE_MODEL_IN_DB=True
LITELLM_LOG=ERROR
EOF
chmod 644 .env
fi
# Template Rendering
if [ ! -f "compose.yaml.template" ]; then
log_error "Template 'compose.yaml.template' missing."
exit 1
fi
if [ "${ENABLE_PROXY}" = true ]; then
log_info "Mode: Proxy-Filtered Egress via Squid"
PROXY_ENV_MAPPING=$(cat << 'EOF'
HTTP_PROXY: "http://squid:3128"
HTTPS_PROXY: "http://squid:3128"
http_proxy: "http://squid:3128"
https_proxy: "http://squid:3128"
NO_PROXY: "localhost,127.0.0.1,db,litellm,litellm_db,squid,litellm_squid"
no_proxy: "localhost,127.0.0.1,db,litellm,litellm_db,squid,litellm_squid"
EOF
)
LITELLM_SQUID_DEPENDENCY=$(cat << 'EOF'
squid:
condition: service_healthy
EOF
)
SQUID_SERVICE_BLOCK=$(cat << 'EOF'
squid:
image: sameersbn/squid:3.5.27-2
restart: unless-stopped
container_name: litellm_squid
user: root
entrypoint:
- /bin/sh
- -c
- |
mkdir -p /etc/squid /var/spool/squid /var/log/squid
cat << 'SQUIDCONF' > /etc/squid/squid.conf
acl SSL_ports port 443
acl Safe_ports port 80
acl Safe_ports port 443
acl CONNECT method CONNECT
http_access deny !Safe_ports
http_access deny CONNECT !SSL_ports
http_access allow all
http_port 3128
SQUIDCONF
chown -R proxy:proxy /var/spool/squid /var/log/squid /etc/squid
exec /sbin/entrypoint.sh
networks:
public-egress:
aliases:
- squid
tmpfs:
- /var/spool/squid:rw,noexec,nosuid,nodev,size=64m
- /var/log/squid:rw,noexec,nosuid,nodev,size=16m
healthcheck:
test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/3128"]
interval: 5s
timeout: 3s
retries: 5
start_period: 5s
EOF
)
export PROXY_ENV_MAPPING
export LITELLM_SQUID_DEPENDENCY
export SQUID_SERVICE_BLOCK
else
log_info "Mode: Option 1 (Direct Egress)"
export PROXY_ENV_MAPPING="# egress proxy disabled"
export LITELLM_SQUID_DEPENDENCY="# no squid dependency"
export SQUID_SERVICE_BLOCK="# no squid service"
fi
# Inject variables into template
envsubst '${PROXY_ENV_MAPPING} ${LITELLM_SQUID_DEPENDENCY} ${SQUID_SERVICE_BLOCK}' < compose.yaml.template > compose.yaml
# Systemd Integration
SERVICE_FILE="/etc/systemd/system/litellm.service"
DOCKER_BIN="$(command -v docker)"
if [ "${INSTALL_SYSTEMD}" = true ]; then
log_info "Installing and enabling Systemd service unit..."
cat > "${SERVICE_FILE}" <<EOF
[Unit]
Description=LiteLLM Proxy Stack
Requires=docker.service
After=docker.service network-online.target
Wants=network-online.target
[Service]
Type=oneshot
RemainAfterExit=yes
WorkingDirectory=${SCRIPT_DIR}
ExecStart=${DOCKER_BIN} compose up -d
ExecStop=${DOCKER_BIN} compose down
TimeoutStartSec=0
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable litellm.service --quiet
else
log_info "Skipping Systemd service installation (flag --install-systemd not set)."
fi
log_info "Validating compose configuration..."
docker compose config --quiet
log_info "Downloading container images in advance..."
docker compose pull
log_info "Starting containers in detached mode..."
if [ "${INSTALL_SYSTEMD}" = true ]; then
systemctl restart litellm.service
else
docker compose up -d
fi
# Calculate expected service count based on target setup
EXPECTED_SERVICES=$(docker compose config --services | wc -l)
# Maximum allowed wait time for all services to report a healthy status
MAX_WAIT_SECONDS=180
WAIT_INTERVAL=5
ELAPSED=0
BAR_WIDTH=30
# Helper function to render the progress bar
draw_progress_bar() {
local elapsed="$1"
local max="$2"
local healthy_count="$3"
local expected="$4"
local percentage=$(( elapsed * 100 / max ))
if [ "$percentage" -gt 100 ]; then percentage=100; fi
local filled=$(( percentage * BAR_WIDTH / 100 ))
local empty=$(( BAR_WIDTH - filled ))
# Build visual bar strings
local bar_filled=$(printf "%${filled}s" "" | tr ' ' '#')
local bar_empty=$(printf "%${empty}s" "" | tr ' ' '-')
# Print bar over the current line (\r)
printf "\r[INFO] Waiting: [%s%s] %3d%% (%ds/%ds) - Healthy services: %d/%d" \
"${bar_filled}" "${bar_empty}" "${percentage}" "${elapsed}" "${max}" "${healthy_count}" "${expected}"
}
log_info "Waiting for all ${EXPECTED_SERVICES} services to pass healthchecks..."
until [ "$(docker compose ps --format json | grep -c '"Health":"healthy"' || true)" -eq "${EXPECTED_SERVICES}" ]; do
CURRENT_HEALTHY=$(docker compose ps --format json | grep -c '"Health":"healthy"' || true)
# Render progress bar
draw_progress_bar "${ELAPSED}" "${MAX_WAIT_SECONDS}" "${CURRENT_HEALTHY}" "${EXPECTED_SERVICES}"
# Abort if the total elapsed time exceeds the global timeout threshold
if [ "$ELAPSED" -ge "$MAX_WAIT_SECONDS" ]; then
echo ""
log_error "Timeout: Services failed to become healthy within ${MAX_WAIT_SECONDS} seconds."
docker compose ps
exit 1
fi
# Fail fast if Docker explicitly marks any container as unhealthy after its start_period
if docker compose ps --format json | grep -q '"Health":"unhealthy"'; then
echo ""
log_error "One or more containers failed their healthcheck."
docker compose ps
exit 1
fi
sleep "$WAIT_INTERVAL"
ELAPSED=$((ELAPSED + WAIT_INTERVAL))
done
# Print completed state (100%) and move to a new line
draw_progress_bar "${ELAPSED}" "${MAX_WAIT_SECONDS}" "${EXPECTED_SERVICES}" "${EXPECTED_SERVICES}"
echo ""
log_info "All services are healthy!"
# Retrieve Master Key for display
LITELLM_MASTER_KEY="$(cat "${SECRETS_DIR}/master_key.txt")"
echo -e "\n${GREEN}=====================================================${NC}"
echo -e "${BOLD}${CYAN} LITELLM SERVICE ACCESS DETAILS ${NC}"
echo -e "${GREEN}=====================================================${NC}"
echo -e " ${BOLD}URL:${NC} http://127.0.0.1:${TARGET_PORT}/ui/login/"
echo -e " ${BOLD}User:${NC} admin (or Bearer Token / UI Auth)"
echo -e " ${BOLD}Password:${NC} ${YELLOW}${LITELLM_MASTER_KEY}${NC}"
echo -e "${GREEN}=====================================================${NC}\n"