scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-14 09:03:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 214

Merge remote-tracking branch 'origin/main' into litellm_bedrock_kb_image_search

Browse source 
# Conflicts:
#	litellm/llms/bedrock/vector_stores/transformation.py
#	litellm/vector_stores/vector_store_registry.py
#	tests/unit/llms/bedrock/vector_stores/test_bedrock_vector_store_transformation.py
This commit is contained in:
mateo-berri 2026-11-11 14:25:50 -08:00
commit 37fe6920ae
882 changed files with 62274 additions and 13368 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
.circleci/config.yml
View file

@ -531,31 +531,36 @@ jobs:
IMAGE_TAG=${CIRCLE_SHA1:-ci}
kind load docker-image litellm-ci:${IMAGE_TAG} --name litellm-test
# Run helm lint
- run:
name: Run helm lint
command: |
helm lint ./helm/litellm-helm
helm lint ./helm/litellm -f ./helm/litellm/ci/test-values.yaml
- run:
name: Start PostgreSQL for the chart
command: |
kubectl apply -f ./helm/litellm/ci/postgres.yaml
kubectl rollout status deployment/litellm-ci-postgres --timeout=3m
# Run helm tests
- run:
name: Run helm tests
command: |
IMAGE_TAG=${CIRCLE_SHA1:-ci}
helm install litellm ./helm/litellm-helm -f ./helm/litellm-helm/ci/test-values.yaml \
helm install litellm ./helm/litellm -f ./helm/litellm/ci/test-values.yaml \
--set image.repository=litellm-ci \
--set image.tag=${IMAGE_TAG} \
--set image.pullPolicy=Never
if ! kubectl rollout status deployment/litellm --timeout=180s; then
kubectl describe pods -l app.kubernetes.io/name=litellm
kubectl logs -l app.kubernetes.io/name=litellm --tail=200
exit 1
fi
--set image.pullPolicy=Never \
--wait --wait-for-jobs --timeout 10m || {
kubectl get pods -o wide
kubectl describe pods -l app.kubernetes.io/instance=litellm
kubectl logs -l app.kubernetes.io/component=proxy --tail=200 || true
kubectl logs -l app.kubernetes.io/component=migrations --tail=200 || true
exit 1
}
echo "Printing pod logs..."
kubectl logs $(kubectl get pods -l app.kubernetes.io/name=litellm -o jsonpath="{.items[0].metadata.name}")
echo "Printing proxy pod logs..."
kubectl logs -l app.kubernetes.io/component=proxy --tail=200
# Run the helm tests
helm test litellm --logs
# Cleanup
@ -1099,7 +1104,7 @@ jobs:
docker build \
--label org.opencontainers.image.revision="$(git rev-parse HEAD)" \
-t litellm-docker-database:ci \
-f docker/Dockerfile.database .
-f Dockerfile .
fi
python3 .circleci/scripts/run_migration_tests.py record-image

1
.dockerignore
View file

@ -7,7 +7,6 @@ tests
.devcontainer
*.tgz
log.txt
docker/Dockerfile.*
# Claude Flow generated files (must be excluded from Docker build)
.claude/

9
.github/ci-coverage-allowlist.yml vendored
View file

@ -57,15 +57,6 @@ test_paths:
- tests/integration/test_oci_proxy_integration.py
dockerfiles:
- reason: >-
The dashboard container is a static Next.js export served by nginx, and the dashboard build
and lint workflows already exercise that output, so building the image adds no signal about it
paths:
- ui/Dockerfile
- reason: >-
An example image under cookbook/ that is documentation rather than a shipped artifact
paths:
- cookbook/litellm-ollama-docker-image/Dockerfile
- reason: >-
The Rust gateway image compiles the whole workspace in release mode, which is too slow for
a per-pull-request job while the gateway binary is still being assembled; the Rust lint,

2
.github/scripts/verify_linux_native_wheel.py vendored
View file

@ -214,7 +214,7 @@ def main(
native_module: Final = load_native_module(native_path)
native_module_loads: Final = native_module is not None
panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test")
native_size_limit: Final = 48_000_000
native_size_limit: Final = 55_000_000
native_size_within_limit: Final = native_member.file_size <= native_size_limit
validations: Final = (
(f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG),

33
.github/workflows/compat-matrix-image.yml vendored
View file

@ -26,17 +26,24 @@ jobs:
with:
persist-credentials: false
- name: Build the Render cron image
run: docker build -f tests/e2e/claude_code/cron_vm/Dockerfile -t compat-matrix:${{ github.sha }} tests/e2e
- name: Resolve and install the Claude Code CLI as the cron user
- name: Install the pinned uv the cron image ships
env:
UV_VERSION: 0.10.9
UV_SHA256: 20d79708222611fa540b5c9ed84f352bcd3937740e51aacc0f8b15b271c57594
run: |
docker run --rm compat-matrix:${{ github.sha }} bash -c '
set -euo pipefail
whoami
gh --version
uv --version
version="$(uv run --no-project --python 3.12 python /opt/litellm/tests/e2e/cli_harness/npm_version_resolver.py @anthropic-ai/claude-code)"
/opt/litellm/tests/e2e/claude_code/cron_vm/install_claude_code.sh "${version}" /tmp/claude-cli
/tmp/claude-cli/claude --version
'
set -euo pipefail
curl -fsSLo "$RUNNER_TEMP/uv.tar.gz" "https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/uv-x86_64-unknown-linux-gnu.tar.gz"
echo "${UV_SHA256} $RUNNER_TEMP/uv.tar.gz" | sha256sum -c -
mkdir -p "$RUNNER_TEMP/bin"
tar -xzf "$RUNNER_TEMP/uv.tar.gz" -C "$RUNNER_TEMP/bin" --strip-components=1 uv-x86_64-unknown-linux-gnu/uv
echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH"
- name: Resolve and install the Claude Code CLI as an unprivileged user
run: |
set -euo pipefail
whoami
gh --version
uv --version
version="$(uv run --no-project --python 3.12 python tests/e2e/cli_harness/npm_version_resolver.py @anthropic-ai/claude-code)"
tests/e2e/claude_code/cron_vm/install_claude_code.sh "${version}" "$RUNNER_TEMP/claude-cli"
"$RUNNER_TEMP/claude-cli/claude" --version

338
.github/workflows/image-scan.yml vendored
View file

@ -8,31 +8,25 @@ on:
- "litellm_**"
paths:
- Dockerfile
- docker/Dockerfile.non_root
- docker/Dockerfile.database
- migrations/Dockerfile
- .dockerignore
- docker-entrypoint.sh
- migrations/run.py
- gateway/Dockerfile
- gateway/launch.py
- gateway/main.py
- gateway/launch.py
- gateway/routes/allowlist.py
- litellm_cli/**
- backend/Dockerfile
- backend/main.py
- backend/routes/allowlist.py
- litellm-rust/**
- litellm/proxy/lens/**
- docker/component_entrypoint.sh
- docker/entrypoint.sh
- litellm/proxy/prisma_migration.py
- litellm/proxy/admin_mcp.py
- litellm/proxy/proxy_server.py
- backend/routes/allowlist.py
- pyproject.toml
- litellm-proxy-extras/**
- tests/integration/database/image/**
- uv.lock
- ui/litellm-dashboard/package-lock.json
- ui/Dockerfile
- ui/nginx.conf
- .github/workflows/image-scan.yml
- .grype.yaml
@ -46,9 +40,13 @@ concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
IMAGE: litellm-image-scan:${{ github.sha }}
ADMIN_MCP_SCHEMA_IMAGE: litellm-admin-mcp-schema:${{ github.sha }}
jobs:
image-scan:
name: image-scan
build:
name: build
runs-on: ubuntu-latest
if: >-
github.event_name != 'pull_request' ||
@ -61,7 +59,129 @@ jobs:
with:
persist-credentials: false
- name: Build the image
run: docker build --build-arg MATURIN_PEP517_ARGS="--profile ci" --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci -t "$IMAGE" .
- name: Save the image for the verify matrix
run: docker save "$IMAGE" | zstd -T0 -3 -o "$RUNNER_TEMP/litellm-image.tar.zst"
- uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: litellm-image
path: ${{ runner.temp }}/litellm-image.tar.zst
retention-days: 1
compression-level: 0
- name: Build the Admin MCP schema image
run: docker build --build-arg MATURIN_PEP517_ARGS="--profile ci" --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci --target builder -t "$ADMIN_MCP_SCHEMA_IMAGE" .
- name: Save the Admin MCP schema image for the verify matrix
run: docker save "$ADMIN_MCP_SCHEMA_IMAGE" | zstd -T0 -3 -o "$RUNNER_TEMP/litellm-admin-mcp-schema.tar.zst"
- uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: litellm-admin-mcp-schema
path: ${{ runner.temp }}/litellm-admin-mcp-schema.tar.zst
retention-days: 1
compression-level: 0
verify:
name: verify (${{ matrix.check }})
needs: build
runs-on: ubuntu-latest
timeout-minutes: 45
permissions:
contents: read
strategy:
fail-fast: false
matrix:
check: [scan, migrations, gateway, backend, ui, collector]
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: litellm-image
path: ${{ runner.temp }}
- name: Load the image
run: zstd -d --stdout "$RUNNER_TEMP/litellm-image.tar.zst" | docker load
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
if: matrix.check == 'migrations' || matrix.check == 'backend'
with:
name: litellm-admin-mcp-schema
path: ${{ runner.temp }}
- name: Load the Admin MCP schema image
if: matrix.check == 'migrations' || matrix.check == 'backend'
run: zstd -d --stdout "$RUNNER_TEMP/litellm-admin-mcp-schema.tar.zst" | docker load
- name: Set up Python
if: matrix.check != 'scan'
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Install pytest
if: matrix.check != 'scan'
run: python -m pip install "pytest==9.0.3"
# The prisma bake must migrate a fresh DB with no egress as an arbitrary
# non-root uid (OpenShift restricted-v2 / air-gapped / readOnlyFilesystem).
# `docker run` as the default uid with network hides a broken bake because
# the migration entrypoint exits 0 even when it applied nothing; asserting
# the schema was created is what catches it.
- name: Verify offline migration as a non-root uid
if: matrix.check == 'migrations'
env:
LITELLM_IMAGE: ${{ env.IMAGE }}
LITELLM_ADMIN_MCP_SCHEMA_IMAGE: ${{ env.ADMIN_MCP_SCHEMA_IMAGE }}
run: |
python -m pytest --noconftest -v \
tests/integration/database/image/test_offline_image_migration.py \
tests/integration/database/image/test_image_bedrock_realtime_extra.py \
tests/integration/database/image/test_image_admin_mcp.py
- name: Verify the gateway component serves offline as a non-root uid
if: matrix.check == 'gateway'
env:
LITELLM_IMAGE: ${{ env.IMAGE }}
LITELLM_COMPONENT: gateway
LITELLM_IMAGE_COMPONENT: gateway
run: |
python -m pytest --noconftest -v \
tests/integration/database/image/test_component_image_serves_offline.py \
tests/integration/database/image/test_image_admin_mcp.py
- name: Verify the backend component serves offline as a non-root uid
if: matrix.check == 'backend'
env:
LITELLM_IMAGE: ${{ env.IMAGE }}
LITELLM_COMPONENT: backend
LITELLM_IMAGE_COMPONENT: backend
LITELLM_ADMIN_MCP_SCHEMA_IMAGE: ${{ env.ADMIN_MCP_SCHEMA_IMAGE }}
run: |
python -m pytest --noconftest -v \
tests/integration/database/image/test_component_image_serves_offline.py \
tests/integration/database/image/test_image_admin_mcp.py
- name: Verify the ui component serves as an arbitrary uid with a read-only root fs
if: matrix.check == 'ui'
env:
LITELLM_IMAGE: ${{ env.IMAGE }}
run: python -m pytest --noconftest -v tests/integration/database/image/test_ui_image_serves_offline.py
- name: Verify the collector creates its socket as the image user
if: matrix.check == 'collector'
env:
LITELLM_IMAGE: ${{ env.IMAGE }}
run: python -m pytest --noconftest -v tests/integration/database/image/test_collector_image_serves_socket.py
- name: Download Grype v0.114.0
if: matrix.check == 'scan'
run: |
curl -fsSL --retry 3 -o "$RUNNER_TEMP/grype.tar.gz" \
https://github.com/anchore/grype/releases/download/v0.114.0/grype_0.114.0_linux_amd64.tar.gz
@ -69,33 +189,6 @@ jobs:
tar xzf "$RUNNER_TEMP/grype.tar.gz" -C "$RUNNER_TEMP" grype
chmod +x "$RUNNER_TEMP/grype"
# Dockerfile.non_root is the rootless variant we ship. The other
# Dockerfiles share the same wolfi base and apk set, so OS-layer coverage
# is the same; matrix-scan if those variants ever diverge.
- name: Build runtime image
run: docker build -f docker/Dockerfile.non_root -t litellm-image-scan:${{ github.sha }} .
- name: Tag the cached builder for Admin MCP schema setup
run: docker build --target builder -f docker/Dockerfile.non_root -t litellm-admin-mcp-schema:${{ github.sha }} .
# The prisma bake must migrate a fresh DB with no egress as an arbitrary
# non-root uid (OpenShift restricted-v2 / air-gapped / readOnlyRootFilesystem).
# `docker run` as the default uid with network hides a broken bake because
# the migration entrypoint exits 0 even when it applied nothing; asserting
# the schema was created is what catches it.
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Verify offline migration as a non-root uid
env:
LITELLM_IMAGE: litellm-image-scan:${{ github.sha }}
LITELLM_ADMIN_MCP_SCHEMA_IMAGE: litellm-admin-mcp-schema:${{ github.sha }}
run: |
python -m pip install "pytest==9.0.3"
python -m pytest --noconftest tests/integration/database/image/test_offline_image_migration.py tests/integration/database/image/test_image_bedrock_realtime_extra.py tests/integration/database/image/test_image_admin_mcp.py -v
# Scans the whole shipped artifact: OS/apk plus every language package
# baked into the image, including ones no lockfile declares (e.g. prisma's
# vendored node engine) that osv-scan cannot see. osv-scan stays the fast
@ -103,175 +196,12 @@ jobs:
# free OSS, run as a pinned, checksum-verified binary; no GitHub Action
# dependency and no vendor SaaS callout.
- name: Scan image for fixable HIGH/CRITICAL CVEs
if: matrix.check == 'scan'
env:
GRYPE_MATCH_PYTHON_USING_CPES: "true"
run: |
"$RUNNER_TEMP/grype" litellm-image-scan:${{ github.sha }} \
"$RUNNER_TEMP/grype" "$IMAGE" \
--config .grype.yaml \
--only-fixed \
--fail-on high \
--output table
runtime-image:
name: runtime-image (${{ matrix.dockerfile }})
runs-on: ubuntu-latest
if: >-
github.event_name != 'pull_request' ||
github.event.pull_request.head.repo.full_name == github.repository
timeout-minutes: 45
permissions:
contents: read
strategy:
fail-fast: false
matrix:
dockerfile: [Dockerfile, docker/Dockerfile.database]
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Build runtime image
run: docker build --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci -f "${{ matrix.dockerfile }}" -t litellm-runtime-scan:${{ github.sha }} .
- name: Tag the cached builder for Admin MCP schema setup
run: docker build --target builder -f "${{ matrix.dockerfile }}" -t litellm-admin-mcp-schema:${{ github.sha }} .
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Verify offline migration as a non-root uid
env:
LITELLM_IMAGE: litellm-runtime-scan:${{ github.sha }}
LITELLM_ADMIN_MCP_SCHEMA_IMAGE: litellm-admin-mcp-schema:${{ github.sha }}
run: |
python -m pip install "pytest==9.0.3"
python -m pytest --noconftest tests/integration/database/image/test_offline_image_migration.py tests/integration/database/image/test_image_bedrock_realtime_extra.py tests/integration/database/image/test_image_admin_mcp.py -v
migrations-image:
name: migrations-image
runs-on: ubuntu-latest
if: >-
github.event_name != 'pull_request' ||
github.event.pull_request.head.repo.full_name == github.repository
timeout-minutes: 30
permissions:
contents: read
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Build migrations image
run: docker build -f migrations/Dockerfile -t litellm-migrations-scan:${{ github.sha }} .
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Verify offline migration as a non-root uid
env:
LITELLM_IMAGE: litellm-migrations-scan:${{ github.sha }}
LITELLM_MIGRATION_INTERPRETER: python3
LITELLM_MIGRATION_SCRIPT: /app/run.py
run: |
python -m pip install "pytest==9.0.3"
python -m pytest --noconftest tests/integration/database/image/test_offline_image_migration.py -v
gateway-image:
name: gateway-image
runs-on: ubuntu-latest
if: >-
github.event_name != 'pull_request' ||
github.event.pull_request.head.repo.full_name == github.repository
timeout-minutes: 30
permissions:
contents: read
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Build gateway image
run: docker build -f gateway/Dockerfile -t litellm-gateway-scan:${{ github.sha }} .
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Verify the gateway serves offline as a non-root uid
env:
LITELLM_IMAGE: litellm-gateway-scan:${{ github.sha }}
LITELLM_COMPONENT_PORT: "4000"
LITELLM_IMAGE_COMPONENT: gateway
run: |
python -m pip install "pytest==9.0.3"
python -m pytest --noconftest tests/integration/database/image/test_component_image_serves_offline.py tests/integration/database/image/test_image_bedrock_realtime_extra.py tests/integration/database/image/test_image_admin_mcp.py -v
ui-image:
name: ui-image
runs-on: ubuntu-latest
if: >-
github.event_name != 'pull_request' ||
github.event.pull_request.head.repo.full_name == github.repository
timeout-minutes: 30
permissions:
contents: read
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Build UI image
run: docker build -f ui/Dockerfile -t litellm-ui-scan:${{ github.sha }} .
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Verify the UI serves offline as an arbitrary uid with a read-only root fs
env:
LITELLM_IMAGE: litellm-ui-scan:${{ github.sha }}
run: |
python -m pip install "pytest==9.0.3"
python -m pytest --noconftest tests/integration/database/image/test_ui_image_serves_offline.py -v
backend-image:
name: backend-image
runs-on: ubuntu-latest
if: >-
github.event_name != 'pull_request' ||
github.event.pull_request.head.repo.full_name == github.repository
timeout-minutes: 30
permissions:
contents: read
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Build backend image
run: docker build -f backend/Dockerfile -t litellm-backend-scan:${{ github.sha }} .
- name: Tag the cached builder for Admin MCP schema setup
run: docker build --target builder -f backend/Dockerfile -t litellm-admin-mcp-schema:${{ github.sha }} .
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Verify the backend serves offline as a non-root uid
env:
LITELLM_IMAGE: litellm-backend-scan:${{ github.sha }}
LITELLM_ADMIN_MCP_SCHEMA_IMAGE: litellm-admin-mcp-schema:${{ github.sha }}
LITELLM_COMPONENT_PORT: "4001"
LITELLM_IMAGE_COMPONENT: backend
run: |
python -m pip install "pytest==9.0.3"
python -m pytest --noconftest tests/integration/database/image/test_component_image_serves_offline.py tests/integration/database/image/test_image_admin_mcp.py -v

64
.github/workflows/test-merge-smoke.yml vendored
View file

@ -200,8 +200,36 @@ jobs:
- name: Install the test client dependencies
run: .github/scripts/uv_sync_with_retries.sh --frozen --no-install-workspace --extra proxy --group dev --python 3.12
- name: Build the image from the published package
run: docker build -t litellm-build-from-pip:smoke -f docker/build_from_pip/Dockerfile.build_from_pip .
- name: Cache Prisma binaries
timeout-minutes: 3
uses: ./.github/actions/cache-prisma-binaries
- name: Install the published proxy into its own venv
run: |
uv --no-config venv --python 3.13 /tmp/litellm-pip
uv --no-config pip install --python /tmp/litellm-pip/bin/python \
"litellm[proxy,proxy-runtime]==1.83.0" \
"google-cloud-aiplatform==1.133.0" \
"google-genai==1.37.0" \
"anthropic[vertex]==0.84.0" \
"grpcio==1.78.0" \
"prometheus-client==0.20.0" \
"langfuse==2.59.7" \
"opentelemetry-api==1.28.0" \
"opentelemetry-sdk==1.28.0" \
"opentelemetry-exporter-otlp==1.28.0" \
"ddtrace==4.11.0" \
"sentry-sdk==2.21.0" \
"mangum==0.17.0" \
"azure-ai-contentsafety==1.0.0" \
"azure-storage-file-datalake==12.20.0" \
"pypdf==6.7.5" \
"llm-sandbox==0.3.31" \
"detect-secrets==1.5.0" \
"prisma==0.11.0" \
"openai==2.24.0"
export PATH="/tmp/litellm-pip/bin:$PATH"
/tmp/litellm-pip/bin/prisma generate --schema=schema.prisma
- name: Start the fake OpenAI endpoint
run: |
@ -219,23 +247,21 @@ jobs:
echo "::add-mask::${key}"
echo "LITELLM_MASTER_KEY=${key}" >> "${GITHUB_ENV}"
- name: Start the proxy container
- name: Start the published proxy
env:
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/circle_test
LITELLM_LICENSE: bad-license
FAKE_OPENAI_API_BASE: http://localhost:8190
OTEL_EXPORTER: in_memory
USE_DDTRACE: "True"
DD_TRACE_OPENAI_ENABLED: "False"
GCS_FLUSH_INTERVAL: "1"
LITELLM_LOG: ERROR
run: |
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e LITELLM_MASTER_KEY \
-e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \
-e OTEL_EXPORTER=in_memory \
-e USE_DDTRACE=True \
-e GCS_FLUSH_INTERVAL=1 \
-e LITELLM_LOG=ERROR \
--add-host host.docker.internal:host-gateway \
--name litellm-build-from-pip \
-v "${GITHUB_WORKSPACE}/docker/build_from_pip/litellm_config.yaml:/app/config.yaml" \
litellm-build-from-pip:smoke \
--config /app/config.yaml \
--port 4000
cd /tmp/litellm-pip
nohup /tmp/litellm-pip/bin/ddtrace-run /tmp/litellm-pip/bin/litellm \
--config "${GITHUB_WORKSPACE}/tests/basic_proxy_startup_tests/build_from_pip_config.yaml" \
--port 4000 > /tmp/litellm-pip.log 2>&1 &
for _ in $(seq 1 150); do
curl -sf --max-time 5 -o /dev/null http://localhost:4000 && exit 0
sleep 2
@ -270,7 +296,7 @@ jobs:
- name: Show the proxy logs
if: always()
run: docker logs litellm-build-from-pip || true
run: cat /tmp/litellm-pip.log || true
smoke-passed:
name: smoke passed

42
.github/workflows/test-unit.yml vendored
View file

@ -397,6 +397,7 @@ jobs:
tests/unit/llms/voyage
tests/unit/llms/wandb
tests/unit/llms/watsonx
tests/unit/llms/webiq
tests/unit/llms/xai
tests/unit/llms/xinference
tests/unit/llms/you_com
@ -590,6 +591,7 @@ jobs:
tests/unit/proxy/credential_endpoints
tests/unit/proxy/discovery_endpoints
tests/unit/proxy/shutdown
tests/unit/proxy/rust_forward
tests/unit/proxy/public_endpoints
tests/unit/proxy/prompts
tests/unit/proxy/rag_endpoints
@ -632,23 +634,32 @@ jobs:
job-timeout-minutes: 60
dist: loadscope
- shard: proxy-infra
- shard: proxy-infra-db-spend
test-path: |-
tests/unit/proxy/db
--ignore=tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py
--ignore=tests/unit/proxy/db/test_update_daily_tag_spend.py
tests/unit/proxy/middleware
--ignore=tests/unit/proxy/middleware/test_request_size_limit_middleware.py
tests/unit/proxy/spend_tracking
--ignore=tests/unit/proxy/spend_tracking/test_search_api_logging.py
tests/unit/proxy/pass_through_endpoints
tests/unit/proxy/mcp_server/test_tool_registry.py
tests/unit/proxy/common_utils
--ignore=tests/unit/proxy/common_utils/test_cache_aware_routing.py
--ignore=tests/unit/proxy/common_utils/test_check_batch_cost.py
--ignore=tests/unit/proxy/common_utils/test_check_responses_cost.py
--ignore=tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py
--ignore=tests/unit/proxy/common_utils/test_realtime_cache.py
python-version: "3.12"
workers: 4
reruns: 0
timeout-minutes: 20
job-timeout-minutes: 60
dist: loadscope
- shard: proxy-infra-routing
test-path: |-
tests/unit/proxy/middleware
--ignore=tests/unit/proxy/middleware/test_request_size_limit_middleware.py
tests/unit/proxy/pass_through_endpoints
tests/unit/proxy/mcp_server/test_tool_registry.py
tests/unit/proxy/enterprise_billing
tests/unit/proxy/types_utils
tests/unit/proxy/logging_endpoints
@ -1343,17 +1354,16 @@ jobs:
- name: Run unit tests
run: |
for chart in helm/litellm-helm helm/litellm; do
declared="$(grep -h '^suite:' "$chart"/tests/*.yaml | wc -l | tr -d '[:space:]')"
output="$(mktemp)"
helm unittest -f 'tests/*.yaml' "$chart" | tee "$output"
executed="$(sed -n 's/^Test Suites:.*[[:space:]]\([0-9][0-9]*\) total$/\1/p' "$output")"
if [ "$declared" != "$executed" ]; then
echo "::error::$chart declares $declared test suites but helm-unittest ran $executed. Suites are being skipped silently, so their assertions never execute."
exit 1
fi
echo "$chart: all $declared declared test suites ran"
done
chart=helm/litellm
declared="$(grep -h '^suite:' "$chart"/tests/*.yaml | wc -l | tr -d '[:space:]')"
output="$(mktemp)"
helm unittest -f 'tests/*.yaml' "$chart" | tee "$output"
executed="$(sed -n 's/^Test Suites:.*[[:space:]]\([0-9][0-9]*\) total$/\1/p' "$output")"
if [ "$declared" != "$executed" ]; then
echo "::error::$chart declares $declared test suites but helm-unittest ran $executed. Suites are being skipped silently, so their assertions never execute."
exit 1
fi
echo "$chart: all $declared declared test suites ran"
openai-sdk-compat:
name: OpenAI ${{ matrix.openai-version }} compatibility
needs: rust-bridge

2
.gitignore vendored
View file

@ -61,7 +61,7 @@ ui/litellm-dashboard/next-env.d.ts
*.tsbuildinfo
ui/litellm-dashboard/package.json
ui/litellm-dashboard/package-lock.json
helm/litellm-helm/*.tgz
helm/litellm/charts/*.tgz
helm/*.tgz
litellm/proxy/vertex_key.json
**/.vim/

4
CONTRIBUTING.md
View file

@ -305,8 +305,8 @@ uv run litellm --config your_config.yaml
If you want to build the Docker image yourself:
```bash
# Build using the non-root Dockerfile
docker build -f docker/Dockerfile.non_root -t litellm_dev .
# Build the image (runs as a non-root user by default)
docker build -t litellm_dev .
# Generate a master key. Requests send it as the bearer token
export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 32)"

62
Dockerfile
View file

@ -102,6 +102,7 @@ COPY --from=ui-builder /ui/out/. litellm/proxy/_experimental/out/
RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Install project and workspace packages (fast - deps already cached)
ARG MATURIN_PEP517_ARGS=""
RUN uv build --package litellm-proxy --wheel --out-dir /wheels
RUN uv sync --frozen --no-install-package litellm-proxy --no-default-groups --no-editable \
--extra proxy \
@ -119,8 +120,12 @@ RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/
npm_config_cache=/root/.npm \
prisma generate --schema=./schema.prisma
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
RUN sed -i 's/\r$//' docker-entrypoint.sh && chmod +x docker-entrypoint.sh
RUN mkdir -p /var/lib/litellm/ui /var/lib/litellm/assets && \
cp -r litellm/proxy/_experimental/out/. /var/lib/litellm/ui/ && \
cp litellm/proxy/logo.png /var/lib/litellm/assets/logo.png && \
touch /var/lib/litellm/ui/.litellm_ui_ready
FROM $LITELLM_BUILD_IMAGE AS liteadmin-builder
COPY --from=uvbin /uv /usr/local/bin/uv
@ -145,24 +150,35 @@ USER root
# https://github.com/BerriAI/litellm/issues/33518
RUN echo "https://packages.wolfi.dev/os" >> /etc/apk/repositories
# node (without npm) is required by the prisma CLI at runtime
RUN apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libevent
RUN for i in 1 2 3; do \
apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libatomic libevent nginx && break; \
[ "$i" = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
WORKDIR /app
ENV PATH="/app/.venv/bin:${PATH}" \
PYTHONPATH="/app" \
PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
HOME=/app \
LITELLM_NON_ROOT=true \
PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
PRISMA_CLI_PATH=/opt/prisma/binaries/node_modules/.bin/prisma \
PRISMA_CLI_QUERY_ENGINE_TYPE=binary \
PRISMA_SKIP_POSTINSTALL_GENERATE=1 \
PRISMA_HIDE_UPDATE_MESSAGE=1 \
PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING=1 \
PRISMA_OFFLINE_MODE=true
# Copy only what runtime needs. The application is installed inside the venv;
# the rest of the builder's /app is source and build metadata that must not
# ship (manifest-scanning tools attribute everything in it to this image).
# entrypoint.sh invokes litellm/proxy/prisma_migration.py by source path.
COPY --from=builder /app/.venv /app/.venv
COPY --from=liteadmin-builder /opt/liteadmin /opt/liteadmin
COPY --from=builder /app/docker /app/docker
COPY --from=builder /app/docker-entrypoint.sh /app/docker-entrypoint.sh
COPY --from=builder /app/schema.prisma /app/schema.prisma
COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/prisma_migration.py
# enterprise/ is imported by source path at runtime (proxy_cli puts the
@ -170,21 +186,33 @@ COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/pr
# enterprise.enterprise_hooks from it)
COPY --from=builder /app/enterprise /app/enterprise
COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras
# Prisma CLI + engines are baked under /opt/prisma, a fixed path every
# runtime uid can read and that no cache volume mount shadows. The paths are
# pinned via PRISMA_BINARY_CACHE_DIR / PRISMA_CLI_PATH and recorded into the
# generated client at build time, so `prisma migrate deploy` on a fresh
# database needs no npm and no network access (#33650, #24554).
COPY --from=builder /app/gateway /app/gateway
COPY --from=builder /app/backend /app/backend
COPY --from=builder /app/migrations /app/migrations
COPY --from=builder /opt/prisma /opt/prisma
COPY --from=builder /var/lib/litellm /var/lib/litellm
COPY ui/nginx.conf /etc/nginx/nginx.conf
COPY ui/docker-entrypoint.d/40-litellm-runtime-config.sh /app/ui/docker-entrypoint.d/40-litellm-runtime-config.sh
RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
find /app/.venv -type d -path "*/tornado/test" -delete && \
RUN find /app/.venv -depth -type d -path "*/tornado/test" -exec rm -rf {} + && \
mkdir -p /app/.cache /var/run/litellm && \
chown -R 65532:0 /app /var/lib/litellm /var/run/litellm && \
chmod -R g=u,g+w /app/.cache /var/lib/litellm /var/run/litellm && \
PRISMA_PATH="$(python -c 'import os, prisma; print(os.path.dirname(prisma.__file__))')" && \
PROXY_EXTRAS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))')" && \
chmod -R g=u,g+w "$PRISMA_PATH" "$PROXY_EXTRAS_PATH" && \
chmod -R a+rX /opt/prisma && \
test -x /opt/prisma/binaries/node_modules/.bin/prisma && \
test -f /opt/prisma/binaries/node_modules/prisma/build/index.js && \
python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths"
ls /opt/prisma/binaries/node_modules/@prisma/engines/query-engine-* >/dev/null && \
python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths" && \
python -c "from litellm.rust_bridge import _native" && \
python -c "import gateway.launch, backend.main"
EXPOSE 4000/tcp
USER 65532:65532
ENTRYPOINT ["docker/prod_entrypoint.sh"]
CMD ["--port", "4000"]
RUN nginx -t
EXPOSE 4000/tcp 4001/tcp 3000/tcp
ENTRYPOINT ["/app/docker-entrypoint.sh"]

2
Makefile
View file

@ -341,7 +341,7 @@ test-integration: install-test-deps
$(UV_RUN) pytest tests/
test-unit-helm: install-helm-unittest
helm unittest -f 'tests/*.yaml' helm/litellm-helm
helm unittest -f 'tests/*.yaml' helm/litellm
# LLM Translation testing targets
test-llm-translation: install-test-deps

20
SECURITY.md
View file

@ -4,23 +4,19 @@
LiteLLM maintains a security announcements mailing list that is open to anyone. Subscribers receive advance notice, typically one to two days, before we release a fix for a particularly severe vulnerability or for any vulnerability exploitable by an unauthenticated attacker. This notice is provided on a best-effort basis
To subscribe, visit [https://berriai.github.io/security-announce-signup/](https://berriai.github.io/security-announce-signup/)
To subscribe, visit [https://security.litellm.ai/mailinglist/](https://security.litellm.ai/mailinglist/)
## Security Vulnerability Reporting Guidelines
> [!WARNING]
> Reports that do not include a video demonstrating the exploit will be closed without review. See [Reproduction Video Requirement](#reproduction-video-requirement) below.
We value the security community's role in protecting our systems and users. To report a security vulnerability:
We value the security community's role in protecting our systems and users.
To report a security vulnerability, please submit a report through our vulnerability disclosure form:
- File a private vulnerability report on GitHub: [Report a vulnerability](https://github.com/BerriAI/litellm/security/advisories/new)
- Include steps to reproduce the issue
- Include a video or screen recording demonstrating the full exploit against a live LiteLLM instance, from initial access through to impact. A terminal recording (for example asciinema) is fine for CLI-only exploits.
- Provide any relevant additional information
[LiteLLM VDP](https://security.litellm.ai/vdp/)
### Reproduction Video Requirement
These reports are submitted to HackerOne and are triaged there, though no account is needed.
A video demonstrating the exploit is required for every report. AI tools have made it easy to produce plausible-sounding vulnerability reports that do not reproduce in practice, and triaging them takes time away from real issues. Reports submitted without a working reproduction video will be closed without review. If you add a video to a closed report, we will reopen and triage it.
**Please do not submit vulnerabilities as draft GHSAs. As of 2026-11-11 new GHSA submissions will be closed.**
### Vulnerability Categories
@ -55,7 +51,9 @@ To qualify for a bounty, reports must include clear reproduction steps, a reprod
### Known Non-Issues
- Attacks that require a misconfiguration on setup (e.g not setting a `master_key` on the proxy configuration), are **explicitly not in scope** and are not considered vulnerable.
- **Misconfigurations:** attacks that require misconfiguration of the application (e.g. not setting a `master_key` on the proxy configuration).
- **Insecure configurations:** attacks that require deliberate configuration changes to disable security functionality. (e.g. IDOR on response IDs requiring configuration of `disable_responses_id_security` to `true`)
- **Social engineering:** attacks that rely on social engineering as their primary mechanism (e.g. convincing a user's agent to install a malicious package we did not publish).
## Security Measures

146
backend/Dockerfile
View file

@ -1,146 +0,0 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin
# ---------- Rust extension ----------
# Copies only the native build's inputs, so the fat-LTO compile stays cached across Python-only changes
FROM $LITELLM_BUILD_IMAGE AS rust-native
WORKDIR /app
USER root
COPY --from=uvbin /uv /uvx /usr/local/bin/
RUN for i in 1 2 3; do \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
ENV UV_PYTHON_DOWNLOADS=0
COPY pyproject.toml README.md rust-toolchain.toml ./
COPY .cargo ./.cargo
COPY litellm-rust ./litellm-rust
COPY model_prices_and_context_window.json ./
RUN --mount=type=cache,target=/root/.cache/uv \
mkdir -p litellm/rust_bridge /wheels && \
touch litellm/__init__.py litellm/rust_bridge/__init__.py && \
uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \
python -c "import maturin; maturin.build_wheel('/wheels')"
# ---------- Builder ----------
FROM $LITELLM_BUILD_IMAGE AS builder
WORKDIR /app
USER root
COPY --from=uvbin /uv /uvx /usr/local/bin/
# nodejs/npm so `prisma generate` uses Wolfi's Node via PRISMA_USE_GLOBAL_NODE
# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
RUN for i in 1 2 3; do \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
# UV_COMPILE_BYTECODE=1 precompiles .pyc at install time → faster cold start.
# UV_LINK_MODE=copy avoids hardlink warnings when uv installs from a
# BuildKit cache mount (different filesystem).
# UV_PYTHON_DOWNLOADS=0 force uv to use the apk-installed CPython instead of
# silently pulling a managed interpreter.
# PRISMA_USE_GLOBAL_NODE explicit (matches default) so an env override can't
# silently re-enable nodeenv's Node download.
ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
UV_LINK_MODE=copy \
UV_COMPILE_BYTECODE=1 \
UV_PYTHON_DOWNLOADS=0 \
PRISMA_USE_GLOBAL_NODE=true \
PATH="/app/.venv/bin:${PATH}"
# Stage 1 — install dependencies only.
RUN --mount=type=cache,target=/root/.cache/uv \
--mount=type=bind,source=pyproject.toml,target=pyproject.toml \
--mount=type=bind,source=uv.lock,target=uv.lock \
--mount=type=bind,source=enterprise/pyproject.toml,target=enterprise/pyproject.toml \
--mount=type=bind,source=litellm-proxy-extras/pyproject.toml,target=litellm-proxy-extras/pyproject.toml \
uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
--extra extra_proxy \
--extra semantic-router \
--extra saml \
--python python3.13
# Stage 2 — copy source and install the project + workspace members.
COPY . .
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-install-project --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
--extra extra_proxy \
--extra semantic-router \
--extra saml \
--python python3.13
RUN --mount=type=bind,from=rust-native,source=/wheels,target=/wheels \
uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm-*.whl
RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
npm_config_cache=/root/.npm \
prisma generate --schema=./schema.prisma
RUN sed -i 's/\r$//' docker/component_entrypoint.sh && chmod +x docker/component_entrypoint.sh
# ---------- Runtime ----------
FROM $LITELLM_RUNTIME_IMAGE AS runtime
ARG LITELLM_RELEASE_TAG=""
ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
USER root
RUN for i in 1 2 3; do \
apk add --no-cache bash openssl libssl3 tzdata python-3.13 libsndfile libatomic && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
# wolfi-base ships an unprivileged `nonroot` account (UID/GID 65532) with
# /home/nonroot. We run the backend as that user
WORKDIR /app
ENV HOME=/home/nonroot \
PATH="/app/.venv/bin:${PATH}" \
PYTHONPATH="/app" \
PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv
COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm
COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise
COPY --from=builder --chown=nonroot:nonroot /app/backend /app/backend
COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh
COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma
COPY --from=builder /opt/prisma /opt/prisma
RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
find /app/.venv -type d -path "*/tornado/test" -delete && \
chmod -R a+rX /opt/prisma && \
python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths"
USER nonroot
EXPOSE 4001/tcp
ENTRYPOINT ["/app/docker/component_entrypoint.sh", "uvicorn", "backend.main:app"]
CMD ["--host", "0.0.0.0", "--port", "4001"]

1
ci_cd/security_scans_readme.md
View file

@ -4,6 +4,5 @@
- Trivy scan on `./docs/` (HIGH/CRITICAL/MEDIUM)
- Trivy scan on `./ui/` (HIGH/CRITICAL/MEDIUM)
- Grype scan on `Dockerfile.database` (fails on CRITICAL)
- Grype scan on main `Dockerfile` (fails on CRITICAL)
- Grype CVSS ≥ 4.0 scan on main `Dockerfile` (fails any vulnerabilities with CVSS ≥ 4.0)

25
cookbook/litellm-ollama-docker-image/Dockerfile
View file

@ -1,25 +0,0 @@
FROM ollama/ollama as ollama
RUN echo "auto installing llama2"
# auto install ollama/llama2
RUN ollama serve & sleep 2 && ollama pull llama2
RUN echo "installing litellm"
RUN apt-get update
# Install Python
RUN apt-get install -y python3 python3-pip
# Set the working directory in the container
WORKDIR /app
# Copy the current directory contents into the container at /app
COPY . /app
# Install any needed packages specified in requirements.txt
RUN python3 -m pip install litellm
COPY start.sh /start.sh
ENTRYPOINT [ "/bin/bash", "/start.sh" ]

1
cookbook/litellm-ollama-docker-image/requirements.txt
View file

@ -1 +0,0 @@
litellm==1.83.14

2
cookbook/litellm-ollama-docker-image/start.sh
View file

@ -1,2 +0,0 @@
ollama serve &
litellm

35
cookbook/litellm-ollama-docker-image/test.py
View file

@ -1,35 +0,0 @@
import openai
api_base = "http://0.0.0.0:8000"
openai.api_base = api_base
openai.api_key = "temp-key"
print(openai.api_base)
print("LiteLLM: response from proxy with streaming")
response = openai.ChatCompletion.create(
model="ollama/llama2",
messages=[
{
"role": "user",
"content": "this is a test request, acknowledge that you got it",
}
],
stream=True,
)
for chunk in response:
print(f"LiteLLM: streaming response from proxy {chunk}")
response = openai.ChatCompletion.create(
model="ollama/llama2",
messages=[
{
"role": "user",
"content": "this is a test request, acknowledge that you got it",
}
],
)
print(f"LiteLLM: response from proxy {response}")

44
docker-compose.hardened.yml
View file

@ -1,44 +0,0 @@
services:
# Hardened stack: for testing the proxy under non-root, read-only, proxy-enforced constraints.
# Keep this file focused on hardening/QA scenarios; leave the main docker-compose.yml for default dev usage.
litellm:
build:
context: .
dockerfile: docker/Dockerfile.non_root
target: runtime
depends_on:
- squid
user: "101:101"
group_add:
- "2345"
read_only: true
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
tmpfs:
- /app/cache:rw,noexec,nosuid,nodev,size=128m,uid=101,gid=101,mode=1777
- /app/migrations:rw,noexec,nosuid,nodev,size=64m,uid=101,gid=101,mode=1777
volumes:
- ./proxy_server_config.yaml:/app/config.yaml:ro
environment:
LITELLM_NON_ROOT: "true"
PRISMA_BINARY_CACHE_DIR: "/app/cache/prisma-python/binaries"
XDG_CACHE_HOME: "/app/cache"
LITELLM_MIGRATION_DIR: "/app/migrations"
HTTP_PROXY: "http://squid:3128"
HTTPS_PROXY: "http://squid:3128"
NO_PROXY: "localhost,127.0.0.1,db"
command:
- "--port"
- "4000"
- "--config"
- "/app/config.yaml"
squid:
image: sameersbn/squid:3.5.27-2
restart: unless-stopped
ports:
- "3128:3128"
tmpfs:
- /var/spool/squid:rw,noexec,nosuid,nodev,size=64m
- /var/log/squid:rw,noexec,nosuid,nodev,size=16m

84
docker-compose.yml
View file

@ -1,52 +1,71 @@
# LiteLLM plus a Postgres database that stores models, virtual keys and spend
# logs. Used by https://docs.litellm.ai/docs/proxy/docker_quick_start
#
# curl -sSLO https://github.com/BerriAI/litellm/raw/main/docker-compose.yml
# printf 'LITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\n' "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" > .env
# docker compose up -d
#
# `docker compose up` runs the published image. From a checkout of the repo,
# `docker compose up --build` builds the Dockerfile next to this file instead.
# `docker compose --profile monitoring up -d` also starts Prometheus on 9090.
#
# Compose reads .env from this directory. Keep it: regenerating LITELLM_SALT_KEY
# makes credentials already stored in the database unreadable. A database
# created without LITELLM_SALT_KEY encrypted them with the master key, so when
# upgrading one, write only LITELLM_MASTER_KEY (the existing one) to .env. For anything
# beyond local evaluation, pin the image to a specific release tag.
services:
litellm:
image: docker.litellm.ai/berriai/litellm:main-stable
build:
context: .
args:
target: runtime
image: docker.litellm.ai/berriai/litellm:main-stable
#########################################
## Uncomment these lines to start proxy with a config.yaml file ##
# volumes:
# - ./config.yaml:/app/config.yaml
# command:
# - "--config=/app/config.yaml"
##############################################
target: runtime
pull_policy: missing
ports:
- "4000:4000" # Map the container port to the host, change the host port if necessary
- "4000:4000"
# The same image runs every component. The first word of `command` picks
# it: proxy (default), gateway, backend, ui, migrations, metrics,
# collector. See docker/README.md
# command: ["--config", "/app/config.yaml"]
# volumes:
# - ./config.yaml:/app/config.yaml:ro
environment:
DATABASE_URL: "postgresql://llmproxy:dbpassword9090@db:5432/litellm"
# Optional: route read-only queries (find_*, count, group_by, query_raw/_first)
# to a separate reader endpoint, e.g. an Aurora reader. Leave unset for
# single-DB deployments. With IAM_TOKEN_DB_AUTH enabled, the reader URL
# is auto-refreshed alongside the writer.
# DATABASE_URL_READ_REPLICA: "postgresql://llmproxy:dbpassword9090@db-reader:5432/litellm"
STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI
LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY:?set it in .env, see the header of this file}
LITELLM_MIGRATION_DIR: /tmp/litellm-migrations
DATABASE_URL: postgresql://llmproxy:dbpassword9090@db:5432/litellm
STORE_MODEL_IN_DB: "True"
env_file:
- .env # Load local .env file
- path: .env
required: false
read_only: true
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
tmpfs:
- /tmp:rw,noexec,nosuid,nodev,size=64m,mode=1777
- /app/.cache:rw,noexec,nosuid,nodev,size=128m,mode=1777
depends_on:
- db # Indicates that this service depends on the 'db' service, ensuring 'db' starts first
healthcheck: # Defines the health check configuration for the container
db:
condition: service_healthy
healthcheck:
test:
- CMD-SHELL
- python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveliness')" # Command to execute for health check
interval: 30s # Perform health check every 30 seconds
timeout: 10s # Health check command times out after 10 seconds
retries: 3 # Retry up to 3 times if health check fails
start_period: 40s # Wait 40 seconds after container start before beginning health checks
- python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveliness')"
interval: 30s
timeout: 10s
retries: 3
start_period: 40s
db:
image: postgres:16
restart: always
container_name: litellm_db
environment:
POSTGRES_DB: litellm
POSTGRES_USER: llmproxy
POSTGRES_PASSWORD: dbpassword9090
ports:
- "5432:5432"
volumes:
- postgres_data:/var/lib/postgresql/data # Persists Postgres data across container restarts
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -d litellm -U llmproxy"]
interval: 1s
@ -55,6 +74,8 @@ services:
prometheus:
image: prom/prometheus
profiles:
- monitoring
volumes:
- prometheus_data:/prometheus
- ./prometheus.yml:/etc/prometheus/prometheus.yml
@ -68,6 +89,5 @@ services:
volumes:
prometheus_data:
driver: local
postgres_data:
name: litellm_postgres_data # Named volume for Postgres data persistence
name: litellm_postgres_data

140
docker-entrypoint.sh Executable file
View file

@ -0,0 +1,140 @@
#!/bin/sh
# LiteLLM image entrypoint: docker-entrypoint.sh [COMPONENT] [ARGS...]
#
# COMPONENT selects the process this container runs; it can also be given as
# LITELLM_COMPONENT when the command carries no component. Anything after it is
# passed to that process. A first argument starting with "-" (or no argument at
# all) keeps the historical behaviour of running the monolithic proxy.
#
# proxy litellm ARGS everything in one process (default)
# gateway python -m gateway.launch ARGS inference routes, 0.0.0.0:4000
# backend uvicorn backend.main:app ARGS management routes, 0.0.0.0:4001
# ui nginx serving the admin UI port 3000
# migrations python migrations/run.py prisma migrate deploy, then exit
# metrics python -m litellm.proxy.prometheus_metrics_server ARGS
# collector python -m litellm.proxy.collector ARGS
# --admin-agent the bundled LiteAdmin Slack agent (litellm-admin-agent --web ARGS)
#
# gateway and backend get their host and port defaults first, so ARGS such as
# --port 8080 override them. With no DATABASE_URL, proxy assembles DATABASE_URL
# and DATABASE_URL_READ_REPLICA from DATABASE_HOST and the other discrete
# variables the way gateway and backend do. An unknown first word is executed
# as-is (docker run <image> sh), through ddtrace-run like every component when
# USE_DDTRACE=true. PgBouncer is not a component: LITELLM_PGBOUNCER_ENABLED=true
# starts it inside proxy and gateway. Components that write Prometheus samples
# start with an empty PROMETHEUS_MULTIPROC_DIR; metrics, collector and raw
# readers keep the workers' files, other raw commands clear stale samples.
set -eu
usage() {
awk 'NR == 1 { next } /^#/ { sub(/^# ?/, ""); print; next } { exit }' "$0" >&2
}
component="${LITELLM_COMPONENT:-proxy}"
case "${1:-}" in
proxy|gateway|backend|ui|migrations|metrics|collector)
component="$1"
shift
;;
--admin-agent)
shift
export CONNECTION_AUTH_MODE=native
exec /opt/liteadmin/bin/litellm-admin-agent --web "$@"
;;
""|-*)
;;
help)
usage
exit 0
;;
*)
if [ -n "${PROMETHEUS_MULTIPROC_DIR:-}" ]; then
mkdir -p "$PROMETHEUS_MULTIPROC_DIR"
case "$*" in
*litellm.proxy.prometheus_metrics_server*|*litellm.proxy.collector*) ;;
*) rm -f "$PROMETHEUS_MULTIPROC_DIR"/*.db "$PROMETHEUS_MULTIPROC_DIR"/litellm_admitted_series_* ;;
esac
fi
component=raw
;;
esac
case "$component" in
raw)
;;
proxy)
if { [ -z "${DATABASE_URL:-}" ] && [ -n "${DATABASE_HOST:-}" ]; } ||
{ [ -z "${DATABASE_URL_READ_REPLICA:-}" ] && [ -n "${DATABASE_HOST_READ_REPLICA:-}" ]; }; then
set -- python -c 'import os, sys
from litellm.proxy.db.db_url_settings import (
DatabaseURLSettings,
add_missing_query_params,
connection_params_from_url,
idle_lifetime_params,
translate_libpq_ssl_params,
)
if os.environ.get("USE_AWS_KMS") == "True":
from litellm.secret_managers.aws_secret_manager import decrypt_env_var
os.environ.update(decrypt_env_var())
settings = DatabaseURLSettings.from_env()
settings.apply_writer_url_to_env()
reader = settings.build_reader_url()
if reader:
os.environ["DATABASE_URL_READ_REPLICA"] = translate_libpq_ssl_params(
add_missing_query_params(reader, connection_params_from_url(os.environ.get("DATABASE_URL", "")))
)
if settings.max_idle_connection_lifetime is not None:
lifetime = idle_lifetime_params(settings.max_idle_connection_lifetime)
for name in ("DATABASE_URL", "DATABASE_URL_READ_REPLICA"):
if os.environ.get(name):
os.environ[name] = add_missing_query_params(os.environ[name], lifetime)
os.execvp("litellm", ["litellm", *sys.argv[1:]])' "$@"
else
set -- litellm "$@"
fi
;;
gateway)
set -- python -m gateway.launch --workers "${NUM_WORKERS:-1}" --host 0.0.0.0 --port 4000 "$@"
;;
backend)
set -- uvicorn backend.main:app --host 0.0.0.0 --port 4001 "$@"
;;
ui)
sh "$(dirname "$0")/ui/docker-entrypoint.d/40-litellm-runtime-config.sh"
exec nginx -g 'daemon off;' "$@"
;;
migrations)
set -- python /app/migrations/run.py "$@"
;;
metrics)
set -- python -m litellm.proxy.prometheus_metrics_server "$@"
;;
collector)
set -- python -m litellm.proxy.collector "$@"
;;
*)
echo "docker-entrypoint.sh: unknown LITELLM_COMPONENT '$component'" >&2
usage
exit 64
;;
esac
if [ -n "${PROMETHEUS_MULTIPROC_DIR:-}" ]; then
case "$component" in
raw) ;;
metrics|collector) mkdir -p "$PROMETHEUS_MULTIPROC_DIR" ;;
*)
mkdir -p "$PROMETHEUS_MULTIPROC_DIR"
rm -f "$PROMETHEUS_MULTIPROC_DIR"/*.db "$PROMETHEUS_MULTIPROC_DIR"/litellm_admitted_series_*
;;
esac
fi
case "${USE_DDTRACE:-}" in
[Tt][Rr][Uu][Ee])
export DD_TRACE_OPENAI_ENABLED="False"
exec ddtrace-run "$@"
;;
esac
exec "$@"

172
docker/Dockerfile.database
View file

@ -1,172 +0,0 @@
# syntax=docker/dockerfile:1.7
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x)
ARG PGBOUNCER_VERSION=1.26.0
ARG PGBOUNCER_SHA256=afd25dd61ee6775d37b40629b87ce08736b3e6955f3057bb212e410fbf21c71d
FROM $UV_IMAGE AS uvbin
FROM $LITELLM_BUILD_IMAGE AS pgbouncer-builder
ARG PGBOUNCER_VERSION
ARG PGBOUNCER_SHA256
USER root
RUN apk add --no-cache build-base pkgconf libevent-dev openssl-dev curl
WORKDIR /build
RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${PGBOUNCER_VERSION}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz" && \
echo "${PGBOUNCER_SHA256} pgbouncer.tar.gz" | sha256sum -c - && \
tar xzf pgbouncer.tar.gz --strip-components=1 && \
./configure --prefix=/usr/local --with-openssl=/usr && \
make -j"$(nproc)" pgbouncer && \
install -m 0755 pgbouncer /usr/local/bin/pgbouncer
# Admin UI builder. Pinned to the build platform so the architecture-independent
# Next.js static export compiles once natively even in a multi-arch build,
# instead of once per target arch under QEMU.
FROM --platform=$BUILDPLATFORM $UI_BUILD_IMAGE AS ui-builder
ENV NEXT_TELEMETRY_DISABLED=1 \
npm_config_fund=false \
npm_config_audit=false
WORKDIR /ui
COPY ui/litellm-dashboard/package.json ui/litellm-dashboard/package-lock.json ./
COPY ui/litellm-dashboard/vendor/ ./vendor/
RUN --mount=type=cache,target=/root/.npm npm ci --prefer-offline
COPY ui/litellm-dashboard/ ./
RUN npm run build
FROM $LITELLM_BUILD_IMAGE AS builder
WORKDIR /app
USER root
COPY --from=uvbin /uv /usr/local/bin/uv
COPY --from=uvbin /uvx /usr/local/bin/uvx
RUN apk add --no-cache \
bash \
gcc \
python-3.13 \
python-3.13-dev \
openssl \
libssl3 \
openssl-dev \
nodejs \
npm \
libsndfile
ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
UV_LINK_MODE=copy \
UV_PYTHON_DOWNLOADS=0 \
PATH="/app/.venv/bin:${PATH}"
# Copy dependency metadata first for layer caching
COPY pyproject.toml uv.lock ./
COPY enterprise/pyproject.toml enterprise/
COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/
COPY packaging/litellm-proxy/pyproject.toml packaging/litellm-proxy/
# Install third-party dependencies (cached unless pyproject.toml/uv.lock change)
RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
--extra extra_proxy \
--extra semantic-router \
--extra saml \
--extra bedrock-realtime \
--python python3.13
# Copy full source tree
COPY . .
# Replace the committed UI bundle with the one built from this exact source.
# Clearing first drops the committed bundle's content-hashed chunks that COPY
# would otherwise leave behind alongside the fresh ones.
RUN rm -rf litellm/proxy/_experimental/out
COPY --from=ui-builder /ui/out/. litellm/proxy/_experimental/out/
# Build Admin UI before final sync (applies the enterprise color override when present)
RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Install project and workspace packages (fast - deps already cached)
RUN uv build --package litellm-proxy --wheel --out-dir /wheels
RUN uv sync --frozen --no-install-package litellm-proxy --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
--extra extra_proxy \
--extra semantic-router \
--extra saml \
--extra bedrock-realtime \
--python python3.13
RUN uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm_proxy-*.whl
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
npm_config_cache=/root/.npm \
prisma generate --schema=./schema.prisma
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
FROM $LITELLM_RUNTIME_IMAGE AS runtime
ARG LITELLM_RELEASE_TAG=""
ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
USER root
# node (without npm) is required by the prisma CLI at runtime
RUN apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libevent
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
WORKDIR /app
ENV PATH="/app/.venv/bin:${PATH}" \
PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
PRISMA_CLI_PATH=/opt/prisma/binaries/node_modules/.bin/prisma \
PRISMA_CLI_QUERY_ENGINE_TYPE=binary \
PRISMA_OFFLINE_MODE=true
# Copy only what runtime needs. The application is installed inside the venv;
# the rest of the builder's /app is source and build metadata that must not
# ship (manifest-scanning tools attribute everything in it to this image).
# entrypoint.sh invokes litellm/proxy/prisma_migration.py by source path.
COPY --from=builder /app/.venv /app/.venv
COPY --from=builder /app/docker /app/docker
COPY --from=builder /app/schema.prisma /app/schema.prisma
COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/prisma_migration.py
# enterprise/ is imported by source path at runtime (proxy_cli puts the
# working directory on sys.path; litellm/proxy/hooks resolves
# enterprise.enterprise_hooks from it)
COPY --from=builder /app/enterprise /app/enterprise
COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras
# Prisma CLI + engines are baked under /opt/prisma, a fixed path every
# runtime uid can read and that no cache volume mount shadows (unlike
# /app/.cache or $HOME/.cache in readOnlyRootFilesystem + emptyDir setups).
# The paths are pinned via PRISMA_BINARY_CACHE_DIR / PRISMA_CLI_PATH and
# recorded into the generated client at build time, so `prisma migrate
# deploy` on a fresh database needs no npm and no network access
# (#33650, #24554).
COPY --from=builder /opt/prisma /opt/prisma
RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
find /app/.venv -type d -path "*/tornado/test" -delete && \
chmod -R a+rX /opt/prisma && \
test -x /opt/prisma/binaries/node_modules/.bin/prisma && \
test -f /opt/prisma/binaries/node_modules/prisma/build/index.js && \
python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths"
EXPOSE 4000/tcp
ENTRYPOINT ["docker/prod_entrypoint.sh"]
CMD ["--port", "4000"]

208
docker/Dockerfile.non_root
View file

@ -1,208 +0,0 @@
# syntax=docker/dockerfile:1.7
# Base images
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x)
ARG PGBOUNCER_VERSION=1.26.0
ARG PGBOUNCER_SHA256=afd25dd61ee6775d37b40629b87ce08736b3e6955f3057bb212e410fbf21c71d
FROM $UV_IMAGE AS uvbin
FROM $LITELLM_BUILD_IMAGE AS pgbouncer-builder
ARG PGBOUNCER_VERSION
ARG PGBOUNCER_SHA256
USER root
RUN apk add --no-cache build-base pkgconf libevent-dev openssl-dev curl
WORKDIR /build
RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${PGBOUNCER_VERSION}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz" && \
echo "${PGBOUNCER_SHA256} pgbouncer.tar.gz" | sha256sum -c - && \
tar xzf pgbouncer.tar.gz --strip-components=1 && \
./configure --prefix=/usr/local --with-openssl=/usr && \
make -j"$(nproc)" pgbouncer && \
install -m 0755 pgbouncer /usr/local/bin/pgbouncer
# Admin UI builder. Pinned to the build platform so the architecture-independent
# Next.js static export compiles once natively even in a multi-arch build,
# instead of once per target arch under QEMU.
FROM --platform=$BUILDPLATFORM $UI_BUILD_IMAGE AS ui-builder
ENV NEXT_TELEMETRY_DISABLED=1 \
npm_config_fund=false \
npm_config_audit=false
WORKDIR /ui
COPY ui/litellm-dashboard/package.json ui/litellm-dashboard/package-lock.json ./
COPY ui/litellm-dashboard/vendor/ ./vendor/
RUN --mount=type=cache,target=/root/.npm npm ci --prefer-offline
COPY ui/litellm-dashboard/ ./
RUN npm run build
FROM $LITELLM_BUILD_IMAGE AS builder
WORKDIR /app
USER root
COPY --from=uvbin /uv /usr/local/bin/uv
COPY --from=uvbin /uvx /usr/local/bin/uvx
RUN for i in 1 2 3; do \
apk add --no-cache \
python-3.13 \
python-3.13-dev \
gcc \
rust \
bash \
coreutils \
curl \
openssl \
libssl3 \
libsndfile \
nodejs \
npm && break || sleep 5; \
done
ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
UV_LINK_MODE=copy \
UV_PYTHON_DOWNLOADS=0 \
PATH="/app/.venv/bin:${PATH}" \
LITELLM_NON_ROOT=true \
XDG_CACHE_HOME=/app/.cache
# Copy dependency metadata first for layer caching
COPY pyproject.toml uv.lock ./
COPY enterprise/pyproject.toml enterprise/
COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/
# Install third-party dependencies (cached unless pyproject.toml/uv.lock change)
RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
--extra extra_proxy \
--extra semantic-router \
--extra saml \
--extra bedrock-realtime \
--python python3.13
# Copy full source tree
COPY . .
# Replace the committed UI bundle with the one built from this exact source.
# Clearing first drops the committed bundle's content-hashed chunks that COPY
# would otherwise leave behind alongside the fresh ones.
RUN rm -rf litellm/proxy/_experimental/out
COPY --from=ui-builder /ui/out/. litellm/proxy/_experimental/out/
# Set non-root flag for build time consistency
ENV LITELLM_NON_ROOT=true
RUN mkdir -p /var/lib/litellm/ui /var/lib/litellm/assets && \
cp -r /app/litellm/proxy/_experimental/out/. /var/lib/litellm/ui/ && \
cp /app/litellm/proxy/logo.png /var/lib/litellm/assets/logo.png && \
touch /var/lib/litellm/ui/.litellm_ui_ready
RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
uv sync --frozen --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
--extra extra_proxy \
--extra semantic-router \
--extra saml \
--extra bedrock-realtime \
--python python3.13
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
npm_config_cache=/root/.npm \
prisma generate --schema=./schema.prisma
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
FROM $LITELLM_RUNTIME_IMAGE AS runtime
ARG LITELLM_RELEASE_TAG=""
ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
WORKDIR /app
USER root
RUN for i in 1 2 3; do \
apk upgrade --no-cache && break || sleep 5; \
done && \
for i in 1 2 3; do \
apk add --no-cache python-3.13 bash openssl libssl3 tzdata libsndfile nodejs libevent && break || sleep 5; \
done
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
# Copy only what runtime needs. The application is installed inside the venv;
# the rest of the builder's /app is source and build metadata that must not
# ship (manifest-scanning tools attribute everything in it to this image).
# entrypoint.sh invokes litellm/proxy/prisma_migration.py by source path.
COPY --from=builder /app/.venv /app/.venv
COPY --from=builder /app/docker /app/docker
COPY --from=builder /app/schema.prisma /app/schema.prisma
COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/prisma_migration.py
# enterprise/ is imported by source path at runtime (proxy_cli puts the
# working directory on sys.path; litellm/proxy/hooks resolves
# enterprise.enterprise_hooks from it)
COPY --from=builder /app/enterprise /app/enterprise
COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras
# Prisma CLI + engines are baked under /opt/prisma, a fixed path every runtime
# uid can read and that no cache volume mount shadows (unlike /app/.cache or
# $HOME/.cache under readOnlyRootFilesystem + emptyDir or arbitrary-uid setups).
# PRISMA_CLI_QUERY_ENGINE_TYPE=binary makes the CLI use the baked binary query
# engine directly, so `prisma migrate deploy` on a fresh database needs no npm
# and no network access; without it the CLI looks for the library engine, which
# prisma stopped baking, and falls back to a download that fails offline or as a
# non-writable uid (#33650, #24554).
COPY --from=builder /opt/prisma /opt/prisma
COPY --from=builder /var/lib/litellm/ui /var/lib/litellm/ui
COPY --from=builder /var/lib/litellm/assets /var/lib/litellm/assets
# XDG_CACHE_HOME is intentionally left unset so it falls back to $HOME/.cache
# (/app/.cache, writable by the runtime uid). The prisma bake at the read-only
# /opt/prisma is anchored by PRISMA_BINARY_CACHE_DIR / PRISMA_CLI_PATH, so
# nothing needs XDG to point there; pointing it at the read-only bake would
# deny any XDG-aware library that writes a cache at runtime.
ENV PATH="/app/.venv/bin:${PATH}" \
PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
PRISMA_CLI_PATH=/opt/prisma/binaries/node_modules/.bin/prisma \
PRISMA_CLI_QUERY_ENGINE_TYPE=binary \
HOME=/app \
LITELLM_NON_ROOT=true \
PRISMA_SKIP_POSTINSTALL_GENERATE=1 \
PRISMA_HIDE_UPDATE_MESSAGE=1 \
PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING=1 \
PRISMA_OFFLINE_MODE=true
RUN mkdir -p /nonexistent /app/.cache /var/lib/litellm/assets /var/lib/litellm/ui && \
chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent && \
PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
chown -R nobody:nogroup "$PRISMA_PATH" && \
LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \
[ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup "$LITELLM_PKG_MIGRATIONS_PATH" || true && \
LITELLM_PROXY_EXTRAS_PATH=$(python -c "import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))" 2>/dev/null || echo "") && \
chgrp -R 0 "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chgrp -R 0 "$LITELLM_PROXY_EXTRAS_PATH" || true && \
chmod -R g=u "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u "$LITELLM_PROXY_EXTRAS_PATH" || true && \
chmod -R g+w "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets && \
[ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w "$LITELLM_PROXY_EXTRAS_PATH" || true && \
chmod -R g+rX "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets && \
chmod -R a+rX /opt/prisma && \
test -x /opt/prisma/binaries/node_modules/.bin/prisma && \
test -f /opt/prisma/binaries/node_modules/prisma/build/index.js && \
ls /opt/prisma/binaries/node_modules/@prisma/engines/query-engine-* >/dev/null 2>&1 && \
python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths"
USER 65534
EXPOSE 4000/tcp
ENTRYPOINT ["/app/docker/prod_entrypoint.sh"]
CMD ["--port", "4000"]

95
docker/README.md
View file

@ -2,15 +2,15 @@
This guide provides instructions for building and running the LiteLLM application using Docker and Docker Compose.
> **Just want to run LiteLLM?** This guide builds from source. To run the published
> image instead, use `docker-compose.quickstart.yml` in this directory — the
> two-service stack (gateway + Postgres) that the
> [Docker quickstart](https://docs.litellm.ai/docs/proxy/docker_quick_start) documents:
> **Just want to run LiteLLM?** `docker-compose.yml` in the repository root runs
> the published image with a Postgres database, the stack the
> [Docker quickstart](https://docs.litellm.ai/docs/proxy/docker_quick_start) documents,
> and it works on its own outside a checkout:
>
> ```bash
> curl -sSLO https://github.com/BerriAI/litellm/raw/main/docker/docker-compose.quickstart.yml
> curl -sSLO https://github.com/BerriAI/litellm/raw/main/docker-compose.yml
> printf 'LITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\n' "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" > .env
> docker compose -f docker-compose.quickstart.yml up -d
> docker compose up -d
> ```
## Prerequisites
@ -20,33 +20,52 @@ This guide provides instructions for building and running the LiteLLM applicatio
## Building and Running the Application
To build and run the application, you will use the `docker-compose.yml` file located in the root of the project. This file is configured to use the `Dockerfile.non_root` for a secure, non-root container environment.
The same `docker-compose.yml` builds from source when you pass `--build`: it builds the `Dockerfile` in the repository root, the one image LiteLLM ships. `docker-entrypoint.sh` next to it is the only entrypoint script; every container starts through it
### 1. Set the Master Key
## One image, many components
The application requires a `LITELLM_MASTER_KEY` for signing and validating tokens. You must set this key as an environment variable before running the application.
Every LiteLLM container runs the same image. The first word of the container command (or the `LITELLM_COMPONENT` environment variable when the command carries only flags) picks the process the container runs, and everything after it is handed to that process unchanged:
Create a `.env` file in the root of the project and add the following line:
| Component | Runs | Port |
|--------------|-----------------------------------------------------|------|
| `proxy` | `litellm ...` (everything in one process, default) | 4000 |
| `gateway` | `python -m gateway.launch ...` (inference routes) | 4000 |
| `backend` | `uvicorn backend.main:app ...` (management routes) | 4001 |
| `ui` | nginx serving the static admin UI | 3000 |
| `migrations` | `python migrations/run.py`, `prisma migrate deploy` then exit | |
| `metrics` | `python -m litellm.proxy.prometheus_metrics_server ...` | `--port` |
| `collector` | `python -m litellm.proxy.collector ...` | |
```
LITELLM_MASTER_KEY=your-secret-key
```bash
docker run -p 4000:4000 litellm --config /app/config.yaml # proxy, exactly as before
docker run -p 4000:4000 litellm gateway --port 4000 # componentized data plane
docker run -p 4001:4001 -e LITELLM_COMPONENT=backend litellm # same, chosen through the env
docker run -p 3000:3000 --read-only --tmpfs /tmp litellm ui # admin UI behind nginx
docker run -e DATABASE_URL=... litellm migrations # one-off schema migration job
docker run -it litellm sh # anything else runs verbatim
```
Replace `your-secret-key` with a strong, randomly generated secret.
PgBouncer is not a separate component: `LITELLM_PGBOUNCER_ENABLED=true` starts an in-container PgBouncer in front of `DATABASE_URL` inside `proxy` and `gateway`. `USE_DDTRACE=true` wraps whichever component runs with `ddtrace-run`, and `PROMETHEUS_MULTIPROC_DIR` is emptied of stale samples before any workers fork (the `metrics` and `collector` sidecars only read it, so their restart keeps the live samples)
The image runs as uid `65532` (`nonroot` in the Wolfi base) and also works as an arbitrary uid in gid 0, the shape OpenShift `restricted-v2` assigns, because everything it writes at runtime lives under `/app/.cache`, `/var/lib/litellm` and `/tmp`. Mount those (or set `readOnlyRootFilesystem` with emptyDirs there) for a read-only root filesystem. Prisma's CLI and engines are baked under `/opt/prisma`, so migrations need neither network nor a writable home
### 1. Set the Master and Salt Keys
The proxy signs virtual keys with `LITELLM_MASTER_KEY` and encrypts stored provider credentials with `LITELLM_SALT_KEY`. Compose reads both from a `.env` file in the directory you run it from, so generate them once:
```bash
printf 'LITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\n' "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" > .env
```
Keep the file: regenerating `LITELLM_SALT_KEY` makes credentials already stored in the database unreadable. A database created without `LITELLM_SALT_KEY` encrypted them with the master key, so when upgrading one, write only the existing `LITELLM_MASTER_KEY` to `.env` and leave `LITELLM_SALT_KEY` out. Provider keys such as `OPENAI_API_KEY` go in the same file, and the whole file is passed to the container
### 2. Build and Run the Containers
Once you have set the `LITELLM_MASTER_KEY`, you can build and run the containers using the following command:
```bash
docker compose up -d --build
```
This command will:
- Build the Docker image using `Dockerfile.non_root`.
- Start the `litellm`, `litellm_db`, and `prometheus` services in detached mode (`-d`).
- The `--build` flag ensures that the image is rebuilt if there are any changes to the Dockerfile or the application code.
This command builds the image from the root `Dockerfile` and starts the `litellm` and `db` services in detached mode. Without `--build`, `docker compose up` pulls the published `main-stable` image instead. Add `--profile monitoring` to also start Prometheus on port 9090, scraping the proxy with the root `prometheus.yml`
### 3. Verifying the Application is Running
@ -80,13 +99,13 @@ LITELLM_LICENSE="your-enterprise-license"
PROXY_BASE_URL=https://gateway.example.com
```
For the unified source deployment described above, put them in its `.env` file and rebuild:
For the compose deployment described above, put them in its `.env` file and rebuild:
```bash
docker compose up -d --build
```
In componentized deployments, set the flag and license on the backend container and route `/admin/mcp` to the backend service. The gateway component excludes this endpoint. The unified, database, non-root, and backend image builds bundle the connector
In componentized deployments, set the flag and license on the `backend` component and route `/admin/mcp` to it. The `gateway` component excludes this endpoint. The image bundles the connector for every component
Hosting is disabled by default. Opting in requires a valid base Enterprise license; an unlicensed opt-in or invalid flag value prevents startup. Enabling it reserves `/admin`, so rename any MCP server alias called `admin` first
@ -96,34 +115,18 @@ Embedded responses default to `full`; selecting `LITELLM_ADMIN_RESPONSE_VIEW=com
See the [LiteAdmin MCP guide](https://docs.litellm.ai/docs/proxy/liteadmin_mcp#run-liteadmin-mcp-inside-litellm) for client configuration, tool restrictions, and verification
## Hardened / Offline Testing
## Hardening
To ensure changes are safe for non-root, read-only root filesystems and restricted egress, always validate with the hardened compose file:
The compose file runs the proxy the way a locked-down cluster would: as the image's non-root user with a read-only root filesystem, every capability dropped, `no-new-privileges` set, and tmpfs mounts only at `/tmp` and `/app/.cache`. The image is built for that, so a change that makes the proxy write anywhere else fails here before it fails in Kubernetes. To try an arbitrary uid in gid 0, the shape OpenShift `restricted-v2` assigns, add `user: "101:0"` to the `litellm` service
Prisma's CLI and engines are baked under `/opt/prisma`, so migrations run without network access. Verify with:
```bash
docker compose -f docker-compose.yml -f docker-compose.hardened.yml build --no-cache
docker compose -f docker-compose.yml -f docker-compose.hardened.yml up -d
docker run --rm --network none --entrypoint prisma docker.litellm.ai/berriai/litellm:main-stable --version
```
This setup:
- Builds from `docker/Dockerfile.non_root` with Prisma engines and Node toolchain baked into the image.
- Runs the proxy as a non-root user with a read-only rootfs and only writable tmpfs mounts:
- `/app/cache` (Prisma/NPM cache; backing `PRISMA_BINARY_CACHE_DIR`, `NPM_CONFIG_CACHE`, `XDG_CACHE_HOME`)
- `/app/migrations` (Prisma migration workspace; backing `LITELLM_MIGRATION_DIR`)
- Pre-builds and serves the admin UI from read-only paths:
- `/var/lib/litellm/ui` (pre-restructured Next.js UI with `.litellm_ui_ready` marker)
- `/var/lib/litellm/assets` (UI logos and assets)
- Routes all outbound traffic through a local Squid proxy that denies egress, so Prisma migrations must use the cached CLI and engines.
You should also verify offline Prisma behaviour with:
```bash
docker run --rm --network none --entrypoint prisma ghcr.io/berriai/litellm:main-stable --version
```
This command should succeed (showing engine versions) even with `--network none`, confirming that Prisma binaries are available without network access.
## Troubleshooting
- **`build_admin_ui.sh: not found`**: This error can occur if the Docker build context is not set correctly. Ensure that you are running the `docker-compose` command from the root of the project.
- **`Master key is not initialized`**: This error means the `LITELLM_MASTER_KEY` environment variable is not set. Make sure you have created a `.env` file in the project root with the `LITELLM_MASTER_KEY` defined.
- **`required variable LITELLM_MASTER_KEY is missing a value`**: Compose did not find a `.env` file with `LITELLM_MASTER_KEY` in the directory you ran it from. Generate one as shown above.
- **`password authentication failed for user "llmproxy"`**: the `litellm_postgres_data` volume was initialised by an older compose file with different credentials. `docker compose down -v` drops it and the next `up` recreates the database.
- **`build_admin_ui.sh: not found`**: the build context is wrong. Run `docker compose` from the root of the repository.

64
docker/build_from_pip/Dockerfile.build_from_pip
View file

@ -1,64 +0,0 @@
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.10.9@sha256:10902f58a1606787602f303954cea099626a4adb02acbac4c69920fe9d278f82
FROM $UV_IMAGE AS uvbin
FROM python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d
ARG LITELLM_VERSION=1.83.0
WORKDIR /app
COPY --from=uvbin /uv /usr/local/bin/uv
COPY --from=uvbin /uvx /usr/local/bin/uvx
RUN apt-get update && \
apt-get install -y --no-install-recommends gcc libffi-dev nodejs npm && \
rm -rf /var/lib/apt/lists/*
ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
UV_LINK_MODE=copy \
PATH="/app/.venv/bin:${PATH}"
COPY schema.prisma .
# This image is specifically for validating/installing the published PyPI
# artifact, not the checked-out source tree.
# Keep the moved proxy-runtime packages explicit until the published PyPI
# artifact includes that extra; newer releases will simply dedupe these.
RUN uv venv --python python && \
uv pip install --python /app/.venv/bin/python \
"litellm[proxy,proxy-runtime]==${LITELLM_VERSION}" \
"google-cloud-aiplatform==1.133.0" \
"google-genai==1.37.0" \
"anthropic[vertex]==0.84.0" \
"grpcio==1.84.1" \
"prometheus-client==0.20.0" \
"langfuse==2.59.7" \
"opentelemetry-api==1.28.0" \
"opentelemetry-sdk==1.28.0" \
"opentelemetry-exporter-otlp==1.28.0" \
"ddtrace==4.11.0" \
"sentry-sdk==2.21.0" \
"mangum==0.17.0" \
"azure-ai-contentsafety==1.0.0" \
"azure-storage-file-datalake==12.20.0" \
"pypdf==6.7.5" \
"llm-sandbox==0.3.31" \
"detect-secrets==1.5.0" \
"prisma==0.11.0" \
"openai==2.24.0"
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
npm_config_cache=/root/.npm \
prisma generate --schema=./schema.prisma && \
chmod -R a+rX /opt/prisma && \
python -c "import sys; from prisma.client import BINARY_PATHS; bad = sorted(p for group in BINARY_PATHS.model_dump().values() for p in group.values() if not p.startswith('/opt/prisma/')); sys.exit('prisma engines baked outside /opt/prisma: %r' % bad) if bad else None"
ENV PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
COPY docker/prod_entrypoint.sh /app/docker/prod_entrypoint.sh
RUN sed -i 's/\r$//' /app/docker/prod_entrypoint.sh && chmod +x /app/docker/prod_entrypoint.sh
EXPOSE 4000/tcp
ENTRYPOINT ["/app/docker/prod_entrypoint.sh"]
CMD ["--port", "4000"]

9
docker/build_from_pip/Readme.md
View file

@ -1,9 +0,0 @@
# Docker to build LiteLLM Proxy from litellm pip package
### When to use this ?
If you need to build LiteLLM Proxy from litellm pip package, you can use this Dockerfile as a reference.
### Why build from pip package ?
- If your company has a strict requirement around security / building images you can follow steps outlined here

16
docker/component_entrypoint.sh
View file

@ -1,16 +0,0 @@
#!/bin/sh
# stale samples from a previous container incarnation would be summed into the aggregate
if [ -n "$PROMETHEUS_MULTIPROC_DIR" ]; then
mkdir -p "$PROMETHEUS_MULTIPROC_DIR"
rm -f "$PROMETHEUS_MULTIPROC_DIR"/*.db "$PROMETHEUS_MULTIPROC_DIR"/litellm_admitted_series_*
fi
case "$USE_DDTRACE" in
[Tt][Rr][Uu][Ee])
export DD_TRACE_OPENAI_ENABLED="False"
exec ddtrace-run "$@"
;;
esac
exec "$@"

16
docker/entrypoint.sh
View file

@ -1,16 +0,0 @@
#!/bin/bash
set -euo pipefail
REPO_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
VENV_PYTHON="$REPO_ROOT/.venv/bin/python"
MIGRATION_SCRIPT="$REPO_ROOT/litellm/proxy/prisma_migration.py"
if [ -x "$VENV_PYTHON" ]; then
"$VENV_PYTHON" "$MIGRATION_SCRIPT"
elif command -v uv >/dev/null 2>&1; then
(cd "$REPO_ROOT" && uv run --no-sync python "$MIGRATION_SCRIPT")
else
python3 "$MIGRATION_SCRIPT"
fi
echo "Migration script ran successfully!"

4
docker/install_auto_router.sh
View file

@ -1,4 +0,0 @@
#!/bin/bash
set -euo pipefail
# semantic-router dependencies are installed via `uv sync`.

16
docker/prod_entrypoint.sh
View file

@ -1,16 +0,0 @@
#!/bin/sh
if [ "$1" = "--admin-agent" ]; then
shift
export CONNECTION_AUTH_MODE=native
exec /opt/liteadmin/bin/litellm-admin-agent --web "$@"
fi
case "$USE_DDTRACE" in
[Tt][Rr][Uu][Ee])
export DD_TRACE_OPENAI_ENABLED="False"
exec ddtrace-run litellm "$@"
;;
esac
exec litellm "$@"

18
docker/tests/nonroot.yaml
View file

@ -1,18 +0,0 @@
schemaVersion: 2.0.0
metadataTest:
entrypoint: ["docker/prod_entrypoint.sh"]
user: "65534"
workdir: "/app"
fileExistenceTests:
- name: "Prisma Folder"
path: "/usr/local/lib/python3.13/site-packages/prisma/"
shouldExist: true
uid: 65534
gid: 65534
- name: "Prisma Schema"
path: "/usr/local/lib/python3.13/site-packages/prisma/schema.prisma"
shouldExist: true
uid: 65534
gid: 65534

3
enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py
View file

@ -17,6 +17,7 @@ from typing import List, Optional, Union
from litellm._logging import verbose_logger
from litellm.caching import DualCache
from litellm.exceptions import MissingMessageRoleError
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
@ -86,6 +87,8 @@ class PagerDutyAlerting(SlackAlerting):
Record a failure event. Only send an alert to PagerDuty if the
configured *failure* threshold is exceeded in the specified window.
"""
if isinstance(kwargs.get("exception"), MissingMessageRoleError):
return
now = datetime.now(timezone.utc)
standard_logging_payload: Optional[StandardLoggingPayload] = kwargs.get(
"standard_logging_object"

3
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -20,6 +20,7 @@ from litellm_enterprise.proxy.common_utils.managed_object_attribution import (
import litellm
from litellm._logging import verbose_proxy_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.cloud_storage_security import stored_s3_bucket_kwargs
from litellm.constants import (
BATCH_LINE_ITEMS_MAX_PART_BYTES,
MANAGED_OBJECT_STALENESS_CUTOFF_DAYS,
@ -765,7 +766,7 @@ class CheckBatchCost:
credentials: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {}
_file_content = await afile_content(
file_id=cast(str, raw_output_file_id),
**cast(FileContentCallOptions, credentials),
**cast(FileContentCallOptions, {**credentials, **stored_s3_bucket_kwargs(cast(str, raw_output_file_id))}),
)
content_bytes: Final = await _file_content_bytes(_file_content)

22
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -49,6 +49,7 @@ from litellm.constants import (
)
from litellm.files.types import FileRetrieveCallOptions, FileRetrieveProvider
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.cloud_storage_security import stored_s3_bucket_kwargs
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
from litellm.litellm_core_utils.internal_call_metadata import requests_background
@ -396,6 +397,7 @@ class _CursorPageArgs(TypedDict, total=False):
class _RouterFileCallKwargs(TypedDict, total=False):
client: ReadOnly[AsyncOpenAI | None]
custom_llm_provider: ReadOnly[str | None]
stored_s3_bucket_name: ReadOnly[str]
_MODEL_FILE_ID_MAPPING: Final = TypeAdapter(dict[str, str])
@ -463,7 +465,7 @@ async def _delete_provider_file(
await llm_router.afile_delete(
model=model_id,
file_id=provider_file_id,
**cast(_RouterFileCallKwargs, delete_data),
**cast(_RouterFileCallKwargs, {**delete_data, **stored_s3_bucket_kwargs(provider_file_id)}),
)
@ -1157,7 +1159,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
retrieve_file: Final = cast(_ManagedFileRetrieve, litellm.afile_retrieve)
retrieve_options: Final = cast(
FileRetrieveCallOptions,
{**call_options, "max_retries": 0},
{**call_options, **stored_s3_bucket_kwargs(provider_file_id), "max_retries": 0},
)
for attempt in range(len(_PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS) + 1):
if attempt > 0:
@ -1795,8 +1797,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if potential_file_id and "llm_output_file_id," in potential_file_id:
model_id = self.get_model_id_from_unified_file_id(potential_file_id)
if model_id:
provider_file_id: Final = self.get_output_file_id_from_unified_file_id(potential_file_id)
data["model"] = model_id
data["file_id"] = self.get_output_file_id_from_unified_file_id(potential_file_id)
data["file_id"] = provider_file_id
data.update(stored_s3_bucket_kwargs(provider_file_id))
elif call_type == CallTypes.acreate_batch.value:
input_file_id = cast(Optional[str], data.get("input_file_id"))
if input_file_id:
@ -2894,7 +2898,17 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
await llm_router.afile_content(
model=model_id,
file_id=provider_file_id,
**cast(_RouterFileCallKwargs, data),
**cast(
_RouterFileCallKwargs,
{
**data,
**(
stored_s3_bucket_kwargs(provider_file_id)
if isinstance(provider_file_id, str)
else {}
),
},
),
),
)
except Exception as e:

2
enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py
View file

@ -123,7 +123,7 @@ class _PROXY_LiteLLMManagedVectorStores(
"""
# Use the router to create the vector store
response: Final = await llm_router.avector_store_create(model=model, **request_data)
return cast(VectorStoreCreateResponse, response)
return response
# ============================================================================
# VECTOR STORE CRUD OPERATIONS

16
gateway/AGENTS.md Normal file
View file

@ -0,0 +1,16 @@
# Gateway image entry point
`python -m gateway.launch` is how the gateway image starts. `launch.py` hands its arguments to uvicorn's own CLI and serves `main.py`, which trims the proxy app to the LLM data plane
- Importing `launch.py` stays cheap. Every uvicorn worker imports it again before it can answer its first health ping, so LiteLLM imports wait until `main` or `worker_app` runs
- PgBouncer starts exactly once per pod, before the workers start, and the workers inherit the loopback URL through `DATABASE_URL`. `configure_database` is the only place that does this
## Rust forwarding
`launch.py` is an entry point of Rust forwarding, and every forwarding decision lives in `litellm/proxy/rust_forward/`, whose `AGENTS.md` states the contract
- `main` dispatches on `rust_forward_mode` and does nothing else about forwarding. With forwarding off or refused it serves exactly as before
- `python_server` reads the flags and environment through uvicorn's own CLI parser, so Rust gets the values uvicorn would run with. Never reparse a uvicorn flag by hand
- Under forwarding the supervisor process runs `configure_database`, as `before_start`, once forwarding is known to be possible. The supervised child never runs it and inherits the exported URL
- The supervised child appends `mode.uvicorn_flags` after the operator's flags, so the private listener and `--no-proxy-headers` win
- `worker_app` wraps the app with `peer.wrap`, which is the identity when forwarding is off

170
gateway/Dockerfile
View file

@ -1,170 +0,0 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x)
ARG PGBOUNCER_VERSION=1.26.0
ARG PGBOUNCER_SHA256=afd25dd61ee6775d37b40629b87ce08736b3e6955f3057bb212e410fbf21c71d
FROM $UV_IMAGE AS uvbin
FROM $LITELLM_BUILD_IMAGE AS pgbouncer-builder
ARG PGBOUNCER_VERSION
ARG PGBOUNCER_SHA256
USER root
RUN apk add --no-cache build-base pkgconf libevent-dev openssl-dev curl
WORKDIR /build
RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${PGBOUNCER_VERSION}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz" && \
echo "${PGBOUNCER_SHA256} pgbouncer.tar.gz" | sha256sum -c - && \
tar xzf pgbouncer.tar.gz --strip-components=1 && \
./configure --prefix=/usr/local --with-openssl=/usr && \
make -j"$(nproc)" pgbouncer && \
install -m 0755 pgbouncer /usr/local/bin/pgbouncer
# ---------- Rust extension ----------
# Copies only the native build's inputs, so the fat-LTO compile stays cached across Python-only changes
FROM $LITELLM_BUILD_IMAGE AS rust-native
WORKDIR /app
USER root
COPY --from=uvbin /uv /uvx /usr/local/bin/
RUN for i in 1 2 3; do \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
ENV UV_PYTHON_DOWNLOADS=0
COPY pyproject.toml README.md rust-toolchain.toml ./
COPY .cargo ./.cargo
COPY litellm-rust ./litellm-rust
COPY model_prices_and_context_window.json ./
COPY packaging/litellm-proxy/pyproject.toml packaging/litellm-proxy/
RUN --mount=type=cache,target=/root/.cache/uv \
mkdir -p litellm/rust_bridge /wheels && \
touch litellm/__init__.py litellm/rust_bridge/__init__.py && \
uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \
python -c "import maturin; maturin.build_wheel('/wheels')" && \
cd packaging/litellm-proxy && \
uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \
python -c "import maturin; maturin.build_wheel('/wheels')"
# ---------- Builder ----------
FROM $LITELLM_BUILD_IMAGE AS builder
WORKDIR /app
USER root
COPY --from=uvbin /uv /uvx /usr/local/bin/
# nodejs/npm so `prisma generate` uses Wolfi's Node via PRISMA_USE_GLOBAL_NODE
# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
RUN for i in 1 2 3; do \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
# UV_COMPILE_BYTECODE=1 precompiles .pyc at install time → faster cold start.
# UV_LINK_MODE=copy avoids hardlink warnings when uv installs from a
# BuildKit cache mount (different filesystem).
# UV_PYTHON_DOWNLOADS=0 force uv to use the apk-installed CPython instead of
# silently pulling a managed interpreter.
# PRISMA_USE_GLOBAL_NODE explicit (matches default) so an env override can't
# silently re-enable nodeenv's Node download.
ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
UV_LINK_MODE=copy \
UV_COMPILE_BYTECODE=1 \
UV_PYTHON_DOWNLOADS=0 \
PRISMA_USE_GLOBAL_NODE=true \
PATH="/app/.venv/bin:${PATH}"
# Stage 1 — install dependencies only.
RUN --mount=type=cache,target=/root/.cache/uv \
--mount=type=bind,source=pyproject.toml,target=pyproject.toml \
--mount=type=bind,source=uv.lock,target=uv.lock \
--mount=type=bind,source=enterprise/pyproject.toml,target=enterprise/pyproject.toml \
--mount=type=bind,source=litellm-proxy-extras/pyproject.toml,target=litellm-proxy-extras/pyproject.toml \
--mount=type=bind,source=packaging/litellm-proxy/pyproject.toml,target=packaging/litellm-proxy/pyproject.toml \
uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
--extra extra_proxy \
--extra semantic-router \
--extra bedrock-realtime \
--python python3.13
# Stage 2 — copy source and install the workspace members.
COPY . .
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-install-project --no-install-package litellm-proxy --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
--extra extra_proxy \
--extra semantic-router \
--extra bedrock-realtime \
--python python3.13
RUN --mount=type=bind,from=rust-native,source=/wheels,target=/wheels \
uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm-*.whl /wheels/litellm_proxy-*.whl
# PYTHONPATH=/app makes the source tree shadow the installed package, so the
# compiled Rust extension must live next to the source or it is never imported.
RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
npm_config_cache=/root/.npm \
prisma generate --schema=./schema.prisma
RUN sed -i 's/\r$//' docker/component_entrypoint.sh && chmod +x docker/component_entrypoint.sh
# ---------- Runtime ----------
FROM $LITELLM_RUNTIME_IMAGE AS runtime
USER root
RUN for i in 1 2 3; do \
apk add --no-cache bash openssl libssl3 tzdata python-3.13 libsndfile libatomic libevent && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
# wolfi-base ships an unprivileged `nonroot` account (UID/GID 65532) with
# /home/nonroot. We run the proxy as that user.
WORKDIR /app
ENV HOME=/home/nonroot \
PATH="/app/.venv/bin:${PATH}" \
PYTHONPATH="/app" \
PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv
COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm
COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise
COPY --from=builder --chown=nonroot:nonroot /app/gateway /app/gateway
COPY --from=builder --chown=nonroot:nonroot /app/litellm_cli /app/litellm_cli
COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh
COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma
COPY --from=builder /opt/prisma /opt/prisma
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
find /app/.venv -type d -path "*/tornado/test" -delete && \
chmod -R a+rX /opt/prisma && \
python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths" && \
python -c "import litellm, litellm.rust_bridge._native; assert litellm.__file__ == '/app/litellm/__init__.py', litellm.__file__"
USER nonroot
EXPOSE 4000/tcp
ENTRYPOINT ["sh", "-c", "exec /app/docker/component_entrypoint.sh python -m gateway.launch --workers \"${NUM_WORKERS:-1}\" \"$@\"", "--"]
CMD ["--host", "0.0.0.0", "--port", "4000"]

85
gateway/launch.py
View file

@ -22,15 +22,16 @@ import sys
from collections.abc import Callable, Mapping, Sequence
from typing import TYPE_CHECKING, Final
from typing_extensions import assert_never
from uvicorn.main import main as uvicorn_main
from litellm_cli.main import without_gc_pauses
if TYPE_CHECKING:
from fastapi import FastAPI
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings
from litellm.proxy.rust_forward.peer import ASGIApp
from litellm.proxy.rust_forward.server import PythonServer
GATEWAY_APP: Final = "gateway.launch:worker_app"
KEEPALIVE_FLAG: Final = "--timeout-keep-alive"
@ -65,17 +66,40 @@ def pool_database_url(
return start_in_container_pgbouncer(pgbouncer, upstream_url, token_auth=settings.token_auth())
def worker_app() -> "FastAPI":
def worker_app() -> "ASGIApp":
with without_gc_pauses():
from gateway.main import app
return app
from litellm.proxy.rust_forward.mode import rust_forward_mode
from litellm.proxy.rust_forward.peer import wrap
return wrap(app, rust_forward_mode(os.environ))
def _serve(argv: Sequence[str]) -> None:
uvicorn_main(tuple(argv), prog_name="uvicorn")
def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None:
def python_server(argv: Sequence[str]) -> "PythonServer":
"""What uvicorn would serve the public port with, read from the same flags and environment it reads."""
from litellm.proxy.rust_forward.server import PythonServer, Server, UvicornCliParams
from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager
params: Final = UvicornCliParams.model_validate(
uvicorn_main.make_context("uvicorn", list(uvicorn_argv(argv, os.environ))).params
)
return PythonServer(
server=Server.UVICORN,
listener=params.listener,
host=params.host,
port=params.port,
keepalive_seconds=params.timeout_keep_alive,
graceful_timeout_seconds=GracefulShutdownManager.get_timeout()
if params.timeout_graceful_shutdown is None
else params.timeout_graceful_shutdown,
trusted_proxies=params.trusted_proxies,
)
def configure_database() -> None:
from litellm.proxy.db.db_url_settings import DatabaseURLSettings
from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, export_pooled_database_url
@ -86,8 +110,59 @@ def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -
sys.exit(f"LiteLLM gateway: in-container pgbouncer could not start: {pooled_url.reason}")
if pooled_url is not None:
export_pooled_database_url(pooled_url)
def _serve_python_alone(argv: Sequence[str], serve: Callable[[Sequence[str]], None]) -> None:
configure_database()
serve(uvicorn_argv(argv, os.environ))
def _python_server_or_exit(argv: Sequence[str]) -> "PythonServer":
"""Report ``--help``, ``--version`` and a bad flag the way uvicorn's own CLI does, since it never gets to."""
import click
try:
return python_server(argv)
except click.exceptions.Exit as stopped:
sys.exit(stopped.exit_code)
except click.ClickException as error:
error.show()
sys.exit(error.exit_code)
def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None:
from pydantic import ValidationError
from litellm.proxy.rust_forward.mode import (
Mode,
Supervised,
environment_error,
rust_forward_mode,
stop_with_supervisor,
uvicorn_flags,
)
from litellm.proxy.rust_forward.supervisor import supervise
from litellm.proxy.rust_forward.supervisor.preflight import StartupError
try:
mode: Final = rust_forward_mode(os.environ)
except ValidationError as error:
sys.exit(environment_error(error))
match mode:
case Mode.DISABLED:
_serve_python_alone(argv, serve)
case Mode.SUPERVISOR:
supervised: Final = supervise(_python_server_or_exit(argv), before_start=configure_database)
if isinstance(supervised, StartupError):
_serve_python_alone(argv, serve)
else:
sys.exit(supervised)
case Supervised():
stop_with_supervisor(mode)
serve((*uvicorn_argv(argv, os.environ), *uvicorn_flags(mode)))
case _:
assert_never(mode)
if __name__ == "__main__":
main(sys.argv[1:])

23
helm/litellm-helm/.helmignore
View file

@ -1,23 +0,0 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/

9
helm/litellm-helm/Chart.lock
View file

@ -1,9 +0,0 @@
dependencies:
- name: postgresql
repository: oci://registry-1.docker.io/bitnamicharts
version: 14.3.1
- name: redis
repository: oci://registry-1.docker.io/bitnamicharts
version: 18.19.1
digest: sha256:38962e231f6596b93f82a8412bbe4cf5de696caecf5775dfbbd163383eb1c009
generated: '2026-10-30T02:13:11.505726000Z'

41
helm/litellm-helm/Chart.yaml
View file

@ -1,41 +0,0 @@
apiVersion: v2
# We can't call ourselves just "litellm" because then we couldn't publish to the
# same OCI repository as the "litellm" OCI image
name: litellm-helm
description: Call all LLM APIs using the OpenAI format
# A chart can be either an 'application' or a 'library' chart.
#
# Application charts are a collection of templates that can be packaged into versioned archives
# to be deployed.
#
# Library charts provide useful utilities or functions for the chart developer. They're included as
# a dependency of application charts to inject those utilities and functions into the rendering
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 1.1.4
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: v1.85.1
annotations:
org.opencontainers.image.source: "https://github.com/BerriAI/litellm"
org.opencontainers.image.url: "https://docs.litellm.ai/"
dependencies:
- name: "postgresql"
version: "14.3.1"
repository: oci://registry-1.docker.io/bitnamicharts
condition: db.deployStandalone
- name: redis
version: "18.19.1"
repository: oci://registry-1.docker.io/bitnamicharts
condition: redis.enabled

266
helm/litellm-helm/README.md
View file

@ -1,266 +0,0 @@
# Helm Chart for LiteLLM
> [!IMPORTANT]
> This is community maintained, Please make an issue if you run into a bug
> We recommend using [Docker or Kubernetes for production deployments](https://docs.litellm.ai/docs/proxy/prod)
## Prerequisites
- Kubernetes 1.21+
- Helm 3.8.0+
If `db.deployStandalone` is used:
- PV provisioner support in the underlying infrastructure
If `db.useStackgresOperator` is used (not yet implemented):
- The Stackgres Operator must already be installed in the Kubernetes Cluster. This chart will **not** install the operator if it is missing.
## Parameters
### LiteLLM Proxy Deployment Settings
| Name | Description | Value |
| --------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------- |
| `replicaCount` | The number of LiteLLM Proxy pods to be deployed | `1` |
| `masterkeySecretName` | The name of the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use the generated secret name. | N/A |
| `masterkeySecretKey` | The key within the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use `masterkey` as the key. | N/A |
| `masterkey` | The Master API Key for LiteLLM. If not specified, a random key in the `sk-...` format is generated on first install and reused on upgrades. | N/A |
| `environmentSecrets` | An optional array of Secret object names. The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` |
| `environmentConfigMaps` | An optional array of ConfigMap object names. The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` |
| `image.repository` | LiteLLM Proxy image repository | `ghcr.io/berriai/litellm` |
| `image.pullPolicy` | LiteLLM Proxy image pull policy | `IfNotPresent` |
| `image.tag` | Overrides the image tag whose default the latest version of LiteLLM at the time this chart was published. | `""` |
| `imagePullSecrets` | Registry credentials for the LiteLLM and initContainer images. | `[]` |
| `serviceAccount.create` | Whether or not to create a Kubernetes Service Account for this deployment. The default is `false` because LiteLLM has no need to access the Kubernetes API. | `false` |
| `service.type` | Kubernetes Service type (e.g. `LoadBalancer`, `ClusterIP`, etc.) | `ClusterIP` |
| `service.port` | TCP port that the Kubernetes Service will listen on. Also the TCP port within the Pod that the proxy will listen on. | `4000` |
| `livenessProbe.*` | Liveness probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay). | See `values.yaml` |
| `readinessProbe.*` | Readiness probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay). | See `values.yaml` |
| `startupProbe.*` | Startup probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay). | See `values.yaml` |
| `resources.*` | CPU/memory requests and limits for the LiteLLM container. Unset by default; production deployments should set 1 CPU and 4Gi of memory per worker. | `{}` |
| `service.loadBalancerClass` | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`) | `""` |
| `ingress.labels` | Additional labels for the Ingress resource | `{}` |
| `ingress.*` | See [values.yaml](./values.yaml) for example settings | N/A |
| `proxyConfigMap.create` | When `true`, render a ConfigMap from `.Values.proxy_config` and mount it. | `true` |
| `proxyConfigMap.name` | When `create=false`, name of the existing ConfigMap to mount. | `""` |
| `proxyConfigMap.key` | Key in the ConfigMap that contains the proxy config file. | `"config.yaml"` |
| `proxy_config.*` | See [values.yaml](./values.yaml) for default settings. Rendered into the ConfigMap’s `config.yaml` only when `proxyConfigMap.create=true`. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | `N/A` |
| `extraContainers[]` | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy. |
| `pdb.enabled` | Enable a PodDisruptionBudget for the LiteLLM proxy Deployment | `false` |
| `pdb.minAvailable` | Minimum number/percentage of pods that must be available during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null` |
| `pdb.maxUnavailable` | Maximum number/percentage of pods that can be unavailable during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null` |
| `pdb.annotations` | Extra metadata annotations to add to the PDB | `{}` |
| `pdb.labels` | Extra metadata labels to add to the PDB | `{}` |
| `billingMetrics.enabled` | Enable enterprise billable-request metering. Requires an enterprise license. | `false` |
| `billingMetrics.endpoint` | Collector that the billable-request counter is pushed to. | `https://telemetry.litellm.ai` |
| `billingMetrics.tokenUrl` | Cognito OAuth2 client-credentials token endpoint. | `https://litellm-prod.auth.us-west-1.amazoncognito.com/oauth2/token` |
| `billingMetrics.secretName` | Name of an existing Secret with `client-id` and `client-secret` keys. | `litellm-billing-metrics` |
| `billingMetrics.exportIntervalMs` | How often the counter is pushed, in milliseconds. The proxy defaults to `60000` when unset. | `""` |
#### Example `proxy_config` ConfigMap from values (default):
```
proxyConfigMap:
create: true
key: "config.yaml"
proxy_config:
general_settings:
master_key: os.environ/PROXY_MASTER_KEY
model_list: []
```
The chart ships no models, so add them under `proxy_config.model_list` or from the Admin UI with `store_model_in_db: true`
```
proxy_config:
model_list:
- model_name: gpt-5.6
litellm_params:
model: openai/gpt-5.6
api_key: os.environ/OPENAI_API_KEY
```
#### Example using existing `proxyConfigMap` instead of creating it:
```
proxyConfigMap:
create: false
name: my-litellm-config
key: config.yaml
# proxy_config is ignored in this mode
```
#### Example `environmentSecrets` Secret
```
apiVersion: v1
kind: Secret
metadata:
name: litellm-envsecrets
data:
AZURE_OPENAI_API_KEY: TXlTZWN1cmVLM3k=
type: Opaque
```
#### Enterprise billable-request metering
Enterprise licenses meter billable requests by pushing a counter to LiteLLM's collector with Cognito bearer authentication. Create a Secret with `client-id` and `client-secret` keys, then enable the chart values:
```
kubectl create secret generic litellm-billing-metrics --from-literal=client-id=... --from-literal=client-secret=...
```
```
billingMetrics:
enabled: true
```
The endpoint and token URL must use HTTPS. The chart requires both URLs and
the Secret name when billing metrics are enabled
### Database Settings
| Name | Description | Value |
| ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ |
| `db.useExisting` | Use an existing Postgres database. A Kubernetes Secret object must exist that contains credentials for connecting to the database. An example secret object definition is provided below. | `false` |
| `db.endpoint` | If `db.useExisting` is `true`, this is the IP, Hostname or Service Name of the Postgres server to connect to. | `localhost` |
| `db.database` | If `db.useExisting` is `true`, the name of the existing database to connect to. | `litellm` |
| `db.url` | If `db.useExisting` is `true`, the connection url of the existing database to connect to can be overwritten with this value. | `postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME)` |
| `db.secret.name` | If `db.useExisting` is `true`, the name of the Kubernetes Secret that contains credentials. | `postgres` |
| `db.secret.usernameKey` | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the username for authenticating with the Postgres instance. | `username` |
| `db.secret.passwordKey` | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the password associates with the above user. | `password` |
| `db.username` | If `db.useExisting` is `true`, the database user as a plain value. When set, `db.secret.usernameKey` is not read. | `""` |
| `db.port` | If `db.useExisting` is `true` with token auth, the Postgres port used to build the connection URL. Password auth takes the port from `db.url`. | `5432` |
| `db.schema` | With token auth, the Postgres schema holding the LiteLLM tables, rendered as `DATABASE_SCHEMA` when the tables are not in `public`. Ignored with password auth, where the schema stays in `db.url` as `?schema=`. | `""` |
| `db.useIAMAuth` | If `db.useExisting` is `true` and `db.deployStandalone` is `false`, authenticate with an AWS RDS IAM token instead of a password. See [Token auth](#token-auth-to-an-existing-postgres-rds-iam-or-microsoft-entra-id). | `false` |
| `db.useAzureEntraAuth` | If `db.useExisting` is `true` and `db.deployStandalone` is `false`, authenticate with a Microsoft Entra ID token instead of a password. Mutually exclusive with `db.useIAMAuth`. | `false` |
| `db.useStackgresOperator` | Not yet implemented. | `false` |
| `db.deployStandalone` | Deploy a standalone, single instance deployment of Postgres, using the Bitnami postgresql chart. This is useful for getting started but doesn't provide HA or (by default) data backups. | `true` |
| `postgresql.*` | If `db.deployStandalone` is `true`, configuration passed to the Bitnami postgresql chart. See the [Bitnami Documentation](https://github.com/bitnami/charts/tree/main/bitnami/postgresql) for full configuration details. See [values.yaml](./values.yaml) for the default configuration. | See [values.yaml](./values.yaml) |
| `postgresql.auth.*` | If `db.deployStandalone` is `true`, care should be taken to ensure the default `password` and `postgres-password` values are **NOT** used. | `NoTaGrEaTpAsSwOrD` |
| `postgresql.image.*` | If `db.deployStandalone` is `true`, the image for the bundled Postgres. Pinned to a `docker.io/bitnamilegacy` build because Bitnami retired the versioned tags under `docker.io/bitnami`. | `bitnamilegacy/postgresql:16.2.0-debian-12-r6` |
| `redis.image.*` | If `redis.enabled` is `true`, the image for the bundled Redis. Pinned to a `docker.io/bitnamilegacy` build for the same reason. | `bitnamilegacy/redis:7.2.4-debian-12-r9` |
#### Bundled Postgres image
Bitnami removed the versioned tags from `docker.io/bitnami` and republished the archived builds under `docker.io/bitnamilegacy`, so the image defaults that ship inside the `postgresql` and `redis` subcharts no longer pull. The chart pins both to the `bitnamilegacy` copies of the exact builds those subchart versions were released with, which keeps the on-disk data directory layout unchanged for existing installs.
Keep `postgresql.image.tag` pinned. `docker.io/bitnami/postgresql` still publishes a floating `latest`, and pointing the bundled Postgres at a different major version starts the server against a data directory it cannot read (`database files are incompatible with server`). There is no in-place way back, so crossing a major version means dumping the database with the old image and restoring it into the new one. The chart refuses to render when the tag is empty or `latest`.
Those images no longer receive updates. For anything beyond getting started, run Postgres outside the chart and point at it with `db.useExisting`.
#### Example Postgres `db.useExisting` Secret
```yaml
apiVersion: v1
kind: Secret
metadata:
name: postgres
data:
# Password for the "postgres" user
postgres-password: <some secure password, base64 encoded>
username: litellm
password: <some secure password, base64 encoded>
type: Opaque
```
#### Token auth to an existing Postgres (RDS IAM or Microsoft Entra ID)
When the database accepts short-lived tokens instead of a password (AWS RDS or Aurora with IAM database authentication, or Azure Database for PostgreSQL with Microsoft Entra ID), there is no password to put in a Secret. Turn on `db.useIAMAuth` or `db.useAzureEntraAuth` and the chart stops referencing `db.secret.passwordKey` and `db.url`. It sets `IAM_TOKEN_DB_AUTH` (or `AZURE_POSTGRESQL_AUTH`) and emits `DATABASE_HOST`, `DATABASE_PORT`, `DATABASE_USERNAME` and `DATABASE_NAME` for the proxy, the migrations job and the collector sidecar. Each of them mints its own token at startup and builds `DATABASE_URL` from those parts, and the in-container pgbouncer (`db.connectionPool`) refreshes the token before it expires.
```yaml
db:
useExisting: true
deployStandalone: false
useIAMAuth: true
endpoint: litellm.cluster-abc123.us-west-1.rds.amazonaws.com
database: litellm
username: litellm_iam # the role granted rds_iam; no db.secret needed
```
The pod needs cloud credentials for the token: on EKS use IRSA or Pod Identity on the chart's `serviceAccount` (and `migrationJob.serviceAccountName` when the Helm hook runs before the ServiceAccount exists), on AKS use workload identity. Set `AWS_RDS_REGION` under `envVars` when the RDS hostname is not a standard regional endpoint (the token must be signed for the region the database lives in). TLS options that would otherwise go in `db.url` are set as `DATABASE_SSLMODE` and `DATABASE_SSLROOTCERT` under `envVars`. Leaving `db.username` empty reads the user from `db.secret.usernameKey` as before.
#### Examples for `environmentSecrets` and `environemntConfigMaps`
```yaml
# Use config map for not-secret configuration data
apiVersion: v1
kind: ConfigMap
metadata:
name: litellm-env-configmap
data:
SOME_KEY: someValue
ANOTHER_KEY: anotherValue
```
```yaml
# Use secrets for things which are actually secret like API keys, credentials, etc
# Base64 encode the values stored in a Kubernetes Secret: $ pbpaste | base64 | pbcopy
# The --decode flag is convenient: $ pbpaste | base64 --decode
apiVersion: v1
kind: Secret
metadata:
name: litellm-env-secret
type: Opaque
data:
SOME_PASSWORD: cDZbUGVXeU5e0ZW # base64 encoded
ANOTHER_PASSWORD: AAZbUGVXeU5e0ZB # base64 encoded
```
Source: [GitHub Gist from troyharvey](https://gist.github.com/troyharvey/4506472732157221e04c6b15e3b3f094)
### Migration Job Settings
The migration job supports both ArgoCD and Helm hooks to ensure database migrations run at the appropriate time during deployments.
| Name | Description | Value |
| -------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | ------- |
| `migrationJob.enabled` | Enable or disable the schema migration Job | `true` |
| `migrationJob.backoffLimit` | Backoff limit for Job restarts | `4` |
| `migrationJob.ttlSecondsAfterFinished` | TTL for completed migration jobs | `120` |
| `migrationJob.annotations` | Additional annotations for the migration job pod | `{}` |
| `migrationJob.extraContainers` | Additional containers to run alongside the migration job | `[]` |
| `migrationJob.hooks.argocd.enabled` | Enable ArgoCD hooks for the migration job (uses PreSync hook with BeforeHookCreation delete policy) | `true` |
| `migrationJob.hooks.helm.enabled` | Enable Helm hooks for the migration job (uses pre-install,pre-upgrade hooks with before-hook-creation delete policy) | `false` |
| `migrationJob.hooks.helm.weight` | Helm hook execution order (lower weights executed first). Optional - defaults to "1" if not specified. | N/A |
## Accessing the Admin UI
When browsing to the URL published per the settings in `ingress.*`, you will
be prompted for **Admin Configuration**. The **Proxy Endpoint** is the internal
(from the `litellm` pod's perspective) URL published by the `<RELEASE>-litellm`
Kubernetes Service. If the deployment uses the default settings for this
service, the **Proxy Endpoint** should be set to `http://<RELEASE>-litellm:4000`.
The **Proxy Key** is the value specified for `masterkey` or, if a `masterkey`
was not provided to the helm command line, the `masterkey` is a randomly
generated string in the `sk-...` format stored in the `<RELEASE>-litellm-masterkey` Kubernetes Secret.
The key is generated once on the first install; later `helm upgrade` runs reuse the
value already in that Secret, so upgrading never rotates the master key.
```bash
kubectl -n litellm get secret <RELEASE>-litellm-masterkey -o jsonpath="{.data.masterkey}"
```
## Admin UI Limitations
At the time of writing, the Admin UI is unable to add models. This is because
it would need to update the `config.yaml` file which is a exposed ConfigMap, and
therefore, read-only. This is a limitation of this helm chart, not the Admin UI
itself.
## Connect Lens
This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render
## Shutdown accounting
`GRACEFUL_SHUTDOWN_TIMEOUT` defaults to 30 seconds. Pending success callbacks share the time remaining since SIGTERM or the first `/health/drain` call. Callback flushing reserves the existing 10-second teardown headroom for cancellation, tracked accounting tasks and persistence. Exhausting this budget can discard callbacks, which is logged
Set `terminationGracePeriodSeconds` to at least `GRACEFUL_SHUTDOWN_TIMEOUT + 10`, allowing additional time for preStop hooks and database writes. This is not a hard shutdown bound or a guarantee of persistence. Database retries retain their own limits, and cancellation-resistant callbacks can remain alive. The existing preStop hook can also be followed by Uvicorn's separate connection drain, as described in `values.yaml`

BIN
helm/litellm-helm/charts/postgresql-14.3.1.tgz
View file

Binary file not shown.

BIN
helm/litellm-helm/charts/redis-18.19.1.tgz
View file

Binary file not shown.

15
helm/litellm-helm/ci/test-values.yaml
View file

@ -1,15 +0,0 @@
fullnameOverride: ""
# Disable database deployment and configuration
db:
deployStandalone: false
useExisting: false
# Test environment variables
envVars:
DD_ENV: "dev_helm"
DD_SERVICE: "litellm"
USE_DDTRACE: "true"
# Disable migration job since we're not using a database
migrationJob:
enabled: false

23
helm/litellm-helm/templates/NOTES.txt
View file

@ -1,23 +0,0 @@
1. Get the application URL by running these commands:
{{- if .Values.ingress.enabled }}
{{- range $host := .Values.ingress.hosts }}
{{- range .paths }}
http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }}
{{- end }}
{{- end }}
{{- else if contains "NodePort" .Values.service.type }}
export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "litellm.fullname" . }})
export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
echo http://$NODE_IP:$NODE_PORT
{{- else if contains "LoadBalancer" .Values.service.type }}
NOTE: It may take a few minutes for the LoadBalancer IP to be available.
You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "litellm.fullname" . }}'
export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "litellm.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
echo http://$SERVICE_IP:{{ .Values.service.port }}
{{- else if contains "ClusterIP" .Values.service.type }}
export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "litellm.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
echo "Visit http://127.0.0.1:8080 to use your application"
kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
{{- end }}
PDB: {{ if .Values.pdb.enabled }}enabled{{ else }}disabled{{ end }}. Configure via .Values.pdb.*

399
helm/litellm-helm/templates/_helpers.tpl
View file

@ -1,399 +0,0 @@
{{/*
Expand the name of the chart.
*/}}
{{- define "litellm.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "litellm.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "litellm.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Common labels
*/}}
{{- define "litellm.labels" -}}
helm.sh/chart: {{ include "litellm.chart" . }}
{{ include "litellm.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{/*
Selector labels
*/}}
{{- define "litellm.selectorLabels" -}}
app.kubernetes.io/name: {{ include "litellm.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{- define "litellm.billingMetricsEnv" -}}
- name: LITELLM_BILLING_METRICS_ENDPOINT
value: {{ required "billingMetrics.endpoint is required when billingMetrics.enabled is true" .Values.billingMetrics.endpoint | quote }}
- name: LITELLM_BILLING_METRICS_TOKEN_URL
value: {{ required "billingMetrics.tokenUrl is required when billingMetrics.enabled is true" .Values.billingMetrics.tokenUrl | quote }}
- name: LITELLM_BILLING_METRICS_CLIENT_ID
valueFrom:
secretKeyRef:
name: {{ required "billingMetrics.secretName is required when billingMetrics.enabled is true" .Values.billingMetrics.secretName }}
key: client-id
- name: LITELLM_BILLING_METRICS_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: {{ required "billingMetrics.secretName is required when billingMetrics.enabled is true" .Values.billingMetrics.secretName }}
key: client-secret
{{- with .Values.billingMetrics.exportIntervalMs }}
- name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS
value: {{ . | quote }}
{{- end }}
{{- end -}}
{{/*
Create the name of the service account to use
*/}}
{{- define "litellm.serviceAccountName" -}}
{{- if .Values.serviceAccount.create }}
{{- default (include "litellm.fullname" .) .Values.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.serviceAccount.name }}
{{- end }}
{{- end }}
{{/*
Create the service account name used by migration jobs.
When Helm hooks are enabled, pre-install/pre-upgrade hooks run before normal resources.
If this chart is creating the ServiceAccount, it is not yet available for the hook job,
so fall back to "default" (or an explicit override) to avoid a cyclic dependency.
*/}}
{{- define "litellm.migrationServiceAccountName" -}}
{{- if and .Values.migrationJob.hooks.helm.enabled .Values.serviceAccount.create }}
{{- default "default" .Values.migrationJob.serviceAccountName }}
{{- else }}
{{- include "litellm.serviceAccountName" . }}
{{- end }}
{{- end }}
{{/*
Get redis service name.
The bundled Redis subchart only serves sentinel in "replication" architecture
(it rejects standalone + sentinel outright), and in that mode the sentinel
Service is named "<release>-redis", not "<release>-redis-master".
*/}}
{{- define "litellm.redis.serviceName" -}}
{{- if .Values.redis.sentinel.enabled -}}
{{- printf "%s-%s" .Release.Name (default "redis" .Values.redis.nameOverride | trunc 63 | trimSuffix "-") -}}
{{- else -}}
{{- printf "%s-%s-master" .Release.Name (default "redis" .Values.redis.nameOverride | trunc 63 | trimSuffix "-") -}}
{{- end -}}
{{- end -}}
{{/*
Get redis service port
*/}}
{{- define "litellm.redis.port" -}}
{{- if .Values.redis.sentinel.enabled -}}
{{ .Values.redis.sentinel.service.ports.sentinel }}
{{- else -}}
{{ .Values.redis.master.service.ports.redis }}
{{- end -}}
{{- end -}}
{{/*
Reject an unpinned image tag for the bundled PostgreSQL.
A floating tag lets a chart upgrade start a newer PostgreSQL major against the
existing PersistentVolumeClaim. The server then refuses to start on a data
directory written by another major version, and the only way back is a dump
taken before the change, which by that point no longer exists.
*/}}
{{- define "litellm.validateBundledPostgresImageTag" -}}
{{- $tag := .Values.postgresql.image.tag | default "" | toString -}}
{{- $digest := .Values.postgresql.image.digest | default "" | toString -}}
{{- if and (eq $digest "") (or (eq $tag "") (eq $tag "latest")) -}}
{{- fail (printf "postgresql.image.tag must be pinned to an explicit version when db.deployStandalone is true (got %q). An unpinned tag can start a different PostgreSQL major against the existing data directory, which makes the database unreadable and is not recoverable in place. Crossing a major version requires a dump and restore." $tag) -}}
{{- end -}}
{{- end -}}
{{/*
Whether the existing Postgres is reached with a short-lived token (AWS RDS IAM
or Microsoft Entra ID) instead of a password.
*/}}
{{- define "litellm.db.tokenAuth" -}}
{{- if or .Values.db.useIAMAuth .Values.db.useAzureEntraAuth -}}true{{- end -}}
{{- end -}}
{{- define "litellm.validateDbTokenAuth" -}}
{{- if and .Values.db.useIAMAuth .Values.db.useAzureEntraAuth -}}
{{- fail "db.useIAMAuth and db.useAzureEntraAuth are mutually exclusive: the database password can only come from one token source" -}}
{{- end -}}
{{- if and (include "litellm.db.tokenAuth" .) (or (not .Values.db.useExisting) .Values.db.deployStandalone) -}}
{{- fail "db.useIAMAuth and db.useAzureEntraAuth need db.useExisting: true and db.deployStandalone: false (token auth targets an existing RDS or Azure Database for PostgreSQL; the bundled postgresql subchart only supports password auth)" -}}
{{- end -}}
{{- end -}}
{{/*
Database env for an existing Postgres, shared by the proxy container, the
collector sidecar and the migrations Job. Under token auth every entrypoint
mints its own token and assembles DATABASE_URL itself, so db.url is left out.
*/}}
{{- define "litellm.existingDbEnv" -}}
{{- $tokenAuth := include "litellm.db.tokenAuth" . -}}
- name: DATABASE_USERNAME
{{- if .Values.db.username }}
value: {{ .Values.db.username | quote }}
{{- else }}
valueFrom:
secretKeyRef:
name: {{ .Values.db.secret.name }}
key: {{ .Values.db.secret.usernameKey }}
{{- end }}
{{- if .Values.db.useIAMAuth }}
- name: IAM_TOKEN_DB_AUTH
value: "true"
{{- else if .Values.db.useAzureEntraAuth }}
- name: AZURE_POSTGRESQL_AUTH
value: "true"
{{- else }}
- name: DATABASE_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.db.secret.name }}
key: {{ .Values.db.secret.passwordKey }}
{{- end }}
- name: DATABASE_HOST
{{- if .Values.db.secret.endpointKey }}
valueFrom:
secretKeyRef:
name: {{ .Values.db.secret.name }}
key: {{ .Values.db.secret.endpointKey }}
{{- else }}
value: {{ .Values.db.endpoint }}
{{- end }}
{{- if $tokenAuth }}
- name: DATABASE_PORT
value: {{ .Values.db.port | quote }}
{{- end }}
- name: DATABASE_NAME
value: {{ .Values.db.database }}
{{- if and $tokenAuth .Values.db.schema }}
- name: DATABASE_SCHEMA
value: {{ .Values.db.schema | quote }}
{{- end }}
{{- if not $tokenAuth }}
- name: DATABASE_URL
value: {{ .Values.db.url | quote }}
{{- end }}
{{- end -}}
{{/*
Environment shared by the proxy container and the opt-in collector sidecar:
database, pgbouncer, master key, redis, user envVars. Both containers must see
the same DATABASE_URL and REDIS_* so the sidecar reaches the pod's pgbouncer
and the same spend transaction buffer.
*/}}
{{- define "litellm.proxyEnv" -}}
{{- include "litellm.validateDbTokenAuth" . -}}
- name: HOST
value: "{{ .Values.listen | default "0.0.0.0" }}"
- name: PORT
value: {{ .Values.service.port | quote}}
{{- if .Values.db.deployStandalone }}
- name: DATABASE_USERNAME
valueFrom:
secretKeyRef:
name: {{ include "litellm.fullname" . }}-dbcredentials
key: username
- name: DATABASE_PASSWORD
valueFrom:
secretKeyRef:
name: {{ include "litellm.fullname" . }}-dbcredentials
key: password
- name: DATABASE_HOST
value: {{ .Release.Name }}-postgresql
- name: DATABASE_NAME
value: litellm
{{- else if .Values.db.useExisting }}
{{ include "litellm.existingDbEnv" . }}
{{- end }}
{{- if and .Values.db.useExisting .Values.db.readReplicaUrl .Values.db.secret.readReplicaEndpointKey (not .Values.db.secret.readReplicaUrlKey) }}
- name: DATABASE_READER_HOST
valueFrom:
secretKeyRef:
name: {{ .Values.db.secret.name }}
key: {{ .Values.db.secret.readReplicaEndpointKey }}
{{- end }}
{{- if and .Values.db.useExisting .Values.db.secret.readReplicaUrlKey }}
- name: DATABASE_URL_READ_REPLICA
valueFrom:
secretKeyRef:
name: {{ .Values.db.secret.name }}
key: {{ .Values.db.secret.readReplicaUrlKey }}
{{- else if .Values.db.readReplicaUrl }}
- name: DATABASE_URL_READ_REPLICA
value: {{ .Values.db.readReplicaUrl | quote }}
{{- end }}
{{- if .Values.db.connectionPool.enabled }}
- name: LITELLM_PGBOUNCER_ENABLED
value: "true"
- name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS
value: {{ .Values.db.connectionPool.maxDbConnections | quote }}
- name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN
value: {{ .Values.db.connectionPool.maxClientConn | quote }}
{{- end }}
- name: PROXY_MASTER_KEY
valueFrom:
secretKeyRef:
name: {{ .Values.masterkeySecretName | default (printf "%s-masterkey" (include "litellm.fullname" .)) }}
key: {{ .Values.masterkeySecretKey | default "masterkey" }}
{{- if .Values.redis.enabled }}
- name: REDIS_HOST
value: {{ include "litellm.redis.serviceName" . }}
- name: REDIS_PORT
value: {{ include "litellm.redis.port" . | quote }}
- name: REDIS_PASSWORD
valueFrom:
secretKeyRef:
name: {{ include "redis.secretName" .Subcharts.redis }}
key: {{include "redis.secretPasswordKey" .Subcharts.redis }}
{{- end }}
{{- /*
Inject LITELLM_LOG only when envVars does not already define it.
*/}}
{{- if and .Values.logLevel (not (hasKey (default dict .Values.envVars) "LITELLM_LOG")) }}
- name: LITELLM_LOG
value: {{ .Values.logLevel | quote }}
{{- end }}
{{- if .Values.envVars }}
{{- range $key, $val := .Values.envVars }}
- name: {{ $key }}
value: {{ $val | quote }}
{{- end }}
{{- end }}
{{- with .Values.extraEnvVars }}
{{ toYaml . }}
{{- end }}
{{- if .Values.migrationJob.enabled }}
# Schema updates are owned by the dedicated migrations Job; skip
# the proxy's startup `prisma db push` so N replicas don't race
# one DB on every rollout. Placed last (after envVars and
# extraEnvVars) so this override can't be silently shadowed by a
# user-supplied DISABLE_SCHEMA_UPDATE under last-wins duplicate-env
# semantics — same pattern the migrations Job uses.
- name: DISABLE_SCHEMA_UPDATE
value: "true"
{{- end }}
{{- end -}}
{{/*
Proxy-only metering and metrics env. The collector sidecar serves no HTTP
traffic, so it gets neither.
*/}}
{{- define "litellm.proxyMetricsEnv" -}}
{{- if .Values.billingMetrics.enabled }}
{{ include "litellm.billingMetricsEnv" . }}
{{- end }}
{{- if .Values.metricsServer.enabled }}
{{- if eq (int .Values.metricsServer.port) (int .Values.service.port) }}
{{- fail "metricsServer.port must differ from service.port" }}
{{- end }}
- name: PROMETHEUS_METRICS_PORT
value: {{ .Values.metricsServer.port | quote }}
{{- end }}
{{- end -}}
{{/*
Directory of the collector's unix socket, shared between the two containers
through an emptyDir. Empty when the sidecar is off or uses 127.0.0.1 TCP.
*/}}
{{- define "litellm.collector.socketDir" -}}
{{- if and .Values.collector.enabled (hasPrefix "unix://" .Values.collector.address) -}}
{{- dir (trimPrefix "unix://" .Values.collector.address) -}}
{{- end -}}
{{- end -}}
{{- define "litellm.collectorEnv" -}}
- name: LITELLM_COLLECTOR_ENABLED
value: "true"
- name: LITELLM_COLLECTOR_ADDRESS
value: {{ .Values.collector.address | quote }}
- name: LITELLM_COLLECTOR_BUFFER_SIZE
value: {{ .Values.collector.bufferSize | quote }}
- name: LITELLM_COLLECTOR_ON_UNAVAILABLE
value: {{ .Values.collector.onUnavailable | quote }}
- name: LITELLM_COLLECTOR_DRAIN_TIMEOUT_SECONDS
value: {{ .Values.collector.drainTimeoutSeconds | quote }}
{{- end -}}
{{- define "litellm.gateway.collectorSocketDir" -}}
{{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
{{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}
{{- end -}}
{{- end -}}
{{/*
LITELLM_COLLECTOR_* env shared by the producer (gateway container) and the
consumer (collector container), so both agree on the transport and the
shutdown drain window.
*/}}
{{- define "litellm.gateway.collectorEnv" -}}
{{- with .Values.gateway.collector }}
- name: LITELLM_COLLECTOR_ENABLED
value: "true"
- name: LITELLM_COLLECTOR_ADDRESS
value: {{ .address | quote }}
- name: LITELLM_COLLECTOR_BUFFER_SIZE
value: {{ .bufferSize | quote }}
- name: LITELLM_COLLECTOR_ON_UNAVAILABLE
value: {{ .onUnavailable | quote }}
- name: LITELLM_COLLECTOR_DRAIN_TIMEOUT_SECONDS
value: {{ .drainTimeoutSeconds | quote }}
{{- end }}
{{- end -}}
{{- define "litellm.lensConnectionEnv" -}}
{{- with .Values.lensWorker }}
{{- if or .enabled (and .mode (ne .mode "disabled")) }}
{{- fail "lensWorker was removed; deploy Lens from its own chart and set lens.* instead, see the chart README" }}
{{- end }}
{{- end }}
{{- with .Values.lens }}
{{- if .enabled }}
- name: LITELLM_LENS_URL
value: {{ required "lens.url is required when lens.enabled is true" .url | quote }}
- name: LITELLM_LENS_PUBLIC_URL
value: {{ required "lens.publicUrl is required when lens.enabled is true" .publicUrl | quote }}
- name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: {{ required "lens.gatewaySecret.name is required when lens.enabled is true" .gatewaySecret.name | quote }}
key: {{ .gatewaySecret.key | quote }}
- name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: {{ required "lens.serviceTokenSecret.name is required when lens.enabled is true" .serviceTokenSecret.name | quote }}
key: {{ .serviceTokenSecret.key | quote }}
{{- end }}
{{- end }}
{{- end -}}

22
helm/litellm-helm/templates/configmap-litellm.yaml
View file

@ -1,22 +0,0 @@
{{- if .Values.proxyConfigMap.create }}
{{- $config := deepCopy .Values.proxy_config }}
{{- if and .Values.redis.enabled (dig "coordination" "enabled" true .Values.redis) }}
{{- $generalSettings := (get $config "general_settings") | default dict }}
{{- if not (hasKey $generalSettings "coordination_redis") }}
{{- $coordinationRedis := dict "host" "os.environ/REDIS_HOST" "port" "os.environ/REDIS_PORT" "password" "os.environ/REDIS_PASSWORD" }}
{{- if .Values.redis.sentinel.enabled }}
{{- $sentinelNode := list (include "litellm.redis.serviceName" .) (include "litellm.redis.port" . | int) }}
{{- $coordinationRedis = dict "sentinel_nodes" (list $sentinelNode) "service_name" (default "mymaster" .Values.redis.sentinel.masterSet) "password" "os.environ/REDIS_PASSWORD" }}
{{- end }}
{{- $_ := set $generalSettings "coordination_redis" $coordinationRedis }}
{{- $_ := set $config "general_settings" $generalSettings }}
{{- end }}
{{- end }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "litellm.fullname" . }}-config
data:
config.yaml: |
{{ $config | toYaml | indent 6 }}
{{- end }}

258
helm/litellm-helm/templates/deployment.yaml
View file

@ -1,258 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
annotations:
{{- toYaml .Values.deploymentAnnotations | nindent 4 }}
name: {{ include "litellm.fullname" . }}
labels:
{{- include "litellm.labels" . | nindent 4 }}
{{- if .Values.deploymentLabels }}
{{- toYaml .Values.deploymentLabels | nindent 4 }}
{{- end }}
spec:
{{- if and (not .Values.keda.enabled) (not .Values.autoscaling.enabled) }}
replicas: {{ .Values.replicaCount }}
{{- end }}
{{- with .Values.strategy }}
strategy:
{{- toYaml . | nindent 4 }}
{{- end }}
selector:
matchLabels:
{{- include "litellm.selectorLabels" . | nindent 6 }}
{{- if .Values.deploymentMinReadySeconds }}
minReadySeconds: {{ .Values.deploymentMinReadySeconds }}
{{- end }}
template:
metadata:
annotations:
{{- if .Values.proxyConfigMap.create }}
checksum/config: {{ include (print $.Template.BasePath "/configmap-litellm.yaml") . | sha256sum }}
{{- end }}
{{- with .Values.podAnnotations }}
{{- tpl (toYaml .) $ | nindent 8 }}
{{- end }}
labels:
{{- include "litellm.labels" . | nindent 8 }}
{{- with .Values.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
{{- with .Values.extraInitContainers }}
initContainers:
{{- tpl (toYaml .) $ | nindent 8 }}
{{- end }}
containers:
- name: {{ include "litellm.name" . }}
securityContext:
{{- toYaml .Values.securityContext | nindent 12 }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
env:
{{- include "litellm.lensConnectionEnv" . | nindent 12 }}
{{- include "litellm.proxyEnv" . | nindent 12 }}
{{- if .Values.liteadmin.enabled }}
- name: LITELLM_ADMIN_AGENT_URL
value: {{ printf "http://%s-liteadmin:10000" (include "litellm.fullname" . | trunc 53 | trimSuffix "-") | quote }}
- name: ADMIN_AGENT_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: {{ required "liteadmin.existingSecret is required" .Values.liteadmin.existingSecret }}
key: ADMIN_AGENT_SERVICE_TOKEN
{{- if not (hasKey (default dict .Values.envVars) "PROXY_BASE_URL") }}
- name: PROXY_BASE_URL
value: {{ required "liteadmin.gatewayUrl is required" .Values.liteadmin.gatewayUrl | quote }}
{{- end }}
{{- end }}
{{- include "litellm.proxyMetricsEnv" . | nindent 12 }}
{{- if .Values.collector.enabled }}
{{- include "litellm.collectorEnv" . | nindent 12 }}
{{- end }}
envFrom:
{{- range .Values.environmentSecrets }}
- secretRef:
name: {{ . }}
{{- end }}
{{- range .Values.environmentConfigMaps }}
- configMapRef:
name: {{ . }}
{{- end }}
{{- if .Values.command }}
command: {{ toYaml .Values.command | nindent 12 }}
{{- end }}
{{- if .Values.args }}
args: {{ toYaml .Values.args | nindent 12 }}
{{- else }}
args:
- --config
- /etc/litellm/config.yaml
{{ if .Values.numWorkers }}
- --num_workers
- {{ .Values.numWorkers | quote }}
{{- end }}
{{- end }}
ports:
- name: http
containerPort: {{ .Values.service.port }}
protocol: TCP
{{- if .Values.metricsServer.enabled }}
- name: metrics
containerPort: {{ .Values.metricsServer.port }}
protocol: TCP
{{- end }}
livenessProbe:
httpGet:
path: {{ .Values.livenessProbe.path | quote }}
port: "http"
initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
successThreshold: {{ .Values.livenessProbe.successThreshold }}
failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
readinessProbe:
httpGet:
path: {{ .Values.readinessProbe.path | quote }}
port: "http"
initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
successThreshold: {{ .Values.readinessProbe.successThreshold }}
failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
startupProbe:
httpGet:
path: {{ .Values.startupProbe.path | quote }}
port: "http"
initialDelaySeconds: {{ .Values.startupProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.startupProbe.periodSeconds }}
timeoutSeconds: {{ .Values.startupProbe.timeoutSeconds }}
successThreshold: {{ .Values.startupProbe.successThreshold }}
failureThreshold: {{ .Values.startupProbe.failureThreshold }}
resources:
{{- toYaml .Values.resources | nindent 12 }}
volumeMounts:
- name: litellm-config
mountPath: /etc/litellm/config.yaml
subPath: config.yaml
{{ if .Values.securityContext.readOnlyRootFilesystem }}
- name: tmp
mountPath: /tmp
- name: cache
mountPath: /.cache
- name: npm
mountPath: /.npm
{{- end }}
{{- if include "litellm.collector.socketDir" . }}
- name: collector-socket
mountPath: {{ include "litellm.collector.socketDir" . }}
{{- end }}
{{- with .Values.volumeMounts }}
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.lifecycle }}
lifecycle:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- if .Values.collector.enabled }}
- name: {{ include "litellm.name" . }}-collector
securityContext:
{{- toYaml .Values.securityContext | nindent 12 }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
command: {{ toYaml .Values.collector.command | nindent 12 }}
env:
{{- include "litellm.proxyEnv" . | nindent 12 }}
{{- include "litellm.collectorEnv" . | nindent 12 }}
- name: LITELLM_JOB_ROLE
value: collector
{{- if not (hasKey (default dict .Values.envVars) "CONFIG_FILE_PATH") }}
- name: CONFIG_FILE_PATH
value: /etc/litellm/config.yaml
{{- end }}
envFrom:
{{- range .Values.environmentSecrets }}
- secretRef:
name: {{ . }}
{{- end }}
{{- range .Values.environmentConfigMaps }}
- configMapRef:
name: {{ . }}
{{- end }}
resources:
{{- toYaml .Values.collector.resources | nindent 12 }}
volumeMounts:
- name: litellm-config
mountPath: /etc/litellm/config.yaml
subPath: config.yaml
{{- if include "litellm.collector.socketDir" . }}
- name: collector-socket
mountPath: {{ include "litellm.collector.socketDir" . }}
{{- end }}
{{ if .Values.securityContext.readOnlyRootFilesystem }}
- name: tmp
mountPath: /tmp
- name: cache
mountPath: /.cache
- name: npm
mountPath: /.npm
{{- end }}
{{- with .Values.volumeMounts }}
{{- toYaml . | nindent 12 }}
{{- end }}
{{- end }}
{{- with .Values.extraContainers }}
{{- tpl (toYaml .) $ | nindent 8 }}
{{- end }}
volumes:
{{ if .Values.securityContext.readOnlyRootFilesystem }}
- name: tmp
emptyDir:
sizeLimit: 500Mi
- name: cache
emptyDir:
sizeLimit: 500Mi
- name: npm
emptyDir:
sizeLimit: 500Mi
{{- end }}
- name: litellm-config
configMap:
{{- if .Values.proxyConfigMap.create }}
name: {{ include "litellm.fullname" . }}-config
{{- else }}
name: {{ .Values.proxyConfigMap.name }}
{{- end }}
items:
- key: {{ .Values.proxyConfigMap.key | default "config.yaml" }}
path: "config.yaml"
{{- if include "litellm.collector.socketDir" . }}
- name: collector-socket
emptyDir:
sizeLimit: 1Mi
{{- end }}
{{- with .Values.volumes }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds | default 90 }}
{{- if .Values.topologySpreadConstraints }}
topologySpreadConstraints:
{{- toYaml .Values.topologySpreadConstraints | nindent 8 }}
{{- end }}

6
helm/litellm-helm/templates/extra-resources.yaml
View file

@ -1,6 +0,0 @@
{{- if .Values.extraResources }}
{{- range .Values.extraResources }}
---
{{ toYaml . | nindent 0 }}
{{- end }}
{{- end }}

64
helm/litellm-helm/templates/hpa.yaml
View file

@ -1,64 +0,0 @@
{{- if .Values.autoscaling.enabled }}
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: {{ include "litellm.fullname" . }}
labels:
{{- include "litellm.labels" . | nindent 4 }}
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: {{ include "litellm.fullname" . }}
minReplicas: {{ .Values.autoscaling.minReplicas }}
maxReplicas: {{ .Values.autoscaling.maxReplicas }}
{{- if .Values.autoscaling.behavior }}
behavior:
{{- toYaml .Values.autoscaling.behavior | nindent 4 }}
{{- end }}
metrics:
{{- if .Values.autoscaling.targetCPUUtilizationPercentage }}
{{- if and .Values.collector.enabled .Values.collector.scaleOnProxyContainerCpu }}
- type: ContainerResource
containerResource:
name: cpu
container: {{ include "litellm.name" . }}
target:
type: Utilization
averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }}
{{- else }}
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }}
{{- end }}
{{- end }}
{{- if .Values.autoscaling.targetMemoryUtilizationPercentage }}
- type: Resource
resource:
name: memory
target:
type: Utilization
averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }}
{{- end }}
{{- with .Values.autoscaling.targetRequestsPerSecond }}
- type: Pods
pods:
metric:
name: litellm_requests_per_second
target:
type: AverageValue
averageValue: {{ toJson . | trimAll "\"" | quote }}
{{- end }}
{{- with .Values.autoscaling.targetTokensPerSecond }}
- type: Pods
pods:
metric:
name: litellm_tokens_per_second
target:
type: AverageValue
averageValue: {{ toJson . | trimAll "\"" | quote }}
{{- end }}
{{- end }}

78
helm/litellm-helm/templates/ingress.yaml
View file

@ -1,78 +0,0 @@
{{- if .Values.ingress.enabled -}}
{{- $fullName := include "litellm.fullname" . -}}
{{- $svcPort := .Values.service.port -}}
{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
{{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
{{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}
{{- end }}
{{- end }}
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1beta1
{{- else -}}
apiVersion: extensions/v1beta1
{{- end }}
kind: Ingress
metadata:
name: {{ $fullName }}
labels:
{{- include "litellm.labels" . | nindent 4 }}
{{- with .Values.ingress.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.ingress.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
ingressClassName: {{ .Values.ingress.className }}
{{- end }}
{{- if .Values.ingress.tls }}
tls:
{{- range .Values.ingress.tls }}
- hosts:
{{- range .hosts }}
- {{ . | quote }}
{{- end }}
secretName: {{ .secretName }}
{{- end }}
{{- end }}
rules:
{{- range .Values.ingress.hosts }}
- host: {{ .host | quote }}
http:
paths:
{{- if and $.Values.lens.enabled $.Values.lens.service.name }}
- path: /lens-ingest
pathType: Prefix
backend:
{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
service:
name: {{ $.Values.lens.service.name }}
port:
number: {{ $.Values.lens.service.port }}
{{- else }}
serviceName: {{ $.Values.lens.service.name }}
servicePort: {{ $.Values.lens.service.port }}
{{- end }}
{{- end }}
{{- range .paths }}
- path: {{ .path }}
{{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
pathType: {{ .pathType }}
{{- end }}
backend:
{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
service:
name: {{ $fullName }}
port:
number: {{ $svcPort }}
{{- else }}
serviceName: {{ $fullName }}
servicePort: {{ $svcPort }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}

58
helm/litellm-helm/templates/keda.yaml
View file

@ -1,58 +0,0 @@
{{- if and .Values.keda.enabled (not .Values.autoscaling.enabled) }}
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
name: {{ include "litellm.fullname" . }}
labels:
{{- include "litellm.labels" . | nindent 4 }}
{{- if .Values.keda.scaledObject.annotations }}
annotations: {{ toYaml .Values.keda.scaledObject.annotations | nindent 4 }}
{{- end }}
spec:
scaleTargetRef:
name: {{ include "litellm.fullname" . }}
pollingInterval: {{ .Values.keda.pollingInterval }}
cooldownPeriod: {{ .Values.keda.cooldownPeriod }}
minReplicaCount: {{ .Values.keda.minReplicas }}
maxReplicaCount: {{ .Values.keda.maxReplicas }}
{{- with .Values.keda.fallback }}
fallback:
failureThreshold: {{ .failureThreshold | default 3 }}
replicas: {{ .replicas | default $.Values.keda.maxReplicas }}
{{- end }}
triggers:
{{- with .Values.keda.triggers }}
{{- toYaml . | nindent 2 }}
{{- end }}
{{- $prom := .Values.keda.prometheus }}
{{- if or $prom.requestsPerSecond $prom.tokensPerSecond }}
{{- if not $prom.serverAddress }}
{{- fail "keda.prometheus.serverAddress is required when keda.prometheus.requestsPerSecond or tokensPerSecond is set" }}
{{- end }}
{{- $selector := printf "namespace=%q,job=%q" .Release.Namespace (printf "%s%s" (include "litellm.fullname" .) (ternary "-metrics" "" .Values.metricsServer.enabled)) }}
{{- with $prom.requestsPerSecond }}
- type: prometheus
metadata:
serverAddress: {{ $prom.serverAddress | quote }}
threshold: {{ toJson . | trimAll "\"" | quote }}
query: {{ printf "sum(rate(litellm_proxy_total_requests_metric_total{%s}[1m]))" $selector | quote }}
{{- end }}
{{- with $prom.tokensPerSecond }}
- type: prometheus
metadata:
serverAddress: {{ $prom.serverAddress | quote }}
threshold: {{ toJson . | trimAll "\"" | quote }}
query: {{ printf "sum(rate(litellm_total_tokens_metric_total{%s}[1m]))" $selector | quote }}
{{- end }}
{{- end }}
advanced:
restoreToOriginalReplicaCount: {{ .Values.keda.restoreToOriginalReplicaCount }}
{{- if .Values.keda.behavior }}
horizontalPodAutoscalerConfig:
behavior:
{{- with .Values.keda.behavior }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}
{{- end }}

122
helm/litellm-helm/templates/migrations-job.yaml
View file

@ -1,122 +0,0 @@
{{- if .Values.migrationJob.enabled }}
# This job runs the Prisma migrations for the LiteLLM DB.
apiVersion: batch/v1
kind: Job
metadata:
name: {{ include "litellm.fullname" . }}-migrations
labels:
{{- include "litellm.labels" . | nindent 4 }}
{{- with .Values.migrationJob.jobLabels }}
{{- toYaml . | nindent 4 }}
{{- end }}
annotations:
{{- if .Values.migrationJob.hooks.argocd.enabled }}
argocd.argoproj.io/hook: PreSync
argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
{{- end }}
{{- if .Values.migrationJob.hooks.helm.enabled }}
helm.sh/hook: "pre-install,pre-upgrade"
helm.sh/hook-delete-policy: "before-hook-creation"
helm.sh/hook-weight: {{ .Values.migrationJob.hooks.helm.weight | default "1" | quote }}
{{- end }}
checksum/config: {{ toYaml .Values | sha256sum }}
{{- with .Values.migrationJob.jobAnnotations }}
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
template:
metadata:
labels:
{{- include "litellm.labels" . | nindent 8 }}
{{- with .Values.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.migrationJob.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
annotations:
{{- with .Values.migrationJob.annotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "litellm.migrationServiceAccountName" . }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
{{- with .Values.migrationJob.extraInitContainers }}
initContainers:
{{- tpl (toYaml .) $ | nindent 8 }}
{{- end }}
containers:
- name: prisma-migrations
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
securityContext:
{{- toYaml .Values.securityContext | nindent 12 }}
{{- if .Values.migrationJob.command }}
command: {{ toYaml .Values.migrationJob.command | nindent 12 }}
{{- else }}
command: ["python", "litellm/proxy/prisma_migration.py"]
{{- end }}
{{- if .Values.migrationJob.args }}
args: {{ toYaml .Values.migrationJob.args | nindent 12 }}
{{- else if .Values.migrationJob.command }}
args: []
{{- end }}
workingDir: "/app"
env:
{{- include "litellm.validateDbTokenAuth" . -}}
{{- if .Values.db.useExisting }}
{{- include "litellm.existingDbEnv" . | nindent 12 }}
{{- else if .Values.db.deployStandalone }}
- name: DATABASE_URL
value: postgresql://{{ .Values.postgresql.auth.username }}:{{ .Values.postgresql.auth.password }}@{{ .Release.Name }}-postgresql/{{ .Values.postgresql.auth.database }}
{{- end }}
{{- if .Values.envVars }}
{{- range $key, $val := .Values.envVars }}
- name: {{ $key }}
value: {{ $val | quote }}
{{- end }}
{{- end }}
{{- with .Values.extraEnvVars }}
{{- toYaml . | nindent 12 }}
{{- end }}
- name: DISABLE_SCHEMA_UPDATE
value: "false" # always run the migration from the Helm PreSync hook, override the value set
{{- with .Values.volumeMounts }}
volumeMounts:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.migrationJob.resources }}
resources:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.migrationJob.extraContainers }}
{{- tpl (toYaml .) $ | nindent 8 }}
{{- end }}
{{- with .Values.volumes }}
volumes:
{{- toYaml . | nindent 8 }}
{{- end }}
restartPolicy: OnFailure
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
ttlSecondsAfterFinished: {{ .Values.migrationJob.ttlSecondsAfterFinished }}
backoffLimit: {{ .Values.migrationJob.backoffLimit }}
{{- with .Values.migrationJob.activeDeadlineSeconds }}
activeDeadlineSeconds: {{ . }}
{{- end }}
{{- end }}

33
helm/litellm-helm/templates/poddisruptionbudget.yaml
View file

@ -1,33 +0,0 @@
{{- /*
PodDisruptionBudget for LiteLLM proxy
Controlled via .Values.pdb.enabled and .Values.pdb.{minAvailable|maxUnavailable}
Only one of minAvailable / maxUnavailable should be set. If both are set, minAvailable wins.
*/ -}}
{{- if .Values.pdb.enabled }}
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: {{ include "litellm.fullname" . }}
labels:
{{- include "litellm.labels" . | nindent 4 }}
{{- with .Values.pdb.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.pdb.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
selector:
matchLabels:
{{- /* Match the Deployment selector to target the same pod set */ -}}
{{- include "litellm.selectorLabels" . | nindent 6 }}
{{- if .Values.pdb.minAvailable }}
minAvailable: {{ .Values.pdb.minAvailable }}
{{- else if .Values.pdb.maxUnavailable }}
maxUnavailable: {{ .Values.pdb.maxUnavailable }}
{{- else }}
# Safe default if enabled but not configured
maxUnavailable: 1
{{- end }}
{{- end }}

13
helm/litellm-helm/templates/secret-dbcredentials.yaml
View file

@ -1,13 +0,0 @@
{{- if .Values.db.deployStandalone -}}
{{- include "litellm.validateBundledPostgresImageTag" . -}}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "litellm.fullname" . }}-dbcredentials
data:
# Password for the "postgres" user
postgres-password: {{ ( index .Values.postgresql.auth "postgres-password") | default "litellm" | b64enc }}
username: {{ .Values.postgresql.auth.username | default "litellm" | b64enc }}
password: {{ .Values.postgresql.auth.password | default "litellm" | b64enc }}
type: Opaque
{{- end -}}

12
helm/litellm-helm/templates/secret-masterkey.yaml
View file

@ -1,12 +0,0 @@
{{- if not .Values.masterkeySecretName }}
{{- $secretName := printf "%s-masterkey" (include "litellm.fullname" .) }}
{{- $existing := lookup "v1" "Secret" .Release.Namespace $secretName }}
{{- $masterkey := .Values.masterkey | default (dig "data" "masterkey" "" $existing | b64dec) | default (printf "sk-%s" (randAlphaNum 18)) }}
apiVersion: v1
kind: Secret
metadata:
name: {{ $secretName }}
data:
masterkey: {{ $masterkey | b64enc }}
type: Opaque
{{- end }}

17
helm/litellm-helm/templates/service-metrics.yaml
View file

@ -1,17 +0,0 @@
{{- if .Values.metricsServer.enabled }}
apiVersion: v1
kind: Service
metadata:
name: {{ include "litellm.fullname" . }}-metrics
labels:
{{- include "litellm.labels" . | nindent 4 }}
spec:
type: ClusterIP
ports:
- port: {{ .Values.metricsServer.port }}
targetPort: metrics
protocol: TCP
name: metrics
selector:
{{- include "litellm.selectorLabels" . | nindent 4 }}
{{- end }}

22
helm/litellm-helm/templates/service.yaml
View file

@ -1,22 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: {{ include "litellm.fullname" . }}
{{- with .Values.service.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
labels:
{{- include "litellm.labels" . | nindent 4 }}
spec:
type: {{ .Values.service.type }}
{{- if and (eq .Values.service.type "LoadBalancer") .Values.service.loadBalancerClass }}
loadBalancerClass: {{ .Values.service.loadBalancerClass }}
{{- end }}
ports:
- port: {{ .Values.service.port }}
targetPort: http
protocol: TCP
name: http
selector:
{{- include "litellm.selectorLabels" . | nindent 4 }}

13
helm/litellm-helm/templates/serviceaccount.yaml
View file

@ -1,13 +0,0 @@
{{- if .Values.serviceAccount.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "litellm.serviceAccountName" . }}
labels:
{{- include "litellm.labels" . | nindent 4 }}
{{- with .Values.serviceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
automountServiceAccountToken: {{ .Values.serviceAccount.automount }}
{{- end }}

39
helm/litellm-helm/templates/servicemonitor.yaml
View file

@ -1,39 +0,0 @@
{{- with .Values.serviceMonitor }}
{{- if and (eq .enabled true) }}
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: {{ include "litellm.fullname" $ }}
labels:
{{- include "litellm.labels" $ | nindent 4 }}
{{- if .labels }}
{{- toYaml .labels | nindent 4 }}
{{- end }}
{{- if .annotations }}
annotations:
{{- toYaml .annotations | nindent 4 }}
{{- end }}
spec:
selector:
matchLabels:
{{- include "litellm.selectorLabels" $ | nindent 6 }}
namespaceSelector:
matchNames:
# if not set, use the release namespace
{{- if not .namespaceSelector.matchNames }}
- {{ $.Release.Namespace | quote }}
{{- else }}
{{- toYaml .namespaceSelector.matchNames | nindent 4 }}
{{- end }}
endpoints:
- port: {{ ternary "metrics" "http" $.Values.metricsServer.enabled }}
path: /metrics/
interval: {{ .interval }}
scrapeTimeout: {{ .scrapeTimeout }}
scheme: http
{{- if .relabelings }}
relabelings:
{{- toYaml .relabelings | nindent 4 }}
{{- end }}
{{- end }}
{{- end }}

25
helm/litellm-helm/templates/tests/test-connection.yaml
View file

@ -1,25 +0,0 @@
apiVersion: v1
kind: Pod
metadata:
name: "{{ include "litellm.fullname" . }}-test-connection"
labels:
{{- include "litellm.labels" . | nindent 4 }}
annotations:
"helm.sh/hook": test
spec:
containers:
- name: wget
image: busybox
command: ['sh', '-c']
args:
- |
# Wait for a bit to allow the service to be ready
sleep 10
# Try multiple times with a delay between attempts
for i in $(seq 1 30); do
wget -T 5 "{{ include "litellm.fullname" . }}:{{ .Values.service.port }}/health/readiness" && exit 0
echo "Attempt $i failed, waiting..."
sleep 2
done
exit 1
restartPolicy: Never

43
helm/litellm-helm/templates/tests/test-env-vars.yaml
View file

@ -1,43 +0,0 @@
apiVersion: v1
kind: Pod
metadata:
name: "{{ include "litellm.fullname" . }}-env-test"
labels:
{{- include "litellm.labels" . | nindent 4 }}
annotations:
"helm.sh/hook": test
spec:
containers:
- name: test
image: busybox
command: ['sh', '-c']
args:
- |
# Test DD_ENV
if [ "$DD_ENV" != "dev_helm" ]; then
echo "❌ Environment variable DD_ENV mismatch. Expected: dev_helm, Got: $DD_ENV"
exit 1
fi
echo "✅ Environment variable DD_ENV matches expected value: $DD_ENV"
# Test DD_SERVICE
if [ "$DD_SERVICE" != "litellm" ]; then
echo "❌ Environment variable DD_SERVICE mismatch. Expected: litellm, Got: $DD_SERVICE"
exit 1
fi
echo "✅ Environment variable DD_SERVICE matches expected value: $DD_SERVICE"
# Test USE_DDTRACE
if [ "$USE_DDTRACE" != "true" ]; then
echo "❌ Environment variable USE_DDTRACE mismatch. Expected: true, Got: $USE_DDTRACE"
exit 1
fi
echo "✅ Environment variable USE_DDTRACE matches expected value: $USE_DDTRACE"
env:
- name: DD_ENV
value: {{ .Values.envVars.DD_ENV | quote }}
- name: DD_SERVICE
value: {{ .Values.envVars.DD_SERVICE | quote }}
- name: USE_DDTRACE
value: {{ .Values.envVars.USE_DDTRACE | quote }}
restartPolicy: Never

152
helm/litellm-helm/templates/tests/test-servicemonitor.yaml
View file

@ -1,152 +0,0 @@
{{- if .Values.serviceMonitor.enabled }}
apiVersion: v1
kind: Pod
metadata:
name: "{{ include "litellm.fullname" . }}-test-servicemonitor"
labels:
{{- include "litellm.labels" . | nindent 4 }}
annotations:
"helm.sh/hook": test
spec:
containers:
- name: test
image: docker.io/bitnamilegacy/kubectl:1.29.2-debian-12-r3
command: ['sh', '-c']
args:
- |
set -e
echo "🔍 Testing ServiceMonitor configuration..."
# Check if ServiceMonitor exists
if ! kubectl get servicemonitor {{ include "litellm.fullname" . }} -n {{ .Release.Namespace }} &>/dev/null; then
echo "❌ ServiceMonitor not found"
exit 1
fi
echo "✅ ServiceMonitor exists"
# Get ServiceMonitor YAML
SM=$(kubectl get servicemonitor {{ include "litellm.fullname" . }} -n {{ .Release.Namespace }} -o yaml)
# Test endpoint configuration
ENDPOINT_PORT=$(echo "$SM" | grep -A 5 "endpoints:" | grep "port:" | awk '{print $2}')
if [ "$ENDPOINT_PORT" != "http" ]; then
echo "❌ Endpoint port mismatch. Expected: http, Got: $ENDPOINT_PORT"
exit 1
fi
echo "✅ Endpoint port is correctly set to: $ENDPOINT_PORT"
# Test endpoint path
ENDPOINT_PATH=$(echo "$SM" | grep -A 5 "endpoints:" | grep "path:" | awk '{print $2}')
if [ "$ENDPOINT_PATH" != "/metrics/" ]; then
echo "❌ Endpoint path mismatch. Expected: /metrics/, Got: $ENDPOINT_PATH"
exit 1
fi
echo "✅ Endpoint path is correctly set to: $ENDPOINT_PATH"
# Test interval
INTERVAL=$(echo "$SM" | grep "interval:" | awk '{print $2}')
if [ "$INTERVAL" != "{{ .Values.serviceMonitor.interval }}" ]; then
echo "❌ Interval mismatch. Expected: {{ .Values.serviceMonitor.interval }}, Got: $INTERVAL"
exit 1
fi
echo "✅ Interval is correctly set to: $INTERVAL"
# Test scrapeTimeout
TIMEOUT=$(echo "$SM" | grep "scrapeTimeout:" | awk '{print $2}')
if [ "$TIMEOUT" != "{{ .Values.serviceMonitor.scrapeTimeout }}" ]; then
echo "❌ ScrapeTimeout mismatch. Expected: {{ .Values.serviceMonitor.scrapeTimeout }}, Got: $TIMEOUT"
exit 1
fi
echo "✅ ScrapeTimeout is correctly set to: $TIMEOUT"
# Test scheme
SCHEME=$(echo "$SM" | grep "scheme:" | awk '{print $2}')
if [ "$SCHEME" != "http" ]; then
echo "❌ Scheme mismatch. Expected: http, Got: $SCHEME"
exit 1
fi
echo "✅ Scheme is correctly set to: $SCHEME"
{{- if .Values.serviceMonitor.labels }}
# Test custom labels
echo "🔍 Checking custom labels..."
{{- range $key, $value := .Values.serviceMonitor.labels }}
LABEL_VALUE=$(echo "$SM" | grep -A 20 "metadata:" | grep "{{ $key }}:" | awk '{print $2}')
if [ "$LABEL_VALUE" != "{{ $value }}" ]; then
echo "❌ Label {{ $key }} mismatch. Expected: {{ $value }}, Got: $LABEL_VALUE"
exit 1
fi
echo "✅ Label {{ $key }} is correctly set to: {{ $value }}"
{{- end }}
{{- end }}
{{- if .Values.serviceMonitor.annotations }}
# Test annotations
echo "🔍 Checking annotations..."
{{- range $key, $value := .Values.serviceMonitor.annotations }}
ANNOTATION_VALUE=$(echo "$SM" | grep -A 10 "annotations:" | grep "{{ $key }}:" | awk '{print $2}')
if [ "$ANNOTATION_VALUE" != "{{ $value }}" ]; then
echo "❌ Annotation {{ $key }} mismatch. Expected: {{ $value }}, Got: $ANNOTATION_VALUE"
exit 1
fi
echo "✅ Annotation {{ $key }} is correctly set to: {{ $value }}"
{{- end }}
{{- end }}
{{- if .Values.serviceMonitor.namespaceSelector.matchNames }}
# Test namespace selector
echo "🔍 Checking namespace selector..."
{{- range .Values.serviceMonitor.namespaceSelector.matchNames }}
if ! echo "$SM" | grep -A 5 "namespaceSelector:" | grep -q "{{ . }}"; then
echo "❌ Namespace {{ . }} not found in namespaceSelector"
exit 1
fi
echo "✅ Namespace {{ . }} found in namespaceSelector"
{{- end }}
{{- else }}
# Test default namespace selector (should be release namespace)
if ! echo "$SM" | grep -A 5 "namespaceSelector:" | grep -q "{{ .Release.Namespace }}"; then
echo "❌ Release namespace {{ .Release.Namespace }} not found in namespaceSelector"
exit 1
fi
echo "✅ Default namespace selector set to release namespace: {{ .Release.Namespace }}"
{{- end }}
{{- if .Values.serviceMonitor.relabelings }}
# Test relabelings
echo "🔍 Checking relabelings configuration..."
if ! echo "$SM" | grep -q "relabelings:"; then
echo "❌ Relabelings section not found"
exit 1
fi
echo "✅ Relabelings section exists"
{{- range .Values.serviceMonitor.relabelings }}
{{- if .targetLabel }}
if ! echo "$SM" | grep -A 50 "relabelings:" | grep -q "targetLabel: {{ .targetLabel }}"; then
echo "❌ Relabeling targetLabel {{ .targetLabel }} not found"
exit 1
fi
echo "✅ Relabeling targetLabel {{ .targetLabel }} found"
{{- end }}
{{- if .action }}
if ! echo "$SM" | grep -A 50 "relabelings:" | grep -q "action: {{ .action }}"; then
echo "❌ Relabeling action {{ .action }} not found"
exit 1
fi
echo "✅ Relabeling action {{ .action }} found"
{{- end }}
{{- end }}
{{- end }}
# Test selector labels match the service
echo "🔍 Checking selector labels match service..."
SVC_LABELS=$(kubectl get svc {{ include "litellm.fullname" . }} -n {{ .Release.Namespace }} -o jsonpath='{.metadata.labels}')
echo "Service labels: $SVC_LABELS"
echo "✅ Selector labels validation passed"
echo ""
echo "🎉 All ServiceMonitor tests passed successfully!"
serviceAccountName: {{ include "litellm.serviceAccountName" . }}
restartPolicy: Never
{{- end }}

100
helm/litellm-helm/tests/billing_metrics_tests.yaml
View file

@ -1,100 +0,0 @@
suite: test bearer-only billingMetrics wiring
templates:
- deployment.yaml
- configmap-litellm.yaml
tests:
- it: is disabled by default
template: deployment.yaml
asserts:
- notContains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_BILLING_METRICS_ENDPOINT
- notContains:
path: spec.template.spec.volumes
content:
name: billing-metrics
- it: renders required URLs and Cognito secret references when enabled
template: deployment.yaml
set:
billingMetrics:
enabled: true
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_BILLING_METRICS_ENDPOINT
value: https://telemetry.litellm.ai
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_BILLING_METRICS_TOKEN_URL
value: https://litellm-prod.auth.us-west-1.amazoncognito.com/oauth2/token
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_BILLING_METRICS_CLIENT_ID
valueFrom:
secretKeyRef:
name: litellm-billing-metrics
key: client-id
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_BILLING_METRICS_CLIENT_SECRET
valueFrom:
secretKeyRef:
name: litellm-billing-metrics
key: client-secret
- notContains:
path: spec.template.spec.volumes
content:
name: billing-metrics
- notContains:
path: spec.template.spec.containers[0].volumeMounts
content:
name: billing-metrics
- it: renders the export interval only when set
template: deployment.yaml
set:
billingMetrics:
enabled: true
exportIntervalMs: 5000
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS
value: "5000"
- it: requires the endpoint when enabled
template: deployment.yaml
set:
billingMetrics:
enabled: true
endpoint: ""
asserts:
- failedTemplate:
errorMessage: billingMetrics.endpoint is required when billingMetrics.enabled is true
- it: requires the token URL when enabled
template: deployment.yaml
set:
billingMetrics:
enabled: true
tokenUrl: ""
asserts:
- failedTemplate:
errorMessage: billingMetrics.tokenUrl is required when billingMetrics.enabled is true
- it: requires the secret name when enabled
template: deployment.yaml
set:
billingMetrics:
enabled: true
secretName: ""
asserts:
- failedTemplate:
errorMessage: billingMetrics.secretName is required when billingMetrics.enabled is true

94
helm/litellm-helm/tests/bundled_db_images_tests.yaml
View file

@ -1,94 +0,0 @@
suite: test bundled database images
templates:
- charts/postgresql/templates/primary/statefulset.yaml
- charts/redis/templates/master/application.yaml
- charts/redis/templates/configmap.yaml
- charts/redis/templates/health-configmap.yaml
- charts/redis/templates/scripts-configmap.yaml
- charts/redis/templates/secret.yaml
- secret-dbcredentials.yaml
- templates/tests/test-servicemonitor.yaml
tests:
- it: should pull the bundled postgres from a repository that still publishes the pinned tag
template: charts/postgresql/templates/primary/statefulset.yaml
set:
db.deployStandalone: true
asserts:
- equal:
path: spec.template.spec.containers[0].image
value: docker.io/bitnamilegacy/postgresql:16.2.0-debian-12-r6
- it: should pull the bundled postgres metrics exporter from the same repository
template: charts/postgresql/templates/primary/statefulset.yaml
set:
db.deployStandalone: true
postgresql.metrics.enabled: true
asserts:
- equal:
path: spec.template.spec.containers[1].image
value: docker.io/bitnamilegacy/postgres-exporter:0.15.0-debian-12-r14
- it: should run the bundled postgres init container from the same repository
template: charts/postgresql/templates/primary/statefulset.yaml
set:
db.deployStandalone: true
postgresql.volumePermissions.enabled: true
asserts:
- equal:
path: spec.template.spec.initContainers[0].image
value: docker.io/bitnamilegacy/os-shell:12-debian-12-r16
- it: should pull the bundled redis from a repository that still publishes the pinned tag
template: charts/redis/templates/master/application.yaml
set:
redis.enabled: true
asserts:
- equal:
path: spec.template.spec.containers[0].image
value: docker.io/bitnamilegacy/redis:7.2.4-debian-12-r9
- it: should reject a floating postgres tag that could cross a major version on an existing volume
template: secret-dbcredentials.yaml
set:
db.deployStandalone: true
postgresql.image.tag: latest
asserts:
- failedTemplate:
errorMessage: 'postgresql.image.tag must be pinned to an explicit version when db.deployStandalone is true (got "latest"). An unpinned tag can start a different PostgreSQL major against the existing data directory, which makes the database unreadable and is not recoverable in place. Crossing a major version requires a dump and restore.'
- it: should reject an empty postgres tag
template: secret-dbcredentials.yaml
set:
db.deployStandalone: true
postgresql.image.tag: ""
asserts:
- failedTemplate:
errorMessage: 'postgresql.image.tag must be pinned to an explicit version when db.deployStandalone is true (got ""). An unpinned tag can start a different PostgreSQL major against the existing data directory, which makes the database unreadable and is not recoverable in place. Crossing a major version requires a dump and restore.'
- it: should accept an empty postgres tag when the image is pinned by digest
template: secret-dbcredentials.yaml
set:
db.deployStandalone: true
postgresql.image.tag: ""
postgresql.image.digest: sha256:0d0e2f1a5b3c4d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e6
asserts:
- hasDocuments:
count: 1
- it: should run the servicemonitor test pod from a pinned image
template: templates/tests/test-servicemonitor.yaml
set:
serviceMonitor.enabled: true
asserts:
- equal:
path: spec.containers[0].image
value: docker.io/bitnamilegacy/kubectl:1.29.2-debian-12-r3
- it: should not constrain the postgres tag when the bundled database is not deployed
template: secret-dbcredentials.yaml
set:
db.deployStandalone: false
postgresql.image.tag: latest
asserts:
- hasDocuments:
count: 0

272
helm/litellm-helm/tests/collector_tests.yaml
View file

@ -1,272 +0,0 @@
suite: test collector sidecar
templates:
- deployment.yaml
- hpa.yaml
- configmap-litellm.yaml
tests:
- it: should run the proxy alone with no collector env by default
template: deployment.yaml
asserts:
- lengthEqual:
path: spec.template.spec.containers
count: 1
- notContains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_COLLECTOR_ENABLED
value: "true"
- notContains:
path: spec.template.spec.volumes
content:
name: collector-socket
any: true
- it: should add the sidecar on the same image and point both containers at the unix socket
template: deployment.yaml
set:
image.tag: test
db.connectionPool.enabled: true
collector.enabled: true
collector.resources:
requests:
cpu: 500m
memory: 1Gi
limits:
cpu: "1"
memory: 2Gi
asserts:
- lengthEqual:
path: spec.template.spec.containers
count: 2
- equal:
path: spec.template.spec.containers[1].name
value: litellm-collector
- equal:
path: spec.template.spec.containers[1].image
value: ghcr.io/berriai/litellm:test
- equal:
path: spec.template.spec.containers[1].command
value: [python, -m, litellm.proxy.collector]
- equal:
path: spec.template.spec.containers[1].resources.requests.cpu
value: 500m
- equal:
path: spec.template.spec.containers[1].resources.limits.memory
value: 2Gi
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_COLLECTOR_ENABLED
value: "true"
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_COLLECTOR_ADDRESS
value: unix:///var/run/litellm/collector.sock
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_COLLECTOR_BUFFER_SIZE
value: "1000"
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_COLLECTOR_ON_UNAVAILABLE
value: fallback
- notContains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_JOB_ROLE
value: collector
- contains:
path: spec.template.spec.containers[1].env
content:
name: LITELLM_JOB_ROLE
value: collector
- contains:
path: spec.template.spec.containers[1].env
content:
name: LITELLM_COLLECTOR_ADDRESS
value: unix:///var/run/litellm/collector.sock
- contains:
path: spec.template.spec.containers[1].env
content:
name: CONFIG_FILE_PATH
value: /etc/litellm/config.yaml
- contains:
path: spec.template.spec.containers[1].env
content:
name: DATABASE_HOST
value: RELEASE-NAME-postgresql
- contains:
path: spec.template.spec.containers[1].env
content:
name: DATABASE_PASSWORD
valueFrom:
secretKeyRef:
name: RELEASE-NAME-litellm-dbcredentials
key: password
- contains:
path: spec.template.spec.containers[1].env
content:
name: LITELLM_PGBOUNCER_ENABLED
value: "true"
- contains:
path: spec.template.spec.containers[1].env
content:
name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS
value: "20"
- contains:
path: spec.template.spec.containers[0].volumeMounts
content:
name: collector-socket
mountPath: /var/run/litellm
- contains:
path: spec.template.spec.containers[1].volumeMounts
content:
name: collector-socket
mountPath: /var/run/litellm
- contains:
path: spec.template.spec.containers[1].volumeMounts
content:
name: litellm-config
mountPath: /etc/litellm/config.yaml
subPath: config.yaml
- contains:
path: spec.template.spec.volumes
content:
name: collector-socket
emptyDir:
sizeLimit: 1Mi
- it: should skip the socket volume and pass the policy through on tcp transport
template: deployment.yaml
set:
collector.enabled: true
collector.address: tcp://127.0.0.1:4100
collector.onUnavailable: drop
collector.bufferSize: 50
envVars:
CONFIG_FILE_PATH: /custom/config.yaml
asserts:
- lengthEqual:
path: spec.template.spec.containers
count: 2
- notContains:
path: spec.template.spec.containers[1].env
content:
name: CONFIG_FILE_PATH
value: /etc/litellm/config.yaml
- contains:
path: spec.template.spec.containers[1].env
content:
name: CONFIG_FILE_PATH
value: /custom/config.yaml
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_COLLECTOR_ADDRESS
value: tcp://127.0.0.1:4100
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_COLLECTOR_ON_UNAVAILABLE
value: drop
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_COLLECTOR_BUFFER_SIZE
value: "50"
- notContains:
path: spec.template.spec.volumes
content:
name: collector-socket
any: true
- it: should keep metrics and billing env on the proxy container only
template: deployment.yaml
set:
collector.enabled: true
metricsServer.enabled: true
metricsServer.port: 9090
billingMetrics.enabled: true
billingMetrics.endpoint: https://metering.example.com
billingMetrics.secretName: billing-metrics
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: PROMETHEUS_METRICS_PORT
value: "9090"
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_BILLING_METRICS_ENDPOINT
value: https://metering.example.com
- notContains:
path: spec.template.spec.containers[1].env
content:
name: PROMETHEUS_METRICS_PORT
any: true
- notContains:
path: spec.template.spec.containers[1].env
content:
name: LITELLM_BILLING_METRICS_ENDPOINT
any: true
- notContains:
path: spec.template.spec.containers[1].env
content:
name: LITELLM_BILLING_METRICS_CLIENT_SECRET
any: true
- it: should give the sidecar the same scratch mounts as the proxy on a read-only root
template: deployment.yaml
set:
collector.enabled: true
securityContext.readOnlyRootFilesystem: true
asserts:
- contains:
path: spec.template.spec.containers[1].volumeMounts
content:
name: npm
mountPath: /.npm
- contains:
path: spec.template.spec.containers[1].volumeMounts
content:
name: cache
mountPath: /.cache
- contains:
path: spec.template.spec.containers[1].volumeMounts
content:
name: tmp
mountPath: /tmp
- it: should keep the pod-wide cpu metric unless asked to scale on the proxy container
template: hpa.yaml
set:
autoscaling.enabled: true
collector.enabled: true
asserts:
- equal: { path: "spec.metrics[0].type", value: Resource }
- equal: { path: "spec.metrics[0].resource.name", value: cpu }
- it: should scale on the proxy container's cpu only when opted in
template: hpa.yaml
set:
autoscaling.enabled: true
collector.enabled: true
collector.scaleOnProxyContainerCpu: true
asserts:
- equal: { path: "spec.metrics[0].type", value: ContainerResource }
- equal: { path: "spec.metrics[0].containerResource.name", value: cpu }
- equal: { path: "spec.metrics[0].containerResource.container", value: litellm }
- equal: { path: "spec.metrics[0].containerResource.target.averageUtilization", value: 60 }
- isNull: { path: "spec.metrics[0].resource" }
- it: should not switch to the container metric while the sidecar is off
template: hpa.yaml
set:
autoscaling.enabled: true
collector.scaleOnProxyContainerCpu: true
asserts:
- equal: { path: "spec.metrics[0].type", value: Resource }

112
helm/litellm-helm/tests/connection_pool_tests.yaml
View file

@ -1,112 +0,0 @@
suite: test in-container connection pool
templates:
- deployment.yaml
- configmap-litellm.yaml
tests:
- it: should not emit pgbouncer env vars by default
template: deployment.yaml
asserts:
- notContains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_PGBOUNCER_ENABLED
value: "true"
- notContains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS
value: "20"
- it: should enable the pool with the default sizing when connectionPool.enabled is set
template: deployment.yaml
set:
db.connectionPool.enabled: true
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_PGBOUNCER_ENABLED
value: "true"
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS
value: "20"
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN
value: "1000"
- it: should pass custom sizing through as strings next to the worker count
template: deployment.yaml
set:
numWorkers: 4
db.connectionPool.enabled: true
db.connectionPool.maxDbConnections: 8
db.connectionPool.maxClientConn: 400
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS
value: "8"
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN
value: "400"
- contains:
path: spec.template.spec.containers[0].args
content: "4"
- it: should give the collector sidecar the same pool env as the proxy container
template: deployment.yaml
set:
collector.enabled: true
db.connectionPool.enabled: true
db.connectionPool.maxDbConnections: 8
db.connectionPool.maxClientConn: 400
asserts:
- equal:
path: spec.template.spec.containers[1].name
value: litellm-collector
- contains:
path: spec.template.spec.containers[1].env
content:
name: LITELLM_PGBOUNCER_ENABLED
value: "true"
- contains:
path: spec.template.spec.containers[1].env
content:
name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS
value: "8"
- contains:
path: spec.template.spec.containers[1].env
content:
name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN
value: "400"
- it: should give the collector sidecar no pool env when the pool is off
template: deployment.yaml
set:
collector.enabled: true
asserts:
- equal:
path: spec.template.spec.containers[1].name
value: litellm-collector
- notContains:
path: spec.template.spec.containers[1].env
content:
name: LITELLM_PGBOUNCER_ENABLED
any: true
- notContains:
path: spec.template.spec.containers[1].env
content:
name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS
any: true
- notContains:
path: spec.template.spec.containers[1].env
content:
name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN
any: true

143
helm/litellm-helm/tests/coordination_redis_tests.yaml
View file

@ -1,143 +0,0 @@
suite: test coordination redis
templates:
- configmap-litellm.yaml
- deployment.yaml
tests:
- it: should not render coordination_redis when redis is disabled
template: configmap-litellm.yaml
set:
redis.enabled: false
asserts:
- notMatchRegex:
path: data["config.yaml"]
pattern: coordination_redis
- it: should not emit redis env vars when redis is disabled
template: deployment.yaml
set:
redis.enabled: false
asserts:
- notContains:
path: spec.template.spec.containers[0].env
content:
name: REDIS_HOST
value: RELEASE-NAME-redis-master
any: true
- it: should render coordination_redis pointing at the bundled redis when enabled
template: configmap-litellm.yaml
set:
redis.enabled: true
asserts:
- matchRegex:
path: data["config.yaml"]
pattern: "coordination_redis:\n host: os.environ/REDIS_HOST\n password: os.environ/REDIS_PASSWORD\n port: os.environ/REDIS_PORT\n"
- matchRegex:
path: data["config.yaml"]
pattern: "master_key: os.environ/PROXY_MASTER_KEY"
- it: should emit redis env vars backing the coordination_redis os.environ refs
template: deployment.yaml
set:
redis.enabled: true
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: REDIS_HOST
value: RELEASE-NAME-redis-master
- contains:
path: spec.template.spec.containers[0].env
content:
name: REDIS_PORT
value: "6379"
- contains:
path: spec.template.spec.containers[0].env
content:
name: REDIS_PASSWORD
valueFrom:
secretKeyRef:
name: RELEASE-NAME-redis
key: redis-password
- it: should not render coordination_redis when coordination is opted out
template: configmap-litellm.yaml
set:
redis.enabled: true
redis.coordination.enabled: false
asserts:
- notMatchRegex:
path: data["config.yaml"]
pattern: coordination_redis
- it: should keep emitting redis env vars when coordination is opted out
template: deployment.yaml
set:
redis.enabled: true
redis.coordination.enabled: false
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: REDIS_HOST
value: RELEASE-NAME-redis-master
- it: should not clobber a user supplied coordination_redis block
template: configmap-litellm.yaml
set:
redis.enabled: true
proxy_config.general_settings.coordination_redis:
url: os.environ/COORDINATION_REDIS_URL
asserts:
- matchRegex:
path: data["config.yaml"]
pattern: "coordination_redis:\n url: os.environ/COORDINATION_REDIS_URL\n"
- notMatchRegex:
path: data["config.yaml"]
pattern: "host: os.environ/REDIS_HOST"
- it: should render sentinel_nodes and service_name in sentinel mode
template: configmap-litellm.yaml
set:
redis.enabled: true
redis.architecture: replication
redis.sentinel.enabled: true
asserts:
# The sentinel Service the redis subchart renders is "<release>-redis", and a
# plain client cannot speak the sentinel protocol, so host/port must not appear
- matchRegex:
path: data["config.yaml"]
pattern: "coordination_redis:\n password: os.environ/REDIS_PASSWORD\n sentinel_nodes:\n - - RELEASE-NAME-redis\n - 26379\n service_name: mymaster\n"
- notMatchRegex:
path: data["config.yaml"]
pattern: "host: os.environ/REDIS_HOST"
- it: should carry a custom sentinel masterSet into service_name
template: configmap-litellm.yaml
set:
redis.enabled: true
redis.architecture: replication
redis.sentinel.enabled: true
redis.sentinel.masterSet: litellm-master
asserts:
- matchRegex:
path: data["config.yaml"]
pattern: "service_name: litellm-master"
- it: should point REDIS_HOST at the sentinel service in sentinel mode
template: deployment.yaml
set:
redis.enabled: true
redis.architecture: replication
redis.sentinel.enabled: true
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: REDIS_HOST
value: RELEASE-NAME-redis
- contains:
path: spec.template.spec.containers[0].env
content:
name: REDIS_PORT
value: "26379"

288
helm/litellm-helm/tests/database_auth_tests.yaml
View file

@ -1,288 +0,0 @@
suite: test existing database token auth
templates:
- deployment.yaml
- migrations-job.yaml
- configmap-litellm.yaml
tests:
- it: should keep password auth from the secret when no token auth is set
template: deployment.yaml
set:
db.useExisting: true
db.deployStandalone: false
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_USERNAME
valueFrom:
secretKeyRef:
name: postgres
key: username
- contains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_PASSWORD
valueFrom:
secretKeyRef:
name: postgres
key: password
- contains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_URL
value: postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME)
- notContains:
path: spec.template.spec.containers[0].env
content:
name: IAM_TOKEN_DB_AUTH
any: true
- notContains:
path: spec.template.spec.containers[0].env
content:
name: AZURE_POSTGRESQL_AUTH
any: true
- notContains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_PORT
any: true
- it: should render RDS IAM auth without a password secret on the proxy and the collector sidecar
template: deployment.yaml
set:
db.useExisting: true
db.deployStandalone: false
db.useIAMAuth: true
db.username: litellm_iam
db.endpoint: writer.cluster-abc.us-west-1.rds.amazonaws.com
db.port: 5433
collector.enabled: true
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: IAM_TOKEN_DB_AUTH
value: "true"
- contains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_USERNAME
value: litellm_iam
- contains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_HOST
value: writer.cluster-abc.us-west-1.rds.amazonaws.com
- contains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_PORT
value: "5433"
- contains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_NAME
value: litellm
- notContains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_PASSWORD
any: true
- notContains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_URL
any: true
- notContains:
path: spec.template.spec.containers[0].env
content:
name: AZURE_POSTGRESQL_AUTH
any: true
- contains:
path: spec.template.spec.containers[1].env
content:
name: IAM_TOKEN_DB_AUTH
value: "true"
- notContains:
path: spec.template.spec.containers[1].env
content:
name: DATABASE_PASSWORD
any: true
- it: should give the migrations job the same token auth env as the proxy
template: migrations-job.yaml
set:
migrationJob.enabled: true
db.useExisting: true
db.deployStandalone: false
db.useIAMAuth: true
db.username: litellm_iam
db.endpoint: writer.cluster-abc.us-west-1.rds.amazonaws.com
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: IAM_TOKEN_DB_AUTH
value: "true"
- contains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_USERNAME
value: litellm_iam
- contains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_HOST
value: writer.cluster-abc.us-west-1.rds.amazonaws.com
- contains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_PORT
value: "5432"
- notContains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_PASSWORD
any: true
- notContains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_URL
any: true
- it: should carry db.schema as DATABASE_SCHEMA under token auth and omit it when empty
template: deployment.yaml
set:
db.useExisting: true
db.deployStandalone: false
db.useIAMAuth: true
db.username: litellm_iam
db.schema: tenant1
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_SCHEMA
value: tenant1
- it: should leave the schema to db.url under password auth even when db.schema is set
template: deployment.yaml
set:
db.useExisting: true
db.deployStandalone: false
db.schema: tenant1
db.url: postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME)?schema=tenant1
asserts:
- notContains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_SCHEMA
any: true
- contains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_URL
value: postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME)?schema=tenant1
- it: should not emit DATABASE_SCHEMA when db.schema is empty
template: deployment.yaml
set:
db.useExisting: true
db.deployStandalone: false
db.useIAMAuth: true
asserts:
- notContains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_SCHEMA
any: true
- it: should read the user from the secret when db.username is empty under token auth
template: deployment.yaml
set:
db.useExisting: true
db.deployStandalone: false
db.useIAMAuth: true
db.secret.name: rds-identity
db.secret.usernameKey: user
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_USERNAME
valueFrom:
secretKeyRef:
name: rds-identity
key: user
- notContains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_PASSWORD
any: true
- it: should render Entra auth with AZURE_POSTGRESQL_AUTH and no password
template: deployment.yaml
set:
db.useExisting: true
db.deployStandalone: false
db.useAzureEntraAuth: true
db.username: litellm@example.onmicrosoft.com
db.endpoint: litellm.postgres.database.azure.com
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: AZURE_POSTGRESQL_AUTH
value: "true"
- contains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_USERNAME
value: litellm@example.onmicrosoft.com
- notContains:
path: spec.template.spec.containers[0].env
content:
name: IAM_TOKEN_DB_AUTH
any: true
- notContains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_PASSWORD
any: true
- notContains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_URL
any: true
- it: should fail when both token sources are enabled
template: deployment.yaml
set:
migrationJob.enabled: false
db.useExisting: true
db.deployStandalone: false
db.useIAMAuth: true
db.useAzureEntraAuth: true
asserts:
- failedTemplate:
errorMessage: "db.useIAMAuth and db.useAzureEntraAuth are mutually exclusive: the database password can only come from one token source"
- it: should fail when token auth is set while the bundled postgres is still deployed
template: deployment.yaml
set:
migrationJob.enabled: false
db.useExisting: true
db.useIAMAuth: true
asserts:
- failedTemplate:
errorPattern: "need db.useExisting: true and db.deployStandalone: false"
- it: should fail when token auth is set without useExisting
template: migrations-job.yaml
set:
migrationJob.enabled: true
db.deployStandalone: false
db.useAzureEntraAuth: true
asserts:
- failedTemplate:
errorPattern: "need db.useExisting: true and db.deployStandalone: false"

68
helm/litellm-helm/tests/deployment_command_args_labels_tests.yaml
View file

@ -1,68 +0,0 @@
suite: test deployment command, args, and deploymentLabels
templates:
- deployment.yaml
- configmap-litellm.yaml
tests:
- it: should override args when custom args specified
template: deployment.yaml
set:
args:
- --custom-arg1
- value1
- --custom-arg2
asserts:
- equal:
path: spec.template.spec.containers[0].args
value:
- --custom-arg1
- value1
- --custom-arg2
- it: should set custom command when specified
template: deployment.yaml
set:
command:
- /bin/sh
- -c
asserts:
- equal:
path: spec.template.spec.containers[0].command
value:
- /bin/sh
- -c
- it: should set custom command and args together
template: deployment.yaml
set:
command:
- python
- -u
args:
- my_script.py
- --verbose
asserts:
- equal:
path: spec.template.spec.containers[0].command
value:
- python
- -u
- equal:
path: spec.template.spec.containers[0].args
value:
- my_script.py
- --verbose
- it: should add deploymentLabels to deployment metadata
template: deployment.yaml
set:
deploymentLabels:
environment: production
team: platform
version: v1.2.3
asserts:
- equal:
path: metadata.labels.environment
value: production
- equal:
path: metadata.labels.team
value: platform
- equal:
path: metadata.labels.version
value: v1.2.3

520
helm/litellm-helm/tests/deployment_tests.yaml
View file

@ -1,520 +0,0 @@
suite: test deployment
templates:
- deployment.yaml
- configmap-litellm.yaml
tests:
- it: should work
template: deployment.yaml
set:
image.tag: test
asserts:
- isKind:
of: Deployment
- matchRegex:
path: metadata.name
pattern: -litellm$
- equal:
path: spec.template.spec.containers[0].image
value: ghcr.io/berriai/litellm:test
- it: should work with tolerations
template: deployment.yaml
set:
tolerations:
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
asserts:
- equal:
path: spec.template.spec.tolerations[0].key
value: node-role.kubernetes.io/master
- equal:
path: spec.template.spec.tolerations[0].operator
value: Exists
- it: should work with affinity
template: deployment.yaml
set:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: topology.kubernetes.io/zone
operator: In
values:
- antarctica-east1
asserts:
- equal:
path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key
value: topology.kubernetes.io/zone
- equal:
path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator
value: In
- equal:
path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values[0]
value: antarctica-east1
- it: should work without masterkeySecretName or masterkeySecretKey
template: deployment.yaml
set:
masterkeySecretName: ""
masterkeySecretKey: ""
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: PROXY_MASTER_KEY
valueFrom:
secretKeyRef:
name: RELEASE-NAME-litellm-masterkey
key: masterkey
- it: should work with masterkeySecretName and masterkeySecretKey
template: deployment.yaml
set:
masterkeySecretName: my-secret
masterkeySecretKey: my-key
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: PROXY_MASTER_KEY
valueFrom:
secretKeyRef:
name: my-secret
key: my-key
- it: should inject DATABASE_READER_HOST from readReplicaEndpointKey before DATABASE_URL_READ_REPLICA
template: deployment.yaml
set:
db:
deployStandalone: false
useExisting: true
secret:
name: postgres
usernameKey: username
passwordKey: password
readReplicaEndpointKey: reader-host
readReplicaUrl: postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_READER_HOST):5432/$(DATABASE_NAME)?sslmode=require
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_READER_HOST
valueFrom:
secretKeyRef:
name: postgres
key: reader-host
- contains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_URL_READ_REPLICA
value: postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_READER_HOST):5432/$(DATABASE_NAME)?sslmode=require
# $(VAR) interpolation only resolves vars defined EARLIER in the env
# array, so the reader host must precede the composed URL
- equal:
path: spec.template.spec.containers[0].env[7].name
value: DATABASE_READER_HOST
- equal:
path: spec.template.spec.containers[0].env[8].name
value: DATABASE_URL_READ_REPLICA
- it: should omit reader host when readReplicaUrl is unset
template: deployment.yaml
set:
db:
deployStandalone: false
useExisting: true
secret:
name: postgres
usernameKey: username
passwordKey: password
readReplicaEndpointKey: reader-host
asserts:
- notContains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_READER_HOST
valueFrom:
secretKeyRef:
name: postgres
key: reader-host
- it: should prefer readReplicaUrlKey over readReplicaEndpointKey composition
template: deployment.yaml
set:
db:
useExisting: true
secret:
name: postgres
usernameKey: username
passwordKey: password
readReplicaUrlKey: reader-url
readReplicaEndpointKey: reader-host
readReplicaUrl: postgresql://ignored
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_URL_READ_REPLICA
valueFrom:
secretKeyRef:
name: postgres
key: reader-url
- notContains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_URL_READ_REPLICA
value: postgresql://ignored
# the unused reader-host secret ref must be suppressed so a missing
# key can't fail pod creation
- notContains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_READER_HOST
valueFrom:
secretKeyRef:
name: postgres
key: reader-host
- it: should work with extraEnvVars
template: deployment.yaml
set:
extraEnvVars:
- name: EXTRA_ENV_VAR
valueFrom:
fieldRef:
fieldPath: metadata.labels['env']
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: EXTRA_ENV_VAR
valueFrom:
fieldRef:
fieldPath: metadata.labels['env']
- it: should work with both extraEnvVars and envVars
template: deployment.yaml
set:
envVars:
ENV_VAR: ENV_VAR_VALUE
extraEnvVars:
- name: EXTRA_ENV_VAR
value: EXTRA_ENV_VAR_VALUE
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: ENV_VAR
value: ENV_VAR_VALUE
- contains:
path: spec.template.spec.containers[0].env
content:
name: EXTRA_ENV_VAR
value: EXTRA_ENV_VAR_VALUE
- it: should mount existing configmap when create=false
template: deployment.yaml
set:
proxyConfigMap:
create: false
name: my-litellm-config
key: custom.yaml
asserts:
- contains:
path: spec.template.spec.volumes
content:
name: litellm-config
configMap:
name: my-litellm-config
items:
- key: custom.yaml
path: config.yaml
- contains:
path: spec.template.spec.containers[0].volumeMounts
content:
name: litellm-config
mountPath: /etc/litellm/config.yaml
subPath: config.yaml
- it: should work with lifecycle hooks
template: deployment.yaml
set:
lifecycle:
preStop:
exec:
command:
- /bin/sh
- -c
- echo "Container stopping"
asserts:
- exists:
path: spec.template.spec.containers[0].lifecycle
- equal:
path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[0]
value: /bin/sh
- equal:
path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[1]
value: -c
- equal:
path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[2]
value: echo "Container stopping"
- it: should render background health check settings from proxy_config.general_settings
template: configmap-litellm.yaml
set:
proxy_config.general_settings.background_health_checks: true
proxy_config.general_settings.health_check_interval: 240
proxy_config.general_settings.health_check_concurrency: 16
proxy_config.general_settings.health_check_details: false
asserts:
- matchRegex:
path: data["config.yaml"]
pattern: '(?m)^\s*background_health_checks:\s*true$'
- matchRegex:
path: data["config.yaml"]
pattern: '(?m)^\s*health_check_interval:\s*240$'
- matchRegex:
path: data["config.yaml"]
pattern: '(?m)^\s*health_check_concurrency:\s*16$'
- matchRegex:
path: data["config.yaml"]
pattern: '(?m)^\s*health_check_details:\s*false$'
- it: should not render any placeholder models when the user sets proxy_config without model_list
template: configmap-litellm.yaml
set:
proxy_config.general_settings.store_model_in_db: true
asserts:
- notMatchRegex:
path: data["config.yaml"]
pattern: 'model_name:'
- matchRegex:
path: data["config.yaml"]
pattern: 'store_model_in_db: true'
- matchRegex:
path: data["config.yaml"]
pattern: 'master_key: os.environ/PROXY_MASTER_KEY'
- it: should render a user-supplied model_list as the only models
template: configmap-litellm.yaml
set:
proxy_config.model_list:
- model_name: my-model
litellm_params:
model: openai/my-model
api_key: os.environ/OPENAI_API_KEY
asserts:
- matchRegex:
path: data["config.yaml"]
pattern: 'model_list:\n- litellm_params:\n api_key: os.environ/OPENAI_API_KEY\n model: openai/my-model\n model_name: my-model\n$'
- it: should allow overriding liveness, readiness, and startup probes
template: deployment.yaml
set:
livenessProbe:
path: /custom/livez
initialDelaySeconds: 5
periodSeconds: 15
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 5
readinessProbe:
path: /custom/readyz
initialDelaySeconds: 10
periodSeconds: 20
timeoutSeconds: 6
successThreshold: 1
failureThreshold: 6
startupProbe:
path: /custom/startupz
initialDelaySeconds: 15
periodSeconds: 25
timeoutSeconds: 7
successThreshold: 1
failureThreshold: 40
asserts:
- equal:
path: spec.template.spec.containers[0].livenessProbe.httpGet.path
value: /custom/livez
- equal:
path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds
value: 5
- equal:
path: spec.template.spec.containers[0].readinessProbe.httpGet.path
value: /custom/readyz
- equal:
path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds
value: 6
- equal:
path: spec.template.spec.containers[0].startupProbe.httpGet.path
value: /custom/startupz
- equal:
path: spec.template.spec.containers[0].startupProbe.failureThreshold
value: 40
- it: should render container resources from values
template: deployment.yaml
set:
resources:
limits:
cpu: 500m
memory: 2Gi
requests:
cpu: 250m
memory: 1Gi
asserts:
- equal:
path: spec.template.spec.containers[0].resources.limits.cpu
value: 500m
- equal:
path: spec.template.spec.containers[0].resources.limits.memory
value: 2Gi
- equal:
path: spec.template.spec.containers[0].resources.requests.cpu
value: 250m
- equal:
path: spec.template.spec.containers[0].resources.requests.memory
value: 1Gi
- it: should keep default probes and empty resources unchanged
template: deployment.yaml
asserts:
- equal:
path: spec.template.spec.containers[0].livenessProbe.httpGet.path
value: /health/liveliness
- equal:
path: spec.template.spec.containers[0].livenessProbe.initialDelaySeconds
value: 0
- equal:
path: spec.template.spec.containers[0].livenessProbe.periodSeconds
value: 15
- equal:
path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds
value: 5
- equal:
path: spec.template.spec.containers[0].livenessProbe.successThreshold
value: 1
- equal:
path: spec.template.spec.containers[0].livenessProbe.failureThreshold
value: 5
- equal:
path: spec.template.spec.containers[0].readinessProbe.httpGet.path
value: /health/readiness
- equal:
path: spec.template.spec.containers[0].readinessProbe.initialDelaySeconds
value: 0
- equal:
path: spec.template.spec.containers[0].readinessProbe.periodSeconds
value: 10
- equal:
path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds
value: 5
- equal:
path: spec.template.spec.containers[0].readinessProbe.successThreshold
value: 1
- equal:
path: spec.template.spec.containers[0].readinessProbe.failureThreshold
value: 3
- equal:
path: spec.template.spec.containers[0].startupProbe.httpGet.path
value: /health/readiness
- equal:
path: spec.template.spec.containers[0].startupProbe.initialDelaySeconds
value: 0
- equal:
path: spec.template.spec.containers[0].startupProbe.periodSeconds
value: 10
- equal:
path: spec.template.spec.containers[0].startupProbe.timeoutSeconds
value: 5
- equal:
path: spec.template.spec.containers[0].startupProbe.successThreshold
value: 1
- equal:
path: spec.template.spec.containers[0].startupProbe.failureThreshold
value: 30
- equal:
path: spec.template.spec.containers[0].resources
value: {}
- it: should be able to set minReadySeconds
template: deployment.yaml
set:
deploymentMinReadySeconds: 5
asserts:
- equal:
path: spec.minReadySeconds
value: 5
- it: should have minReadySeconds absent when deploymentMinReadySeconds is not set
template: deployment.yaml
asserts:
- notExists:
path: spec.minReadySeconds
- it: should work with extraInitContainers
template: deployment.yaml
set:
extraInitContainers:
- name: init-test
image: busybox:latest
command: ["echo", "hello"]
asserts:
- contains:
path: spec.template.spec.initContainers
content:
name: init-test
image: busybox:latest
command: ["echo", "hello"]
- it: should support tpl in extraInitContainers
template: deployment.yaml
set:
image:
repository: ghcr.io/berriai/litellm
tag: test
extraInitContainers:
- name: init-tpl
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
command: ["echo", "hello"]
asserts:
- contains:
path: spec.template.spec.initContainers
content:
name: init-tpl
image: "ghcr.io/berriai/litellm:test"
command: ["echo", "hello"]
- it: should work with extraContainers
template: deployment.yaml
set:
extraContainers:
- name: sidecar
image: busybox:latest
asserts:
- contains:
path: spec.template.spec.containers
content:
name: sidecar
image: busybox:latest
- it: should support tpl in extraContainers
template: deployment.yaml
set:
image:
repository: ghcr.io/berriai/litellm
tag: test
extraContainers:
- name: sidecar-tpl
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
asserts:
- contains:
path: spec.template.spec.containers
content:
name: sidecar-tpl
image: "ghcr.io/berriai/litellm:test"
- it: should support tpl in podAnnotations
template: deployment.yaml
set:
image:
repository: ghcr.io/berriai/litellm
tag: test
# Mirrors the real-world scenario this feature unblocks:
# user disables the built-in ConfigMap (and its built-in checksum/config
# annotation) and re-implements checksum/config themselves via tpl.
proxyConfigMap:
create: false
podAnnotations:
checksum/config: "{{ .Values.image.tag }}"
example.com/some-key: "{{ .Values.image.repository }}"
example.com/literal: "plain-string-value"
asserts:
- equal:
path: spec.template.metadata.annotations["checksum/config"]
value: "test"
- equal:
path: spec.template.metadata.annotations["example.com/some-key"]
value: "ghcr.io/berriai/litellm"
- equal:
path: spec.template.metadata.annotations["example.com/literal"]
value: "plain-string-value"

144
helm/litellm-helm/tests/hpa_tests.yaml
View file

@ -1,144 +0,0 @@
suite: "hpa"
templates:
- hpa.yaml
tests:
- it: "renders behavior when set"
set:
autoscaling.enabled: true
autoscaling.behavior:
scaleUp:
stabilizationWindowSeconds: 60
policies:
- type: Pods
value: 2
periodSeconds: 60
scaleDown:
stabilizationWindowSeconds: 90
policies:
- type: Pods
value: 1
periodSeconds: 60
asserts:
- isKind: { of: HorizontalPodAutoscaler }
- equal: { path: spec.behavior.scaleUp.stabilizationWindowSeconds, value: 60 }
- equal: { path: spec.behavior.scaleDown.stabilizationWindowSeconds, value: 90 }
- it: "does not render behavior when not set"
set:
autoscaling.enabled: true
asserts:
- isKind: { of: HorizontalPodAutoscaler }
- isNull: { path: spec.behavior }
- it: "scales on cpu at the documented 60 percent by default"
set:
autoscaling.enabled: true
asserts:
- isKind: { of: HorizontalPodAutoscaler }
- equal: { path: "spec.metrics[0].resource.name", value: cpu }
- equal: { path: "spec.metrics[0].resource.target.type", value: Utilization }
- equal: { path: "spec.metrics[0].resource.target.averageUtilization", value: 60 }
- it: "does not scale on memory by default"
set:
autoscaling.enabled: true
asserts:
- lengthEqual: { path: spec.metrics, count: 1 }
- it: "honours an explicit cpu target override"
set:
autoscaling.enabled: true
autoscaling.targetCPUUtilizationPercentage: 75
asserts:
- equal: { path: "spec.metrics[0].resource.target.averageUtilization", value: 75 }
- it: "renders a memory metric only when a memory target is set"
set:
autoscaling.enabled: true
autoscaling.targetMemoryUtilizationPercentage: 80
asserts:
- lengthEqual: { path: spec.metrics, count: 2 }
- equal: { path: "spec.metrics[1].resource.name", value: memory }
- equal: { path: "spec.metrics[1].resource.target.averageUtilization", value: 80 }
- it: "renders no workload metrics by default"
set:
autoscaling.enabled: true
autoscaling.targetMemoryUtilizationPercentage: 80
asserts:
- lengthEqual: { path: spec.metrics, count: 2 }
- notContains: { path: spec.metrics, content: { type: Pods }, any: true }
- it: "adds a requests-per-second Pods metric after the cpu metric"
set:
autoscaling.enabled: true
autoscaling.targetRequestsPerSecond: 90
asserts:
- lengthEqual: { path: spec.metrics, count: 2 }
- equal: { path: "spec.metrics[0].resource.name", value: cpu }
- equal:
path: "spec.metrics[1]"
value:
type: Pods
pods:
metric: { name: litellm_requests_per_second }
target: { type: AverageValue, averageValue: "90" }
- it: "adds a tokens-per-second Pods metric on its own"
set:
autoscaling.enabled: true
autoscaling.targetTokensPerSecond: 6M
asserts:
- lengthEqual: { path: spec.metrics, count: 2 }
- equal:
path: "spec.metrics[1]"
value:
type: Pods
pods:
metric: { name: litellm_tokens_per_second }
target: { type: AverageValue, averageValue: "6M" }
- notContains:
path: spec.metrics
content: { type: Pods, pods: { metric: { name: litellm_requests_per_second } } }
any: true
- it: "renders requests, tokens, cpu and memory metrics together"
set:
autoscaling.enabled: true
autoscaling.targetMemoryUtilizationPercentage: 80
autoscaling.targetRequestsPerSecond: 90
autoscaling.targetTokensPerSecond: 6000000
asserts:
- lengthEqual: { path: spec.metrics, count: 4 }
- equal: { path: "spec.metrics[0].resource.name", value: cpu }
- equal: { path: "spec.metrics[1].resource.name", value: memory }
- equal: { path: "spec.metrics[2].pods.metric.name", value: litellm_requests_per_second }
- equal: { path: "spec.metrics[2].pods.target.averageValue", value: "90" }
- equal: { path: "spec.metrics[3].pods.metric.name", value: litellm_tokens_per_second }
- equal: { path: "spec.metrics[3].pods.target.averageValue", value: "6000000" }
- it: "scales on workload metrics alone when the cpu target is cleared"
set:
autoscaling.enabled: true
autoscaling.targetCPUUtilizationPercentage: null
autoscaling.targetRequestsPerSecond: 90
autoscaling.targetTokensPerSecond: 6000000
asserts:
- lengthEqual: { path: spec.metrics, count: 2 }
- notContains: { path: spec.metrics, content: { type: Resource }, any: true }
- equal: { path: "spec.metrics[0].pods.metric.name", value: litellm_requests_per_second }
- equal: { path: "spec.metrics[1].pods.metric.name", value: litellm_tokens_per_second }
- notMatchRegexRaw: { pattern: per_minute }
- it: "ignores the per-minute keys, which the chart never shipped"
set:
autoscaling.enabled: true
autoscaling.targetRequestsPerMinute: 5400
autoscaling.targetTokensPerMinute: 360000000
asserts:
- lengthEqual: { path: spec.metrics, count: 1 }
- notContains: { path: spec.metrics, content: { type: Pods }, any: true }
- it: "renders no hpa when autoscaling is disabled"
asserts:
- hasDocuments: { count: 0 }

45
helm/litellm-helm/tests/ingress_tests.yaml
View file

@ -1,45 +0,0 @@
suite: Ingress Configuration Tests
templates:
- ingress.yaml
tests:
- it: should not create Ingress by default
asserts:
- hasDocuments:
count: 0
- it: should create Ingress when enabled
set:
ingress.enabled: true
asserts:
- hasDocuments:
count: 1
- isKind:
of: Ingress
- it: should add custom labels
set:
ingress.enabled: true
ingress.labels:
custom-label: "true"
another-label: "value"
asserts:
- isKind:
of: Ingress
- equal:
path: metadata.labels.custom-label
value: "true"
- equal:
path: metadata.labels.another-label
value: "value"
- it: should add annotations
set:
ingress.enabled: true
ingress.annotations:
kubernetes.io/ingress.class: "nginx"
asserts:
- isKind:
of: Ingress
- equal:
path: metadata.annotations["kubernetes.io/ingress.class"]
value: "nginx"

106
helm/litellm-helm/tests/keda_tests.yaml
View file

@ -1,106 +0,0 @@
suite: "keda"
templates:
- keda.yaml
release:
name: rel
namespace: llm
tests:
- it: "renders no scaled object by default"
asserts:
- hasDocuments: { count: 0 }
- it: "passes user triggers through and adds no prometheus triggers by default"
set:
keda.enabled: true
keda.triggers:
- type: cpu
metricType: Utilization
metadata: { value: "60" }
asserts:
- isKind: { of: ScaledObject }
- equal:
path: spec.triggers
value:
- type: cpu
metricType: Utilization
metadata: { value: "60" }
- it: "scales on release-wide requests per second divided by the per-replica target"
set:
keda.enabled: true
keda.prometheus.serverAddress: http://prometheus-operated.monitoring.svc:9090
keda.prometheus.requestsPerSecond: 90
asserts:
- lengthEqual: { path: spec.triggers, count: 1 }
- equal:
path: "spec.triggers[0]"
value:
type: prometheus
metadata:
serverAddress: http://prometheus-operated.monitoring.svc:9090
threshold: "90"
query: sum(rate(litellm_proxy_total_requests_metric_total{namespace="llm",job="rel-litellm"}[1m]))
- it: "scales on tokens per second on its own"
set:
keda.enabled: true
keda.prometheus.serverAddress: http://prom:9090
keda.prometheus.tokensPerSecond: 6000000
asserts:
- lengthEqual: { path: spec.triggers, count: 1 }
- equal: { path: "spec.triggers[0].type", value: prometheus }
- equal: { path: "spec.triggers[0].metadata.threshold", value: "6000000" }
- equal:
path: "spec.triggers[0].metadata.query"
value: sum(rate(litellm_total_tokens_metric_total{namespace="llm",job="rel-litellm"}[1m]))
- it: "appends requests and tokens triggers after user triggers and selects the metrics service job"
set:
keda.enabled: true
metricsServer.enabled: true
keda.triggers:
- type: cpu
metricType: Utilization
metadata: { value: "60" }
keda.prometheus.serverAddress: http://prom:9090
keda.prometheus.requestsPerSecond: 90
keda.prometheus.tokensPerSecond: 6000000
asserts:
- lengthEqual: { path: spec.triggers, count: 3 }
- equal: { path: "spec.triggers[0].type", value: cpu }
- equal: { path: "spec.triggers[1].metadata.threshold", value: "90" }
- equal:
path: "spec.triggers[1].metadata.query"
value: sum(rate(litellm_proxy_total_requests_metric_total{namespace="llm",job="rel-litellm-metrics"}[1m]))
- equal: { path: "spec.triggers[2].metadata.threshold", value: "6000000" }
- equal:
path: "spec.triggers[2].metadata.query"
value: sum(rate(litellm_total_tokens_metric_total{namespace="llm",job="rel-litellm-metrics"}[1m]))
- notMatchRegexRaw: { pattern: "\\* *60|per_minute|PerMinute" }
- it: "ignores the per-minute keys, which the chart never shipped"
set:
keda.enabled: true
keda.prometheus.serverAddress: http://prom:9090
keda.prometheus.requestsPerMinute: 5400
keda.prometheus.tokensPerMinute: 360000000
asserts:
- isKind: { of: ScaledObject }
- isNullOrEmpty: { path: spec.triggers }
- it: "refuses a workload target without a prometheus server address"
set:
keda.enabled: true
keda.prometheus.requestsPerSecond: 90
asserts:
- failedTemplate:
errorMessage: keda.prometheus.serverAddress is required when keda.prometheus.requestsPerSecond or tokensPerSecond is set
- it: "yields to the hpa when both autoscalers are enabled"
set:
autoscaling.enabled: true
keda.enabled: true
keda.prometheus.serverAddress: http://prom:9090
keda.prometheus.requestsPerSecond: 90
asserts:
- hasDocuments: { count: 0 }

128
helm/litellm-helm/tests/lens_connection_tests.yaml
View file

@ -1,128 +0,0 @@
suite: lens connection
templates:
- deployment.yaml
- configmap-litellm.yaml
- ingress.yaml
tests:
- it: connects the proxy to an external Lens
template: deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_URL
value: http://lens.lens.svc:4318
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_PUBLIC_URL
value: https://lens.example.com/lens-ingest
- contains:
path: spec.template.spec.containers[0].env
content:
name: LENS_GATEWAY_SECRET
valueFrom:
secretKeyRef:
name: lens-gateway
key: gateway-secret
- contains:
path: spec.template.spec.containers[0].env
content:
name: LITELLM_LENS_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: lens-service
key: service-token
- it: requires lens.url when Lens is enabled
template: deployment.yaml
set:
lens.enabled: true
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- failedTemplate:
errorMessage: lens.url is required when lens.enabled is true
- it: requires lens.publicUrl when Lens is enabled
template: deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- failedTemplate:
errorMessage: lens.publicUrl is required when lens.enabled is true
- it: requires lens.gatewaySecret.name when Lens is enabled
template: deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.serviceTokenSecret.name: lens-service
asserts:
- failedTemplate:
errorMessage: lens.gatewaySecret.name is required when lens.enabled is true
- it: requires lens.serviceTokenSecret.name when Lens is enabled
template: deployment.yaml
set:
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
asserts:
- failedTemplate:
errorMessage: lens.serviceTokenSecret.name is required when lens.enabled is true
- it: routes /lens-ingest to the named Lens service ahead of the host paths
template: ingress.yaml
set:
ingress.enabled: true
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
lens.service.name: existing-lens
asserts:
- equal:
path: spec.rules[0].http.paths[0].path
value: /lens-ingest
- contains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
pathType: Prefix
backend:
service:
name: existing-lens
port:
number: 4318
- it: leaves /lens-ingest off the ingress without a Lens service name
template: ingress.yaml
set:
ingress.enabled: true
lens.enabled: true
lens.url: http://lens.lens.svc:4318
lens.publicUrl: https://lens.example.com/lens-ingest
lens.gatewaySecret.name: lens-gateway
lens.serviceTokenSecret.name: lens-service
asserts:
- exists:
path: spec.rules[0].http.paths
- notContains:
path: spec.rules[0].http.paths
content:
path: /lens-ingest
any: true

71
helm/litellm-helm/tests/masterkey-secret_tests.yaml
View file

@ -1,71 +0,0 @@
suite: test masterkey secret
templates:
- secret-masterkey.yaml
tests:
- it: should create a secret if masterkeySecretName is not set. should start with sk-xxxx (base64 encoded as c2st*)
template: secret-masterkey.yaml
set:
masterkeySecretName: ""
asserts:
- isKind:
of: Secret
- matchRegex:
path: data.masterkey
pattern: ^c2st
# Note: The masterkey is generated as "sk-<18-random-chars>" in plain text,
# but stored as base64 encoded in Kubernetes secret (requirement).
# "sk-" base64 encodes to "c2st", so we check for "^c2st" pattern.
- it: should reuse the master key already stored in the cluster instead of generating a new one on upgrade
template: secret-masterkey.yaml
set:
masterkeySecretName: ""
kubernetesProvider:
scheme:
"v1/Secret":
gvr:
version: "v1"
resource: "secrets"
namespaced: true
objects:
- kind: Secret
apiVersion: v1
metadata:
name: RELEASE-NAME-litellm-masterkey
namespace: NAMESPACE
data:
masterkey: c2stZXhpc3Rpbmcta2V5
asserts:
- equal:
path: data.masterkey
value: c2stZXhpc3Rpbmcta2V5
- it: should let an explicit masterkey value override the one already stored in the cluster
template: secret-masterkey.yaml
set:
masterkeySecretName: ""
masterkey: sk-explicit
kubernetesProvider:
scheme:
"v1/Secret":
gvr:
version: "v1"
resource: "secrets"
namespaced: true
objects:
- kind: Secret
apiVersion: v1
metadata:
name: RELEASE-NAME-litellm-masterkey
namespace: NAMESPACE
data:
masterkey: c2stZXhpc3Rpbmcta2V5
asserts:
- equal:
path: data.masterkey
value: c2stZXhwbGljaXQ=
- it: should not create a secret if masterkeySecretName is set
template: secret-masterkey.yaml
set:
masterkeySecretName: my-secret
asserts:
- hasDocuments:
count: 0

106
helm/litellm-helm/tests/metrics_server_tests.yaml
View file

@ -1,106 +0,0 @@
suite: separate metrics server
templates:
- configmap-litellm.yaml
- deployment.yaml
- service.yaml
- service-metrics.yaml
- servicemonitor.yaml
tests:
- it: should not expose a metrics port or PROMETHEUS_METRICS_PORT by default
asserts:
- notContains:
path: spec.template.spec.containers[0].ports
content:
name: metrics
any: true
template: deployment.yaml
- notContains:
path: spec.template.spec.containers[0].env
content:
name: PROMETHEUS_METRICS_PORT
any: true
template: deployment.yaml
- lengthEqual:
path: spec.ports
count: 1
template: service.yaml
- hasDocuments:
count: 0
template: service-metrics.yaml
- it: should scrape the proxy port when the metrics server is disabled
template: servicemonitor.yaml
set:
serviceMonitor.enabled: true
asserts:
- equal:
path: spec.endpoints[0].port
value: http
- it: should wire the separate metrics server through container, a ClusterIP metrics service and servicemonitor
set:
metricsServer.enabled: true
metricsServer.port: 4101
serviceMonitor.enabled: true
service.type: LoadBalancer
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: PROMETHEUS_METRICS_PORT
value: "4101"
template: deployment.yaml
- contains:
path: spec.template.spec.containers[0].ports
content:
name: metrics
containerPort: 4101
protocol: TCP
template: deployment.yaml
- lengthEqual:
path: spec.ports
count: 1
template: service.yaml
- equal:
path: spec.type
value: LoadBalancer
template: service.yaml
- equal:
path: metadata.name
value: RELEASE-NAME-litellm-metrics
template: service-metrics.yaml
- equal:
path: spec.type
value: ClusterIP
template: service-metrics.yaml
- equal:
path: spec.ports
value:
- port: 4101
targetPort: metrics
protocol: TCP
name: metrics
template: service-metrics.yaml
- equal:
path: spec.selector
value:
app.kubernetes.io/name: litellm
app.kubernetes.io/instance: RELEASE-NAME
template: service-metrics.yaml
- equal:
path: spec.endpoints[0].port
value: metrics
template: servicemonitor.yaml
- equal:
path: spec.endpoints[0].path
value: /metrics/
template: servicemonitor.yaml
- it: should reject a metrics port equal to the proxy port
template: deployment.yaml
set:
metricsServer.enabled: true
metricsServer.port: 4000
asserts:
- failedTemplate:
errorMessage: metricsServer.port must differ from service.port

432
helm/litellm-helm/tests/migrations-job_tests.yaml
View file

@ -1,432 +0,0 @@
suite: test migrations job
templates:
- migrations-job.yaml
tests:
- it: should work with envVars
template: migrations-job.yaml
set:
envVars:
TEST_ENV_VAR: "test_value"
ANOTHER_VAR: "another_value"
migrationJob:
enabled: true
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: TEST_ENV_VAR
value: "test_value"
- contains:
path: spec.template.spec.containers[0].env
content:
name: ANOTHER_VAR
value: "another_value"
- it: should work with extraEnvVars
template: migrations-job.yaml
set:
extraEnvVars:
- name: EXTRA_ENV_VAR
valueFrom:
fieldRef:
fieldPath: metadata.labels['env']
- name: SIMPLE_EXTRA_VAR
value: "simple_value"
migrationJob:
enabled: true
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: EXTRA_ENV_VAR
valueFrom:
fieldRef:
fieldPath: metadata.labels['env']
- contains:
path: spec.template.spec.containers[0].env
content:
name: SIMPLE_EXTRA_VAR
value: "simple_value"
- it: should work with both envVars and extraEnvVars
template: migrations-job.yaml
set:
envVars:
ENV_VAR: "env_var_value"
extraEnvVars:
- name: EXTRA_ENV_VAR
value: "extra_env_var_value"
migrationJob:
enabled: true
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: ENV_VAR
value: "env_var_value"
- contains:
path: spec.template.spec.containers[0].env
content:
name: EXTRA_ENV_VAR
value: "extra_env_var_value"
- it: should not render when migrations job is disabled
template: migrations-job.yaml
set:
migrationJob:
enabled: false
asserts:
- hasDocuments:
count: 0
- it: should still include default env vars
template: migrations-job.yaml
set:
envVars:
CUSTOM_VAR: "custom_value"
migrationJob:
enabled: true
db:
useExisting: true
endpoint: "test-db"
database: "testdb"
url: "postgresql://user:pass@test-db:5432/testdb"
secret:
name: "test-secret"
usernameKey: "username"
passwordKey: "password"
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: DISABLE_SCHEMA_UPDATE
value: "false"
- contains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_HOST
value: "test-db"
- contains:
path: spec.template.spec.containers[0].env
content:
name: CUSTOM_VAR
value: "custom_value"
- it: should override a user-supplied DISABLE_SCHEMA_UPDATE so the Job always migrates
template: migrations-job.yaml
set:
envVars:
DISABLE_SCHEMA_UPDATE: "true"
migrationJob:
enabled: true
asserts:
# The Job is what owns the schema, so it renders its own
# DISABLE_SCHEMA_UPDATE=false after envVars and extraEnvVars. Kubernetes
# takes the last value for a duplicated name, so the user's "true" cannot
# leave the schema unmigrated. Skipping migrations is migrationJob.enabled.
- equal:
path: spec.template.spec.containers[0].env[-1]
value:
name: DISABLE_SCHEMA_UPDATE
value: "false"
- it: should not include DATABASE_URL when deployStandalone is false
template: migrations-job.yaml
set:
migrationJob:
enabled: true
db:
deployStandalone: false
useExisting: false
asserts:
- notContains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_URL
- it: should use default service account for helm hooks when serviceAccount.create is true
template: migrations-job.yaml
set:
migrationJob:
enabled: true
hooks:
helm:
enabled: true
serviceAccount:
create: true
asserts:
- equal:
path: spec.template.spec.serviceAccountName
value: default
- it: should use migrationJob.serviceAccountName override for helm hooks when serviceAccount.create is true
template: migrations-job.yaml
set:
migrationJob:
enabled: true
serviceAccountName: migration-sa
hooks:
helm:
enabled: true
serviceAccount:
create: true
asserts:
- equal:
path: spec.template.spec.serviceAccountName
value: migration-sa
- it: should use chart service account when helm hooks are disabled
template: migrations-job.yaml
set:
migrationJob:
enabled: true
hooks:
helm:
enabled: false
serviceAccount:
create: true
name: my-custom-sa
asserts:
- equal:
path: spec.template.spec.serviceAccountName
value: my-custom-sa
- it: should use pre-existing service account when helm hooks are enabled but serviceAccount.create is false
template: migrations-job.yaml
set:
migrationJob:
enabled: true
hooks:
helm:
enabled: true
serviceAccount:
create: false
name: pre-existing-sa
asserts:
- equal:
path: spec.template.spec.serviceAccountName
value: pre-existing-sa
- it: should work with extraInitContainers
template: migrations-job.yaml
set:
migrationJob:
enabled: true
extraInitContainers:
- name: init-test
image: busybox:latest
command: ["echo", "hello"]
asserts:
- contains:
path: spec.template.spec.initContainers
content:
name: init-test
image: busybox:latest
command: ["echo", "hello"]
- it: should support tpl in extraInitContainers
template: migrations-job.yaml
set:
image:
repository: ghcr.io/berriai/litellm
tag: test
migrationJob:
enabled: true
extraInitContainers:
- name: init-tpl
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
command: ["echo", "hello"]
asserts:
- contains:
path: spec.template.spec.initContainers
content:
name: init-tpl
image: "ghcr.io/berriai/litellm:test"
command: ["echo", "hello"]
- it: should work with extraContainers
template: migrations-job.yaml
set:
migrationJob:
enabled: true
extraContainers:
- name: sidecar
image: busybox:latest
asserts:
- contains:
path: spec.template.spec.containers
content:
name: sidecar
image: busybox:latest
- it: should support tpl in extraContainers
template: migrations-job.yaml
set:
image:
repository: ghcr.io/berriai/litellm
tag: test
migrationJob:
enabled: true
extraContainers:
- name: sidecar-tpl
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
asserts:
- contains:
path: spec.template.spec.containers
content:
name: sidecar-tpl
image: "ghcr.io/berriai/litellm:test"
- it: should render the pod-level securityContext from podSecurityContext
template: migrations-job.yaml
set:
migrationJob:
enabled: true
podSecurityContext:
fsGroup: 10000
runAsUser: 10000
runAsNonRoot: true
asserts:
- equal:
path: spec.template.spec.securityContext
value:
fsGroup: 10000
runAsUser: 10000
runAsNonRoot: true
- it: should keep the pod-level and container-level securityContext separate
template: migrations-job.yaml
set:
migrationJob:
enabled: true
podSecurityContext:
fsGroup: 10000
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
asserts:
- equal:
path: spec.template.spec.securityContext
value:
fsGroup: 10000
- equal:
path: spec.template.spec.containers[0].securityContext
value:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
- it: should schedule onto the same nodes as the gateway
template: migrations-job.yaml
set:
migrationJob:
enabled: true
nodeSelector:
karpenter.sh/nodepool: litellm-e2e
tolerations:
- key: workload
operator: Equal
value: litellm-e2e
effect: NoSchedule
asserts:
- equal:
path: spec.template.spec.nodeSelector
value:
karpenter.sh/nodepool: litellm-e2e
- equal:
path: spec.template.spec.tolerations
value:
- key: workload
operator: Equal
value: litellm-e2e
effect: NoSchedule
- it: bounds the Job with a deadline by default, so a blocked migration cannot stall the release forever
set:
migrationJob:
enabled: true
asserts:
- equal:
path: spec.activeDeadlineSeconds
value: 1800
- it: honours an operator-supplied deadline
set:
migrationJob:
enabled: true
activeDeadlineSeconds: 600
asserts:
- equal:
path: spec.activeDeadlineSeconds
value: 600
- it: omits the deadline entirely when it is nulled out, restoring the unbounded behaviour
set:
migrationJob:
enabled: true
activeDeadlineSeconds: null
asserts:
- notExists:
path: spec.activeDeadlineSeconds
- it: should set custom jobLabels and jobAnnotations on Job metadata
template: migrations-job.yaml
set:
migrationJob:
enabled: true
jobLabels:
environment: production
team: platform
jobAnnotations:
example.com/cost-center: "1234"
asserts:
- equal:
path: metadata.labels.environment
value: production
- equal:
path: metadata.labels.team
value: platform
- equal:
path: metadata.annotations['example.com/cost-center']
value: "1234"
- it: should set custom podLabels on Pod template
template: migrations-job.yaml
set:
migrationJob:
enabled: true
podLabels:
custom.io/pod-role: migration
asserts:
- equal:
path: spec.template.metadata.labels['custom.io/pod-role']
value: migration
- it: should override container command and args
template: migrations-job.yaml
set:
migrationJob:
enabled: true
command:
- sh
args:
- -c
- echo migrating
asserts:
- equal:
path: spec.template.spec.containers[0].command
value:
- sh
- equal:
path: spec.template.spec.containers[0].args
value:
- -c
- echo migrating
- it: should clear container args when only command is specified
template: migrations-job.yaml
set:
migrationJob:
enabled: true
command:
- sh
asserts:
- equal:
path: spec.template.spec.containers[0].command
value:
- sh
- equal:
path: spec.template.spec.containers[0].args
value: []

45
helm/litellm-helm/tests/pdb_tests.yaml
View file

@ -1,45 +0,0 @@
suite: "pdb enabled"
templates:
- poddisruptionbudget.yaml
tests:
- it: "renders a PDB with maxUnavailable=1"
set:
pdb.enabled: true
pdb.maxUnavailable: 1
asserts:
- hasDocuments: { count: 1 }
- isKind: { of: PodDisruptionBudget }
- equal: { path: apiVersion, value: policy/v1 }
- equal: { path: spec.maxUnavailable, value: 1 }
- equal:
path: spec.selector.matchLabels
value:
app.kubernetes.io/name: litellm
app.kubernetes.io/instance: RELEASE-NAME
---
suite: "pdb disabled"
templates:
- poddisruptionbudget.yaml
tests:
- it: "does not render when disabled"
set:
pdb.enabled: false
asserts:
- hasDocuments: { count: 0 }
---
suite: "pdb minAvailable precedence"
templates:
- poddisruptionbudget.yaml
tests:
- it: "uses minAvailable when both are set"
set:
pdb.enabled: true
pdb.minAvailable: "50%"
pdb.maxUnavailable: 1
asserts:
- isKind: { of: PodDisruptionBudget }
- equal: { path: apiVersion, value: policy/v1 }
- equal: { path: spec.minAvailable, value: "50%" }
- isNull: { path: spec.maxUnavailable }

116
helm/litellm-helm/tests/service_tests.yaml
View file

@ -1,116 +0,0 @@
suite: Service Configuration Tests
templates:
- service.yaml
tests:
- it: should create a default ClusterIP service
template: service.yaml
asserts:
- isKind:
of: Service
- equal:
path: spec.type
value: ClusterIP
- equal:
path: spec.ports[0].port
value: 4000
- equal:
path: spec.ports[0].targetPort
value: http
- equal:
path: spec.ports[0].protocol
value: TCP
- equal:
path: spec.ports[0].name
value: http
- isNull:
path: spec.loadBalancerClass
- it: should create a NodePort service when specified
template: service.yaml
set:
service.type: NodePort
asserts:
- isKind:
of: Service
- equal:
path: spec.type
value: NodePort
- isNull:
path: spec.loadBalancerClass
- it: should create a LoadBalancer service when specified
template: service.yaml
set:
service.type: LoadBalancer
asserts:
- isKind:
of: Service
- equal:
path: spec.type
value: LoadBalancer
- isNull:
path: spec.loadBalancerClass
- it: should add loadBalancerClass when specified with LoadBalancer type
template: service.yaml
set:
service.type: LoadBalancer
service.loadBalancerClass: tailscale
asserts:
- isKind:
of: Service
- equal:
path: spec.type
value: LoadBalancer
- equal:
path: spec.loadBalancerClass
value: tailscale
- it: should not add loadBalancerClass when specified with ClusterIP type
template: service.yaml
set:
service.type: ClusterIP
service.loadBalancerClass: tailscale
asserts:
- isKind:
of: Service
- equal:
path: spec.type
value: ClusterIP
- isNull:
path: spec.loadBalancerClass
- it: should use custom port when specified
template: service.yaml
set:
service.port: 8080
asserts:
- equal:
path: spec.ports[0].port
value: 8080
- it: should add service annotations when specified
template: service.yaml
set:
service.annotations:
cloud.google.com/load-balancer-type: "Internal"
service.beta.kubernetes.io/aws-load-balancer-internal: "true"
asserts:
- isKind:
of: Service
- equal:
path: metadata.annotations
value:
cloud.google.com/load-balancer-type: "Internal"
service.beta.kubernetes.io/aws-load-balancer-internal: "true"
- it: should use the correct selector labels
template: service.yaml
asserts:
- isNotNull:
path: spec.selector
- equal:
path: spec.selector
value:
app.kubernetes.io/name: litellm
app.kubernetes.io/instance: RELEASE-NAME

676
helm/litellm-helm/values.yaml
View file

@ -1,676 +0,0 @@
# Default values for litellm.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
replicaCount: 1
liteadmin:
enabled: false
existingSecret: ""
gatewayUrl: ""
model: ""
readOnly: false
storageSize: 1Gi
storageClassName: ""
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
memory: 1Gi
# numWorkers: 2
image:
# Bundles the prisma CLI and engines, which is what lets the migrations job
# and the proxy's own schema check run without network access.
repository: ghcr.io/berriai/litellm
pullPolicy: Always
# Overrides the image tag whose default is the chart appVersion.
# tag: "latest"
tag: ""
imagePullSecrets: []
nameOverride: "litellm"
fullnameOverride: ""
serviceAccount:
# Specifies whether a service account should be created
create: false
# Automatically mount a ServiceAccount's API credentials?
automount: true
# Annotations to add to the service account
annotations: {}
# The name of the service account to use.
# If not set and create is true, a name is generated using the fullname template
name: ""
# annotations for litellm deployment
deploymentAnnotations: {}
deploymentLabels: {}
deploymentMinReadySeconds: 0
# annotations for litellm pods
podAnnotations: {}
podLabels: {}
# -- Deployment strategy configuration
# Example:
# type: RollingUpdate
# rollingUpdate:
# maxUnavailable: 0
# maxSurge: 1
strategy: {}
terminationGracePeriodSeconds: 90
topologySpreadConstraints:
[]
# - maxSkew: 1
# topologyKey: kubernetes.io/hostname
# whenUnsatisfiable: DoNotSchedule
# labelSelector:
# matchLabels:
# app: litellm
# At the time of writing, the litellm docker image requires write access to the
# filesystem on startup so that prisma can install some dependencies.
podSecurityContext: {}
securityContext:
{}
# capabilities:
# drop:
# - ALL
# readOnlyRootFilesystem: false
# runAsNonRoot: true
# runAsUser: 1000
# A list of Kubernetes Secret objects that will be exported to the LiteLLM proxy
# pod as environment variables. These secrets can then be referenced in the
# configuration file (or "litellm" ConfigMap) with `os.environ/<Env Var Name>`
environmentSecrets:
[]
# - litellm-env-secret
# A list of Kubernetes ConfigMap objects that will be exported to the LiteLLM proxy
# pod as environment variables. The ConfigMap kv-pairs can then be referenced in the
# configuration file (or "litellm" ConfigMap) with `os.environ/<Env Var Name>`
environmentConfigMaps:
[]
# - litellm-env-configmap
service:
type: ClusterIP
port: 4000
# If service type is `LoadBalancer` you can
# optionally specify loadBalancerClass
# loadBalancerClass: tailscale
# Probes for LiteLLM gateway container
livenessProbe:
path: /health/liveliness
initialDelaySeconds: 0
periodSeconds: 15
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 5
readinessProbe:
path: /health/readiness
initialDelaySeconds: 0
periodSeconds: 10
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 3
startupProbe:
path: /health/readiness
initialDelaySeconds: 0
periodSeconds: 10
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 30
ingress:
enabled: false
className: "nginx"
labels: {}
annotations:
{}
# kubernetes.io/ingress.class: nginx
# kubernetes.io/tls-acme: "true"
hosts:
- host: api.example.local
paths:
- path: /
pathType: ImplementationSpecific
tls: []
# - secretName: chart-example-tls
# hosts:
# - chart-example.local
# masterkey: changeit
# if set, use this secret for the master key; otherwise, autogenerate a new one
masterkeySecretName: ""
# if set, use this secret key for the master key; otherwise, use the default key
masterkeySecretKey: ""
# Optional: enterprise billable-request metering. Requires an enterprise license.
billingMetrics:
enabled: false
endpoint: https://telemetry.litellm.ai
tokenUrl: https://litellm-prod.auth.us-west-1.amazoncognito.com/oauth2/token
secretName: litellm-billing-metrics
exportIntervalMs: ""
proxyConfigMap:
# when true, creates a new configmap
create: true
# if create is false and name is set, use existing ConfigMap
# create: false
# name: ""
# key: "config.yaml"
# The elements within proxy_config are rendered as config.yaml for the proxy
# Examples: https://github.com/BerriAI/litellm/tree/main/litellm/proxy/example_config_yaml
# Reference: https://docs.litellm.ai/docs/proxy/configs
proxy_config:
model_list: []
general_settings:
master_key: os.environ/PROXY_MASTER_KEY
# Serve Prometheus /metrics from a separate process (PROMETHEUS_METRICS_PORT)
# so a scrape never runs on an inference worker. Adds a `metrics` port to the
# container and a dedicated ClusterIP `<release>-metrics` Service, and the
# ServiceMonitor scrapes it instead of the proxy port. The separate port has
# no virtual-key auth: keep it off public ingress. Needs the proxy image
# v1.101.0 or newer.
metricsServer:
enabled: false
port: 4001
# Opt-in sidecar that runs the post-response spend pipeline (cost calculation,
# spend logs, spend counters, budget reservation reconciliation) so the proxy's
# uvicorn workers only serialise a compact typed event and go back to serving
# inference. Same image and tag as the proxy, second container in the same pod,
# fed over loopback (a unix socket on a shared emptyDir, or 127.0.0.1 TCP). It
# reuses the pod's in-container pgbouncer (db.connectionPool) and the same Redis
# spend transaction buffer, so the per-pod DB connection budget is unchanged.
# Delivery is at-most-once inside the pod: events already handed to the sidecar
# are lost if it crashes before writing them; events the workers could not hand
# over follow onUnavailable. Both containers drain on SIGTERM within
# terminationGracePeriodSeconds
collector:
enabled: false
# unix:///<dir>/<file>.sock (the <dir> becomes a shared emptyDir) or tcp://127.0.0.1:<port>
address: unix:///var/run/litellm/collector.sock
# Events each uvicorn worker holds in memory while the sidecar is slow or restarting
bufferSize: 1000
# fallback: run the pipeline in the worker when the sidecar is unreachable or the
# buffer is full (spend stays exact, that request costs proxy CPU again)
# drop: count and discard the event instead (spend under-reports)
onUnavailable: fallback
# How long the workers keep pushing buffered events on shutdown, and how long the
# sidecar keeps serving its open connections after SIGTERM
drainTimeoutSeconds: 10
command:
- python
- -m
- litellm.proxy.collector
# Sized independently of the proxy container; the pipeline is CPU bound
resources: {}
# requests:
# cpu: 500m
# memory: 1Gi
# limits:
# cpu: "1"
# memory: 2Gi
# When autoscaling.enabled, swap the pod-wide cpu Resource metric for an
# autoscaling/v2 ContainerResource metric on the proxy container only, so the
# sidecar's CPU never scales inference replicas. Needs Kubernetes 1.30+ (or the
# HPAContainerMetrics feature gate on 1.27 to 1.29)
scaleOnProxyContainerCpu: false
resources:
{}
# Unset by default so the chart installs on small clusters such as Minikube, and so an
# upgrade never leaves a running pod Pending. Production deployments should set these.
# A proxy at DB-connected steady state needs about 1 CPU and 4Gi of memory per worker;
# sizing below that gets the pod OOMKilled once traffic and DB connections ramp up.
# Scale both figures with --num_workers, then uncomment the lines below and remove the
# curly braces after 'resources:'. See "Recommended Machine Specifications" in
# https://docs.litellm.ai/docs/proxy/prod.
# requests:
# cpu: "1"
# memory: 4Gi
# limits:
# cpu: "1"
# memory: 4Gi
autoscaling:
enabled: false
minReplicas: 1
maxReplicas: 100
# 60 is the documented recommendation. See "Recommended Machine Specifications"
# in https://docs.litellm.ai/docs/proxy/prod. A new replica clears the startupProbe
# above only after up to failureThreshold x periodSeconds = 300 seconds, so a target
# high enough to trip near saturation adds capacity minutes after it was needed.
targetCPUUtilizationPercentage: 60
# Deliberately left unset rather than given a value. The prisma query engine's
# resident memory is a high-water mark that ratchets to the pod's worst-ever write
# and is never returned, so a memory target reads the largest write a pod ever did
# rather than what it is doing now, and replicas ratchet up without scaling back in.
# Memory is a floor to provision under 'resources', not a signal to scale on.
# targetMemoryUtilizationPercentage: 80
# behavior: {}
# Opt-in per-pod workload targets, rendered as autoscaling/v2 `Pods` metrics
# named `litellm_requests_per_second` and `litellm_tokens_per_second` with an
# AverageValue target, alongside whichever resource targets are set (the HPA
# follows the metric asking for the most replicas). A Prometheus Adapter must
# serve those two names on custom.metrics.k8s.io from the proxy's counters,
# grouped by the scrape target's `pod` label (enable serviceMonitor below so
# every pod is scraped on its own):
# litellm_requests_per_second:
# sum(rate(litellm_proxy_total_requests_metric_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
# litellm_tokens_per_second:
# sum(rate(litellm_total_tokens_metric_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
# rate() over [1m] is already per second, so no `* 60`. How fast the HPA
# reacts is set by that window, the scrape interval and the HPA sync period
# (15s by default), not by the unit: keep serviceMonitor.interval at 15s or
# faster so a 1m window holds at least 4 samples. averageValue takes SI
# suffixes, so "6M" is six million tokens per second per pod. Tokens are
# counted when a response completes, so TPS trails long streams.
targetRequestsPerSecond: ""
targetTokensPerSecond: ""
# Autoscaling with keda is mutually exclusive with hpa
keda:
enabled: false
minReplicas: 1
maxReplicas: 100
pollingInterval: 30
cooldownPeriod: 300
# fallback:
# failureThreshold: 3
# replicas: 11
restoreToOriginalReplicaCount: false
scaledObject:
annotations: {}
triggers: []
# - type: prometheus
# metadata:
# serverAddress: http://<prometheus-host>:9090
# metricName: http_requests_total
# threshold: '100'
# query: sum(rate(http_requests_total{deployment="my-deployment"}[2m]))
# First-class Prometheus triggers on the proxy's own request and token
# counters, appended to `triggers`. Each target is the per-second load one
# replica should carry: KEDA divides the release-wide
# `sum(rate(<counter>[1m]))` by it to pick the replica count. Thresholds
# are plain numbers (KEDA parses them as floats, no SI suffixes). The
# queries select samples by the release namespace and the `job` label the
# chart's ServiceMonitor produces (the metrics Service name), so enable
# serviceMonitor below together with metricsServer: the http port serves
# /metrics/ behind virtual-key auth and answers an unauthenticated scrape
# with 401. Reaction time comes from the [1m] window, the scrape interval
# and pollingInterval above, so keep both at 15s or faster. Tokens are
# counted at completion, so TPS trails long streams. serverAddress is
# required once either target is set.
prometheus:
serverAddress: ""
requestsPerSecond: ""
tokensPerSecond: ""
behavior: {}
# scaleDown:
# stabilizationWindowSeconds: 300
# policies:
# - type: Pods
# value: 1
# periodSeconds: 180
# scaleUp:
# stabilizationWindowSeconds: 300
# policies:
# - type: Pods
# value: 2
# periodSeconds: 60
# Additional volumes on the output Deployment definition.
volumes: []
# - name: foo
# secret:
# secretName: mysecret
# optional: false
# Additional volumeMounts on the output Deployment definition.
volumeMounts: []
# - name: foo
# mountPath: "/etc/foo"
# readOnly: true
nodeSelector: {}
tolerations: []
affinity: {}
db:
# Use an existing postgres server/cluster
useExisting: false
# How to connect to the existing postgres server/cluster
endpoint: localhost
database: litellm
url: postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME)
# Authenticate with a short-lived token instead of a password. useIAMAuth is
# AWS RDS / Aurora IAM database authentication (the pod needs AWS credentials,
# e.g. IRSA or EKS Pod Identity, and the database user needs GRANT rds_iam);
# useAzureEntraAuth is Azure Database for PostgreSQL with a Microsoft Entra ID
# identity (workload identity). Mutually exclusive, and both need
# useExisting: true with deployStandalone: false. The chart then sets
# IAM_TOKEN_DB_AUTH or AZURE_POSTGRESQL_AUTH, emits DATABASE_HOST, DATABASE_PORT,
# DATABASE_USERNAME and DATABASE_NAME and skips DATABASE_PASSWORD and db.url:
# the proxy, the migrations job and the collector sidecar each mint their own
# token and build DATABASE_URL from those parts. Set DATABASE_SSLMODE /
# DATABASE_SSLROOTCERT under envVars for TLS options you would otherwise put
# in db.url.
useIAMAuth: false
useAzureEntraAuth: false
# Postgres port, used to build the token-auth URL (password auth takes the
# port from db.url)
port: 5432
# Postgres schema for the LiteLLM tables, rendered as DATABASE_SCHEMA under
# token auth only (password auth carries it as ?schema= in db.url)
schema: ""
# Database user as a plain value. Empty means read it from
# db.secret.usernameKey. With token auth the user name is the only credential
# the chart needs, so setting it here removes the need for db.secret entirely.
username: ""
secret:
name: postgres
usernameKey: username
passwordKey: password
# Optional: when set, DATABASE_HOST will be sourced from this secret key instead of db.endpoint
endpointKey: ""
# Optional: when set, DATABASE_URL_READ_REPLICA will be sourced from this
# secret key instead of db.readReplicaUrl. Prefer this over the plain
# value: read-replica URLs typically embed credentials, and a value
# written to db.readReplicaUrl ends up visible in the rendered pod spec
# and the Helm release secret.
readReplicaUrlKey: ""
# Optional: when set, a DATABASE_READER_HOST env var is sourced from this
# secret key, so db.readReplicaUrl can compose the reader URL from
# individual secret components, e.g.
# postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_READER_HOST):5432/$(DATABASE_NAME)
# Use this when your secret store holds the bare reader hostname rather
# than a full connection URL. Only takes effect when readReplicaUrl is
# set; ignored when readReplicaUrlKey is set.
readReplicaEndpointKey: ""
# Optional read-replica routing. When set, the proxy sends read-only
# queries (find_*, count, group_by, query_raw/_first) to this URL while
# writes continue to go to db.url. Useful for Aurora-style clusters with
# separate reader/writer endpoints. Leave empty to keep single-DB behavior.
# When IAM_TOKEN_DB_AUTH is enabled, the reader URL is auto-refreshed
# alongside the writer (host/port/user/db are parsed from this URL once
# at startup; only the IAM token rotates).
#
# If the URL embeds credentials, prefer db.secret.readReplicaUrlKey over
# this field — the plain value is rendered into the pod spec and the
# Helm release secret. This field is intended for credential-less URLs
# only (e.g. when IAM_TOKEN_DB_AUTH supplies the token at runtime).
readReplicaUrl: ""
# In-container connection pool (PgBouncer, transaction mode) shared by every
# worker in the pod. Without it each --num_workers worker opens its own
# connection_limit connections to Postgres, so a pod's footprint against the
# database's connection ceiling is workers x connection_limit and grows with
# every replica. With it, the pod holds at most maxDbConnections upstream
# connections no matter how many workers run; the workers connect to the pool
# over loopback, with no extra network hop. Migrations still go straight to
# Postgres. Starting profile for numWorkers: 4 is maxDbConnections: 20, so
# a database with a 5000-connection ceiling fits roughly 200 replicas.
connectionPool:
enabled: false
maxDbConnections: 20
maxClientConn: 1000
# Use the Stackgres Helm chart to deploy an instance of a Stackgres cluster.
# The Stackgres Operator must already be installed within the target
# Kubernetes cluster.
# TODO: Stackgres deployment currently unsupported
useStackgresOperator: false
# Use the Postgres Helm chart to create a single node, stand alone postgres
# instance. See the "postgresql" top level key for additional configuration.
deployStandalone: true
# Lifecycle hooks for the LiteLLM container
#
# Prefer the native /health/drain preStop hook over a fixed `sleep`: it marks
# the pod NotReady and blocks only until in-flight requests actually finish
# (bounded by GRACEFUL_SHUTDOWN_TIMEOUT, default 30s), instead of always
# waiting the worst-case duration. After SIGTERM the server waits up to
# GRACEFUL_SHUTDOWN_TIMEOUT again for connections still open and then cancels
# them, so set terminationGracePeriodSeconds a few seconds above twice
# GRACEFUL_SHUTDOWN_TIMEOUT to leave room for teardown before SIGKILL.
#
# /health/drain is off by default; enable it with
# general_settings.enable_drain_endpoint: true. The kubelet calls preStop
# hooks without proxy credentials, so when the health port is reachable from
# other pods (the common case) also set
# general_settings.drain_endpoint_token (or the DRAIN_ENDPOINT_TOKEN env
# var) and send the same value on the X-Drain-Token header from the hook.
# Calls missing/wrong the token get a 401 and have no side effect.
# Example:
# lifecycle:
# preStop:
# httpGet:
# path: /health/drain
# port: 4000
# httpHeaders:
# - name: X-Drain-Token
# value: <same value as drain_endpoint_token>
lifecycle: {}
# Settings for Bitnami postgresql chart (if db.deployStandalone is true, ignored
# otherwise)
#
# Bitnami retired the versioned tags under docker.io/bitnami and republished the
# archived builds under docker.io/bitnamilegacy, so the subchart's own image
# defaults no longer resolve. The repository below points at the same build the
# subchart was released with, which keeps the on-disk data directory layout
# identical for existing installs.
#
# Keep the tag pinned. docker.io/bitnami still publishes a floating `latest`,
# and starting a newer PostgreSQL major against an existing data directory
# leaves the server refusing to boot ("database files are incompatible with
# server") with no way back other than a dump taken beforehand. Crossing a major
# version is a dump-and-restore, not an image bump. The chart refuses to render
# an unpinned tag for this reason
postgresql:
architecture: standalone
image:
repository: bitnamilegacy/postgresql
tag: 16.2.0-debian-12-r6
volumePermissions:
image:
repository: bitnamilegacy/os-shell
tag: 12-debian-12-r16
metrics:
image:
repository: bitnamilegacy/postgres-exporter
tag: 0.15.0-debian-12-r14
auth:
username: litellm
database: litellm
# You should override these on the helm command line with
# `--set postgresql.auth.postgres-password=<some good password>,postgresql.auth.password=<some good password>`
password: NoTaGrEaTpAsSwOrD
postgres-password: NoTaGrEaTpAsSwOrD
# A secret is created by this chart (litellm-helm) with the credentials that
# the new Postgres instance should use.
# existingSecret: ""
# secretKeys:
# userPasswordKey: password
# Redis is the proxy's coordination store: cross-pod tpm/rpm rate limits, spend
# tracking, and the pod lock manager. Enabling this deploys the bundled Redis
# subchart, wires REDIS_HOST / REDIS_PORT / REDIS_PASSWORD into the proxy, and
# renders a `general_settings.coordination_redis` block into the proxy config.
#
# To point at an existing Redis instead, leave `enabled: false` and pass a
# secret for REDIS_HOST, REDIS_PORT, REDIS_PASSWORD or REDIS_URL; the proxy
# falls back to those env vars for coordination. Set `cache: true` in the proxy
# config only if you also want LLM response caching, which is independent of
# coordination
#
# When `redis.sentinel.enabled` is set, the coordination block is rendered with
# `sentinel_nodes` and `service_name` (from `redis.sentinel.masterSet`) instead
# of host/port, because a plain Redis client cannot talk to the sentinel port
#
# The image repositories carry the same bitnamilegacy repoint as postgresql
# above; the versioned tags the subchart ships with are gone from
# docker.io/bitnami
redis:
enabled: false
architecture: standalone
image:
repository: bitnamilegacy/redis
tag: 7.2.4-debian-12-r9
sentinel:
image:
repository: bitnamilegacy/redis-sentinel
tag: 7.2.4-debian-12-r7
metrics:
image:
repository: bitnamilegacy/redis-exporter
tag: 1.58.0-debian-12-r4
volumePermissions:
image:
repository: bitnamilegacy/os-shell
tag: 12-debian-12-r16
sysctl:
image:
repository: bitnamilegacy/os-shell
tag: 12-debian-12-r16
kubectl:
image:
repository: bitnamilegacy/kubectl
tag: 1.29.2-debian-12-r3
coordination:
# Set to false to keep the bundled Redis for response caching only and leave
# `general_settings.coordination_redis` out of the rendered config. A
# `coordination_redis` block you define yourself in `proxy_config` always wins
enabled: true
# Prisma migration job settings
migrationJob:
enabled: true # Enable or disable the schema migration Job
backoffLimit: 4 # Backoff limit for Job restarts
# Wall-clock budget for the whole Job, shared across every `backoffLimit`
# retry rather than granted per attempt. Without it a migration that blocks
# on the database never fails, and when the Helm hook is enabled the release
# waits on it forever: `helm upgrade` and any GitOps controller driving it
# stop reconciling the whole chart until someone deletes the Job by hand.
# Set to null to opt out and restore the unbounded behaviour.
activeDeadlineSeconds: 1800
# Optional service account for the migration job.
# Only used when migrationJob.hooks.helm.enabled=true and serviceAccount.create=true.
# In that case, pre-install/pre-upgrade hooks run before normal resources, so this defaults to "default".
serviceAccountName: ""
annotations: {}
jobLabels: {} # Custom labels for the Job metadata
jobAnnotations: {} # Custom annotations for the Job metadata
podLabels: {} # Custom labels for the Job pod template
command: [] # Override container command (defaults to ["python", "litellm/proxy/prisma_migration.py"])
args: [] # Override container args
ttlSecondsAfterFinished: 120
resources: {}
# Unset by default. This job runs the database migration and exits, so it does not
# need the steady-state headroom the proxy does; size it from your own migration
# runs rather than from the proxy figures above.
extraContainers: []
extraInitContainers: []
# Hook configuration
hooks:
argocd:
enabled: true
helm:
enabled: false
# Log level for the litellm proxy (sets LITELLM_LOG in the deployment env).
# Rendered as a direct `env:` entry, which in Kubernetes takes precedence over
# any `envFrom:` source. If you currently source LITELLM_LOG from an
# environmentSecret or environmentConfigMap, set `logLevel: ""` here to
# disable injection — otherwise this value silently overrides your secret /
# configmap entry.
#
# Setting LITELLM_LOG inside `envVars:` below also wins: the template skips
# this injection entirely when envVars already defines LITELLM_LOG.
logLevel: INFO
# Additional environment variables to be added to the deployment as a map of key-value pairs
envVars: {}
# USE_DDTRACE: "true"
# Additional environment variables to be added to the deployment as a list of k8s env vars
extraEnvVars: {}
# if you want to override the container command, you can do so here
command: {}
# if you want to override the container args, you can do so here
args: {}
# - name: EXTRA_ENV_VAR
# value: EXTRA_ENV_VAR_VALUE
# Additional Kubernetes resources to deploy with litellm
extraResources: []
# - apiVersion: v1
# kind: ConfigMap
# metadata:
# name: my-extra-config
# data:
# foo: bar
# Pod Disruption Budget
pdb:
enabled: false
# Set exactly one of the following. If both are set, minAvailable takes precedence.
minAvailable: null # e.g. "50%" or 1
maxUnavailable: null # e.g. 1 or "20%"
annotations: {}
labels: {}
serviceMonitor:
enabled: false
labels:
{}
# test: test
annotations:
{}
# kubernetes.io/test: test
interval: 15s
scrapeTimeout: 10s
relabelings: []
# - targetLabel: __meta_kubernetes_pod_node_name
# replacement: $1
# action: replace
namespaceSelector:
matchNames: []
# - test-namespace
lens:
enabled: false
url: ""
publicUrl: ""
service:
name: ""
port: 4318
gatewaySecret:
name: ""
key: gateway-secret
serviceTokenSecret:
name: ""
key: service-token

20
helm/litellm/.helmignore Normal file
View file

@ -0,0 +1,20 @@
# Patterns to ignore when building packages.
.DS_Store
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
*.swp
*.bak
*.tmp
*.orig
*~
.project
.idea/
*.tmproj
.vscode/
/tests/
/ci/

6
helm/litellm/Chart.yaml
View file

@ -1,8 +1,8 @@
apiVersion: v2
name: litellm
description: LiteLLM componentized — gateway, UI backend, and UI as separate services
description: Deploys LiteLLM from one image, either componentized (gateway, UI backend, and UI as separate Deployments) or as a single monolith Deployment
type: application
version: 0.1.1
appVersion: "0.1.0"
version: 1.0.0
appVersion: "1.104.0"
annotations:
org.opencontainers.image.source: "https://github.com/BerriAI/litellm"

174
helm/litellm/README.md
View file

@ -1,6 +1,51 @@
# LiteLLM componentized chart
# litellm Helm chart
This chart deploys separate gateway, backend and UI services
Deploys [LiteLLM](https://github.com/BerriAI/litellm) from one image, `ghcr.io/berriai/litellm` (mirrored at `docker.litellm.ai/berriai/litellm`), in one of two layouts:
- componentized (default): the `gateway` (LLM data plane, port 4000), the `backend` (management API, port 4001) and the `ui` (static dashboard, port 3000) each get their own Deployment, Service, HPA and PDB so they scale independently
- monolith: `monolith.enabled: true` renders one Deployment and Service running the full proxy (`args: [proxy, ...]`), which serves the gateway routes, the management routes and the Admin UI from a single process
Both layouts run the same image. The image entrypoint dispatches on the first container argument (`proxy`, `gateway`, `backend`, `ui`, `migrations`, `metrics`, `collector`), so the chart only ever sets `args` and never `command`
## Requirements
Kubernetes 1.25+ and Helm 3.8+. Bring your own PostgreSQL (`database.writer.*`) and, optionally, Redis (`redis.*`). The proxy also needs a master key, either an existing Secret named by `masterKey.secretName` or one the chart generates with `masterKey.generate: true` and `masterKey.secretName: ""`
## Install
```bash
kubectl create secret generic litellm-master-key-secret --from-literal=master-key=sk-change-me
helm install litellm helm/litellm \
--set database.writer.host=postgres.example.com \
--set database.writer.dbname=litellm \
--set database.writer.passwordSecret.name=litellm-db-secret
```
The image tag defaults to the chart's `appVersion`. Override it with `image.tag`, or pin the bytes with `image.digest`, which renders as `repository:tag@digest`
## Monolith quickstart
```bash
helm install litellm helm/litellm \
--set monolith.enabled=true \
--set masterKey.secretName="" \
--set masterKey.generate=true \
--set database.writer.host=postgres.example.com \
--set database.writer.dbname=litellm \
--set database.writer.passwordSecret.name=litellm-db-secret
kubectl port-forward svc/litellm-litellm 4000:4000
```
With `monolith.enabled: true`:
- one Deployment and one Service named `<release>-litellm` render, running `args: [proxy, --port, 4000, --config, /app/config/config.yaml]` plus `monolith.extraArgs`
- the gateway, backend and ui Deployments, Services, HPAs, PDBs and ServiceMonitors are not rendered, whatever `gateway.enabled`, `backend.enabled` and `ui.enabled` say
- every `gateway.*` value configures the monolith pod: `config`, `numWorkers`, `resources`, probes, `securityContext`, `hpa`, `keda`, `pdb`, `metricsServer`, `collector`, `volumes`, `extraEnv`, scheduling. The monolith runs as `serviceAccounts.gateway`
- every `backend.*` and `ui.*` value is ignored
- the Ingress sends every path, built in or from `ingress.extraPaths`, to the monolith Service
- the migrations Job renders exactly as in componentized mode
## Lens
This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render
@ -22,39 +67,114 @@ backend:
gatewayUrl: https://gateway.example.com
```
## Enterprise billing metrics
Billing metrics use Cognito bearer authentication. Create the Secret with
`client-id` and `client-secret` keys, then enable billing metrics:
## Testing
```bash
kubectl create secret generic litellm-billing-metrics --from-literal=client-id=... --from-literal=client-secret=...
helm lint helm/litellm
helm unittest -f 'tests/*.yaml' helm/litellm
helm test <release> --logs
```
## Migrating from the litellm-helm chart
The `litellm-helm` chart (`oci://ghcr.io/berriai/litellm-helm`) is retired; its published packages stay available for a grace period. Its flat values described one monolith Deployment, so the equivalent install here is `monolith.enabled: true` with the values moved under `gateway.*`. The per-component `gateway.image`, `backend.image`, `ui.image` and `migrations.image` blocks of earlier `helm/litellm` versions are gone too: the chart is a major bump to `1.0.0` and every container uses the top-level `image`
| litellm-helm value | litellm value |
|---|---|
| (implicit single Deployment) | `monolith.enabled: true` |
| `image.repository` / `image.tag` / `image.pullPolicy` | `image.repository` / `image.tag` / `image.pullPolicy` (`image.digest` is new) |
| `replicaCount` | `gateway.replicaCount` |
| `args` | `monolith.extraArgs` (appended after the chart's proxy arguments). Drop any `--port` argument and its value so the proxy keeps listening on port 4000, matching the container port and health checks. Set `gateway.service.port` to change the external Service port |
| `command` | removed: the image entrypoint dispatcher must stay in place |
| `proxy_config` | `gateway.config.proxy_config` |
| `proxyConfigMap.create: false` + `proxyConfigMap.name` | Set `gateway.config.create: false`, mount your ConfigMap with `gateway.volumes` / `gateway.volumeMounts`, and pass `--config` followed by the mounted file path in `monolith.extraArgs` |
| `masterkeySecretName` / `masterkeySecretKey` | `masterKey.secretName` / `masterKey.secretKey`. The default Secret field changes from `masterkey` to `master-key`. When reusing a Secret with the old default field, explicitly set `masterKey.secretKey: masterkey` |
| `masterkeySecretName: ""` (auto generated Secret) | `masterKey.generate: true` with `masterKey.secretName: ""` (creates a new key; copy the old key first, see below, to keep existing credentials valid) |
| `db.useExisting`, `db.endpoint`, `db.database` | `database.writer.host`, `database.writer.port`, `database.writer.dbname` |
| `db.useIAMAuth` / `db.useAzureEntraAuth` | `database.writer.useIAMAuth` / `database.writer.useAzureEntraAuth` (mutually exclusive). Both token-auth modes still require a username in the Secret referenced by `database.writer.passwordSecret.name` and `usernameKey` |
| `db.schema` | `database.writer.schema` |
| `db.username` | Store the username in the Secret referenced by `database.writer.passwordSecret.name`, under its `usernameKey`. There is no literal username value in the new chart |
| `db.secret.name` / `db.secret.usernameKey` / `db.secret.passwordKey` | `database.writer.passwordSecret.name` / `usernameKey` / `passwordKey` |
| `db.secret.endpointKey` (writer host read from a Secret) | `database.writer.passwordSecret.hostKey` with `database.writer.host` left empty |
| `db.url` (custom URL template) | `database.writer.passwordSecret.urlKey` pointing at a Secret key that holds the complete writer URL, with the other `database.writer.*` connection values left empty |
| `db.readReplicaUrl` (credential-less reader URL) | `database.reader.host`, `database.reader.port`, `database.reader.dbname`, with `database.reader.passwordSecret.*` for its credentials (or `database.reader.useIAMAuth: true`) |
| `db.secret.readReplicaEndpointKey` (reader host read from a Secret) | `database.reader.passwordSecret.hostKey` with `database.reader.host` left empty |
| `db.secret.readReplicaUrlKey` (full reader URL in a Secret) | `database.reader.passwordSecret.urlKey` with the other `database.reader.*` connection values left empty |
| `db.connectionPool.*` | `database.connectionPool.*` |
| `db.deployStandalone: true` / `postgresql.*` | removed: the chart ships no PostgreSQL, point `database.writer.*` at your own |
| `redis.enabled: true` (bundled) | removed: the chart ships no Redis, point `redis.host` at your own |
| external Redis via `envVars` | `redis.host`, `redis.port`, `redis.passwordSecret.*`, `redis.cluster` |
| `envVars` / `extraEnvVars` | `gateway.extraEnv` (list of `name` / `value` or `valueFrom` entries). The retired chart also passed these to the migration Job, so copy any it needs, such as the AWS region and credentials for IAM database auth, into `migrationJob.extraEnv` too |
| `environmentSecrets` | `gateway.envSecrets` |
| `environmentConfigMaps` | `gateway.envConfigMaps` |
| `logLevel` | `gateway.logLevel` |
| `resources` | `gateway.resources` |
| `livenessProbe` / `readinessProbe` / `startupProbe` | `gateway.livenessProbe` / `gateway.readinessProbe` / `gateway.startupProbe`, written as full Kubernetes probes: the old `path: /health/readiness` becomes `httpGet: { path: /health/readiness, port: http }`, the timing fields (`initialDelaySeconds`, `periodSeconds`, `timeoutSeconds`, `successThreshold`, `failureThreshold`) carry over unchanged |
| `securityContext` / `podSecurityContext` | `gateway.securityContext` / `gateway.podSecurityContext`. With `readOnlyRootFilesystem: true` the retired chart added `emptyDir` mounts at `/tmp`, `/.cache` and `/.npm`; this chart does not, so add them through `gateway.volumes` / `gateway.volumeMounts`. The retired chart applied both to the migration Job as well; set `migrationJob.securityContext` / `migrationJob.podSecurityContext` to keep the Job under the same policy |
| `service.*` | `gateway.service.*` |
| `ingress.className`, `ingress.annotations`, `ingress.tls` | same keys (routes to the monolith Service in monolith mode) |
| `ingress.hosts[].host` | `ingress.host`, one host per release; the built-in paths are rendered for that host, so `ingress.hosts[].paths` has no equivalent. For more than one hostname, add a second Ingress through `extraResources` or point a DNS alias at the one host |
| `autoscaling.*` | `gateway.hpa.*`. Defaults change from 100 to 10 maximum replicas and from 60% to 70% CPU utilization, with an 80% memory target now enabled. To keep the old scaling behavior, set `gateway.hpa.maxReplicas: 100`, `gateway.hpa.targetCPUUtilizationPercentage: 60` and `gateway.hpa.targetMemoryUtilizationPercentage: 0` |
| `keda.*` | `gateway.keda.*` with these renames: `minReplicas` / `maxReplicas` are `minReplicaCount` / `maxReplicaCount` (the new default maximum is 10, the old one was 100), `behavior` moves to `advanced.horizontalPodAutoscalerConfig.behavior`, `restoreToOriginalReplicaCount` moves to `advanced.restoreToOriginalReplicaCount`, and `keda.prometheus.requestsPerSecond` / `tokensPerSecond` are `gateway.keda.prometheus.targetRequestsPerSecond` / `targetTokensPerSecond`. `pollingInterval`, `cooldownPeriod`, `fallback`, `triggers` and `scaledObject.annotations` (for example KEDA's pause annotations) keep their names |
| `numWorkers` | `gateway.numWorkers` |
| `pdb.*` | `gateway.pdb.*`, except `labels` and `annotations` are no longer supported. To retain custom PDB metadata, set `gateway.pdb.enabled: false` and define the PDB through `extraResources` |
| `metricsServer.*` | `gateway.metricsServer.*` |
| `serviceMonitor.*` | `gateway.serviceMonitor.*` (`labels`, `annotations`, `interval`, `scrapeTimeout`, `relabelings`, `namespaceSelector.matchNames`); it now requires `gateway.metricsServer.enabled` |
| `collector.*` | `gateway.collector.*`, except `command` is no longer supported and `scaleOnProxyContainerCpu` becomes `scaleOnGatewayContainerCpu`. Its default changes from `false` to `true`. Set `gateway.collector.scaleOnGatewayContainerCpu: false` to keep pod-wide CPU scaling. For a custom collector command, disable the built-in sidecar with `gateway.collector.enabled: false` and define it through `gateway.extraContainers`, with its environment, volumes and volume mounts configured explicitly |
| `billingMetrics.*` | `billingMetrics.*` |
| `migrationJob.*` | `migrationJob.*` |
| `volumes` / `volumeMounts` | `gateway.volumes` / `gateway.volumeMounts`. The retired chart also mounted these in the migration Job; copy any mounts the Job needs, such as a database CA certificate, to `migrationJob.volumes` / `migrationJob.volumeMounts` too |
| `extraContainers` / `extraInitContainers` | `gateway.extraContainers` / `gateway.extraInitContainers` |
| `lifecycle` | `gateway.lifecycle` |
| `strategy` | `gateway.strategy` |
| `deploymentAnnotations` / `deploymentLabels` / `deploymentMinReadySeconds` | `gateway.deploymentAnnotations` / `gateway.deploymentLabels` / `gateway.minReadySeconds` |
| `podAnnotations` / `podLabels` | `gateway.podAnnotations` / `gateway.podLabels`. The retired chart ran `podAnnotations` through `tpl`; this chart renders them verbatim, so a copied template such as `checksum/config: {{ ... }}` stays literal text. Resolve it before install instead, e.g. `--set-string gateway.podAnnotations.checksum/config=$(kubectl get configmap my-config -o yaml \| sha256sum \| cut -d' ' -f1)`. The chart's own proxy ConfigMap already rolls the pods when `gateway.config` changes. The retired chart also put `podLabels` on the migration Job pod; copy any the Job needs, such as a network-policy selector, to `migrationJob.podLabels` |
| `nodeSelector` / `tolerations` / `affinity` / `topologySpreadConstraints` | `gateway.nodeSelector` / `gateway.tolerations` / `gateway.affinity` / `gateway.topologySpreadConstraints`. The retired chart also applied `nodeSelector`, `tolerations` and `affinity` to the migration Job; copy them to `migrationJob.nodeSelector` / `migrationJob.tolerations` / `migrationJob.affinity` too, or a Job that needs tainted or zoned nodes to reach the database stays Pending and blocks the install |
| `terminationGracePeriodSeconds` | `gateway.terminationGracePeriodSeconds` |
| `serviceAccount.*` | `serviceAccounts.gateway.*`. The retired chart also shared an existing ServiceAccount with migrations; set `migrationJob.serviceAccountName` to that existing account too when the Job needs its permissions, such as for IAM database auth |
| `lens.*` | `lens.*`. Deploy Lens from its own chart. The removed `lensWorker.*` values fail the render when enabled |
| `extraResources` | `extraResources` |
| `nameOverride: "litellm"` | `nameOverride: ""` (the chart name is already `litellm`) |
| `fullnameOverride` | `fullnameOverride`. The new default always includes the chart-name suffix, even when the release name already contains it. Set this to the existing Service name to retain its DNS name |
A minimal migration:
```yaml
billingMetrics:
monolith:
enabled: true
masterKey:
secretName: litellm-master-key-secret
database:
writer:
host: postgres.example.com
dbname: litellm
passwordSecret:
name: litellm-db-secret
usernameKey: username
passwordKey: password
gateway:
replicaCount: 2
config:
proxy_config:
model_list:
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
envSecrets:
- litellm-provider-keys
```
The chart requires an HTTPS endpoint, an HTTPS token URL, and the Secret name
when billing metrics are enabled
The monolith Service is named `<release>-litellm`. To retain a different old Service name, set `fullnameOverride` to that existing name. The generated Secret in this chart has `helm.sh/resource-policy: keep` and is reused across upgrades of this chart. The retired chart stored its generated key under a different Secret and data key, and `helm uninstall` deletes that Secret, so copy it into a Secret you own before uninstalling the old release to keep existing credentials valid. Run the following in the old release's namespace, replacing `OLD_MASTER_KEY_SECRET` with the actual existing Secret name. For a release named `litellm` without a `fullnameOverride`, that name is `litellm-masterkey`:
## Routes to another Service
For a shared hostname, `ingress.extraPaths` can route to a Service in the same namespace using `serviceName` and `servicePort`. The port can be a number or a named Service port. Omit `service` when using these fields
```yaml
ingress:
enabled: true
className: alb
host: gateway.example.com
extraPaths:
- path: /moyai
pathType: Prefix
serviceName: moyai-ingress
servicePort: 8080
```bash
set -euo pipefail
OLD_MASTER_KEY_SECRET=litellm-masterkey
old_master_key="$(kubectl get secret "$OLD_MASTER_KEY_SECRET" -o jsonpath='{.data.masterkey}' | base64 -d)"
test -n "$old_master_key"
kubectl create secret generic litellm-master-key-secret \
--from-literal=master-key="$old_master_key"
unset old_master_key
```
The Service and application must already support the path prefix. The chart adds the path to the existing Ingress and leaves its built-in routes in place. It does not create the target Service or rewrite requests
For AWS ALB, configure the target Service's `alb.ingress.kubernetes.io/healthcheck-path` annotation when the application needs a different health check from LiteLLM. Existing `service` aliases (`gateway`, `backend`, `ui`) remain supported
Then set `masterKey.secretName: litellm-master-key-secret`, as in the minimal example above. Releases that set `masterkeySecretName` already point `masterKey.secretName` and `masterKey.secretKey` at that Secret

58
helm/litellm/ci/postgres.yaml Normal file
View file

@ -0,0 +1,58 @@
# Throwaway PostgreSQL for the CircleCI helm_chart_testing job. Applied with
# kubectl before `helm install`; the chart itself ships no database.
apiVersion: v1
kind: Secret
metadata:
name: litellm-ci-postgres
type: Opaque
stringData:
username: litellm
password: litellm-ci
---
apiVersion: v1
kind: Service
metadata:
name: litellm-ci-postgres
spec:
selector:
app: litellm-ci-postgres
ports:
- port: 5432
targetPort: 5432
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: litellm-ci-postgres
spec:
replicas: 1
selector:
matchLabels:
app: litellm-ci-postgres
template:
metadata:
labels:
app: litellm-ci-postgres
spec:
containers:
- name: postgres
image: postgres:16.10-alpine@sha256:029660641a0cfc575b14f336ba448fb8a75fd595d42e1fa316b9fb4378742297
ports:
- containerPort: 5432
env:
- name: POSTGRES_DB
value: litellm
- name: POSTGRES_USER
valueFrom:
secretKeyRef:
name: litellm-ci-postgres
key: username
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: litellm-ci-postgres
key: password
readinessProbe:
exec:
command: [pg_isready, -U, litellm, -d, litellm]
periodSeconds: 2

34
helm/litellm/ci/test-values.yaml Normal file
View file

@ -0,0 +1,34 @@
# Values for the CircleCI helm_chart_testing job: a monolith install on a
# kind cluster from the image built by the pipeline (image.repository / tag /
# pullPolicy are passed with --set), backed by the throwaway PostgreSQL in
# ci/postgres.yaml so the migrations Job and the proxy have a database.
monolith:
enabled: true
masterKey:
secretName: ""
generate: true
gateway:
replicaCount: 1
hpa:
enabled: false
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
memory: 2Gi
config:
create: true
proxy_config:
general_settings: {}
database:
writer:
host: litellm-ci-postgres
dbname: litellm
passwordSecret:
name: litellm-ci-postgres
usernameKey: username
passwordKey: password

43
helm/litellm/templates/NOTES.txt
View file

@ -1,4 +1,16 @@
LiteLLM componentized — release {{ .Release.Name }} in namespace {{ .Release.Namespace }}.
{{- if .Values.monolith.enabled }}
LiteLLM monolith: release {{ .Release.Name }} in namespace {{ .Release.Namespace }}, image {{ include "litellm.image" . }}
One Deployment runs the full proxy (gateway routes, management routes and the Admin UI):
- proxy : Service {{ include "litellm.workload.fullname" . }} on port {{ .Values.gateway.service.port }}
Port-forward:
kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "litellm.workload.fullname" . }} {{ .Values.gateway.service.port }}
The monolith takes its configuration from the gateway.* values (config, resources, probes, hpa, pdb, keda,
metricsServer, collector, volumes, env, scheduling). backend.* and ui.* are ignored while monolith.enabled is true
{{- else }}
LiteLLM componentized: release {{ .Release.Name }} in namespace {{ .Release.Namespace }}, image {{ include "litellm.image" . }}
Components:
{{- if .Values.gateway.enabled }}
@ -16,43 +28,48 @@ Port-forward examples:
kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "litellm.backend.fullname" . }} {{ .Values.backend.service.port }}
kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "litellm.ui.fullname" . }} {{ .Values.ui.service.port }}
Set monolith.enabled=true to run everything as one Deployment instead
{{- end }}
Reminders:
- Sensitive values come from Secret references only. Before installing, set:
- masterKey.secretName (Secret with the proxy master key)
- masterKey.secretName (Secret with the proxy master key, or
masterKey.generate: true with secretName "" to let the chart create one)
- database.writer.{host,port,dbname} (writer connection pieces)
- database.writer.passwordSecret.{name,usernameKey,passwordKey}
(Secret holding the writer DB username + password)
- database.writer.passwordSecret.{hostKey,urlKey} (optional, read writer host or full URL from that Secret)
- database.writer.useIAMAuth: true (optional — chart sets IAM_TOKEN_DB_AUTH=true and
- database.writer.useIAMAuth: true (optional: chart sets IAM_TOKEN_DB_AUTH=true and
omits DATABASE_PASSWORD / DATABASE_URL so the proxy
mints the URL from an IAM token at startup)
- database.reader.host / database.reader.passwordSecret.{hostKey,urlKey}
(optional, enables read-replica routing. The Secret
name is required for hostKey, urlKey, and for host
with password auth)
- database.reader.useIAMAuth: true (optional, requires database.writer.useIAMAuth: true —
- database.reader.useIAMAuth: true (optional, requires database.writer.useIAMAuth: true;
chart emits DATABASE_*_READ_REPLICA env vars and
omits DATABASE_PASSWORD_READ_REPLICA /
DATABASE_URL_READ_REPLICA so the proxy mints the
reader URL from an IAM token at startup)
- redis.passwordSecret.name (optional — set when redis.host is provided and the
- redis.host (optional: the proxy's coordination Redis)
- redis.passwordSecret.name (optional: set when redis.host is provided and the
cache requires auth)
- redis.passwordSecret.{hostKey,urlKey} (optional, read Redis host or full URL from that Secret)
(the Secret name is required when either key is set)
- redis.cluster: true (optional — chart sets REDIS_CLUSTER_NODES from
- redis.cluster: true (optional: chart sets REDIS_CLUSTER_NODES from
redis.host / redis.port so the proxy's Cache()
constructs a RedisClusterCache; the cluster client
discovers remaining nodes from CLUSTER SLOTS)
- Per-component extras (gateway / backend / ui):
- {component}.extraEnv / envConfigMaps / envSecrets (the latter two are lists of resource names →
envFrom configMapRef / secretRef)
- Per-component extras (gateway / backend / ui; gateway.* also drives the monolith):
- {component}.extraEnv / envConfigMaps / envSecrets (the latter two are lists of resource names rendered
as envFrom configMapRef / secretRef)
- {component}.logLevel (renders as LITELLM_LOG)
- gateway.config.proxy_config (rendered into a ConfigMap and mounted at
/app/config/config.yaml; gateway reads it via
CONFIG_FILE_PATH)
/app/config/config.yaml; read via CONFIG_FILE_PATH)
- {component}.pdb.{enabled,minAvailable,maxUnavailable} (per-component PodDisruptionBudget; disabled by
default — with hpa.minReplicas of 1, minAvailable: 1
default: with hpa.minReplicas of 1, minAvailable: 1
would block node drains)
- {component}.topologySpreadConstraints (standard k8s list, e.g. spread replicas across
topology.kubernetes.io/zone)
- Enable ingress.enabled=true to dispatch / → ui, gateway data-plane prefixes → gateway, and the catch-all → backend.
- Enable ingress.enabled=true to dispatch / to ui, gateway data-plane prefixes to gateway and the catch-all to
backend; in monolith mode every path goes to the monolith Service

172
helm/litellm/templates/_helpers.tpl
View file

@ -1,11 +1,116 @@
{{/*
Common naming + label helpers shared by gateway, backend, and ui templates.
Common naming + label helpers shared by gateway, backend, ui, and monolith templates.
*/}}
{{- define "litellm.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{/*
The one image reference every container in the chart uses. `repository:tag`,
or `repository:tag@digest` when image.digest is set; the tag falls back to
the chart appVersion.
*/}}
{{- define "litellm.image" -}}
{{- $tag := .Values.image.tag | default .Chart.AppVersion -}}
{{- if .Values.image.digest -}}
{{- printf "%s:%s@%s" .Values.image.repository $tag .Values.image.digest -}}
{{- else -}}
{{- printf "%s:%s" .Values.image.repository $tag -}}
{{- end -}}
{{- end -}}
{{/*
Componentized mode renders the gateway / backend / ui Deployments only when
monolith mode is off.
*/}}
{{- define "litellm.gateway.render" -}}
{{- if and .Values.gateway.enabled (not .Values.monolith.enabled) -}}true{{- end -}}
{{- end -}}
{{- define "litellm.backend.render" -}}
{{- if and .Values.backend.enabled (not .Values.monolith.enabled) -}}true{{- end -}}
{{- end -}}
{{- define "litellm.ui.render" -}}
{{- if and .Values.ui.enabled (not .Values.monolith.enabled) -}}true{{- end -}}
{{- end -}}
{{/*
The proxy Deployment (monolith mode) and the gateway Deployment
(componentized mode) are the same pod spec fed by .Values.gateway, so the
templates that serve both (HPA, KEDA, PDB, metrics Service, ServiceMonitor)
resolve their name, component label and selector through these.
*/}}
{{- define "litellm.proxy.fullname" -}}
{{- include "litellm.fullname" . -}}
{{- end -}}
{{- define "litellm.proxy.selectorLabels" -}}
app.kubernetes.io/name: {{ include "litellm.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: proxy
{{- end -}}
{{- define "litellm.workload.componentName" -}}
{{- if .Values.monolith.enabled -}}proxy{{- else -}}gateway{{- end -}}
{{- end -}}
{{- define "litellm.workload.fullname" -}}
{{- if .Values.monolith.enabled -}}
{{- include "litellm.proxy.fullname" . -}}
{{- else -}}
{{- include "litellm.gateway.fullname" . -}}
{{- end -}}
{{- end -}}
{{- define "litellm.workload.selectorLabels" -}}
{{- if .Values.monolith.enabled -}}
{{- include "litellm.proxy.selectorLabels" . -}}
{{- else -}}
{{- include "litellm.gateway.selectorLabels" . -}}
{{- end -}}
{{- end -}}
{{- define "litellm.workload.render" -}}
{{- if or .Values.monolith.enabled .Values.gateway.enabled -}}true{{- end -}}
{{- end -}}
{{/*
Container arguments for the proxy (monolith) container. The image entrypoint
dispatches on the first argument.
*/}}
{{- define "litellm.proxy.args" -}}
- proxy
- --port
- "4000"
{{- if .Values.gateway.config.create }}
- --config
- /app/config/config.yaml
{{- end }}
{{- with .Values.monolith.extraArgs }}
{{ toYaml . }}
{{- end }}
{{- end -}}
{{/*
Master key Secret reference. `masterKey.secretName` when set, otherwise the
Secret the chart generates when `masterKey.generate` is true.
*/}}
{{- define "litellm.masterKey.generatedSecretName" -}}
{{- printf "%s-masterkey" (include "litellm.fullname" .) -}}
{{- end -}}
{{- define "litellm.masterKey.secretName" -}}
{{- if .Values.masterKey.secretName -}}
{{- .Values.masterKey.secretName -}}
{{- else if .Values.masterKey.generate -}}
{{- include "litellm.masterKey.generatedSecretName" . -}}
{{- else -}}
{{- fail "masterKey.secretName is required (the chart never accepts an inline master key); set it to an existing Secret or set masterKey.generate: true" -}}
{{- end -}}
{{- end -}}
{{- define "litellm.fullname" -}}
{{- if .Values.fullnameOverride -}}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
@ -108,6 +213,7 @@ Each component (gateway, backend, ui) has its own SA config under
.Values.serviceAccounts.<component>. When `create` is true and `name` is
empty the chart defaults to "<release>-litellm-<component>". When `create`
is false the chart uses the provided name, or the namespace `default` SA.
The monolith pod runs as the gateway ServiceAccount.
*/}}
{{- define "litellm.gateway.serviceAccountName" -}}
{{- if .Values.serviceAccounts.gateway.create -}}
@ -186,6 +292,22 @@ by the controller rather than declared, so nothing there can collide.
{{- toYaml .podLabels }}
{{- end -}}
{{/*
LITELLM_MASTER_KEY, and PROXY_MASTER_KEY for configs copied from litellm-helm,
for the app containers only. The migrations Job runs as a
pre-install hook, before the chart's generated master key Secret exists, and
migrations/run.py never reads the key, so the Job must not reference it.
*/}}
{{- define "litellm.masterKeyEnv" -}}
{{- range list "LITELLM_MASTER_KEY" "PROXY_MASTER_KEY" }}
- name: {{ . }}
valueFrom:
secretKeyRef:
name: {{ include "litellm.masterKey.secretName" $ }}
key: {{ $.Values.masterKey.secretKey | default "master-key" }}
{{- end }}
{{- end -}}
{{/*
Master-key + database + redis env block — shared by gateway, backend, and the
migrations Job.
@ -229,11 +351,6 @@ IAM_TOKEN_DB_AUTH / AZURE_POSTGRESQL_AUTH toggle that only the writer sets.
{{- define "litellm.serverEnv" -}}
{{- $root := .root -}}
{{- $component := .component -}}
- name: LITELLM_MASTER_KEY
valueFrom:
secretKeyRef:
name: {{ required "masterKey.secretName is required (the chart no longer accepts an inline master key)" $root.Values.masterKey.secretName }}
key: {{ $root.Values.masterKey.secretKey | default "master-key" }}
{{- if $component.logLevel }}
- name: LITELLM_LOG
value: {{ $component.logLevel | quote }}
@ -276,6 +393,11 @@ IAM_TOKEN_DB_AUTH / AZURE_POSTGRESQL_AUTH toggle that only the writer sets.
secretKeyRef:
name: {{ required "database.writer.passwordSecret.name is required" .passwordSecret.name }}
key: {{ .passwordSecret.usernameKey | default "username" }}
- name: DATABASE_USERNAME
valueFrom:
secretKeyRef:
name: {{ .passwordSecret.name }}
key: {{ .passwordSecret.usernameKey | default "username" }}
- name: DATABASE_NAME
value: {{ required "database.writer.dbname is required" .dbname | quote }}
{{- if .schema }}
@ -473,7 +595,7 @@ harmless no-op for the Job and authoritative for the app pods.
{{- end -}}
{{/*
In-container PgBouncer env for the gateway container. Under IAM or Entra auth the pooler mints and renews the database token itself.
In-container PgBouncer env for the gateway and proxy containers. Under IAM or Entra auth the pooler mints and renews the database token itself.
*/}}
{{- define "litellm.connectionPoolEnv" -}}
{{- with .Values.database.connectionPool -}}
@ -510,7 +632,7 @@ than silently replaced by the fallback.
{{- $max := $component.pdb.maxUnavailable -}}
{{- $minSet := not (or (kindIs "invalid" $min) (eq (printf "%v" $min) "")) -}}
{{- $maxSet := not (or (kindIs "invalid" $max) (eq (printf "%v" $max) "")) -}}
{{- if and $component.enabled $component.pdb $component.pdb.enabled }}
{{- if and .enabled $component.pdb $component.pdb.enabled }}
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
@ -568,6 +690,16 @@ ImplementationSpecific
{{- end -}}
{{- end -}}
{{/*
Service spec fields shared by the component and monolith Services.
Invoke with the component's `service` dict.
*/}}
{{- define "litellm.service.extras" -}}
{{- if and (eq .type "LoadBalancer") .loadBalancerClass -}}
loadBalancerClass: {{ .loadBalancerClass | quote }}
{{- end }}
{{- end -}}
{{- define "litellm.gateway.prometheusMultiprocDir" -}}/tmp/litellm_prometheus_multiproc{{- end -}}
{{/*
@ -626,3 +758,27 @@ shutdown drain window.
{{- end }}
{{- end }}
{{- end -}}
{{- define "litellm.liteadminEnv" -}}
{{- $root := .root -}}
{{- with $root.Values.liteadmin }}
{{- if .enabled }}
- name: LITELLM_ADMIN_AGENT_URL
value: {{ printf "http://%s-liteadmin:10000" (include "litellm.fullname" $root | trunc 53 | trimSuffix "-") | quote }}
- name: ADMIN_AGENT_SERVICE_TOKEN
valueFrom:
secretKeyRef:
name: {{ required "liteadmin.existingSecret is required" .existingSecret }}
key: ADMIN_AGENT_SERVICE_TOKEN
{{- $hasProxyBaseUrl := false }}
{{- range $.component.extraEnv }}
{{- if eq .name "PROXY_BASE_URL" }}
{{- $hasProxyBaseUrl = true }}
{{- end }}
{{- end }}
{{- if not $hasProxyBaseUrl }}
- name: PROXY_BASE_URL
value: {{ required "liteadmin.gatewayUrl is required" .gatewayUrl | quote }}
{{- end }}
{{- end }}
{{- end }}
{{- end -}}

515
helm/litellm/templates/_workload.tpl Normal file
View file

@ -0,0 +1,515 @@
{{/*
The pod that runs the LLM data plane. In componentized mode it is the gateway
Deployment (`args: [gateway, ...]`), in monolith mode the proxy Deployment
(`args: [proxy, ...]`) that also serves the management API and the Admin UI.
Both are configured by .Values.gateway; the name, component label, selector
and args come from the litellm.workload.* helpers.
*/}}
{{- define "litellm.workload.deployment" -}}
{{- $component := include "litellm.workload.componentName" . -}}
{{- $fullname := include "litellm.workload.fullname" . -}}
{{- if and .Values.gateway.hpa.enabled .Values.gateway.keda.enabled }}
{{- fail "gateway.hpa.enabled and gateway.keda.enabled are mutually exclusive: two autoscalers on one Deployment fight over the replica count, so set gateway.hpa.enabled: false when using KEDA" }}
{{- end }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ $fullname }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: {{ $component }}
{{- with .Values.gateway.deploymentLabels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.gateway.deploymentAnnotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if and (not .Values.gateway.hpa.enabled) (not .Values.gateway.keda.enabled) (not (kindIs "invalid" .Values.gateway.replicaCount)) }}
replicas: {{ .Values.gateway.replicaCount }}
{{- end }}
{{- $minReady := .Values.gateway.minReadySeconds }}
{{- if not (or (kindIs "invalid" $minReady) (eq (printf "%v" $minReady) "")) }}
minReadySeconds: {{ $minReady }}
{{- end }}
{{- with .Values.gateway.strategy }}
strategy:
{{- toYaml . | nindent 4 }}
{{- end }}
selector:
matchLabels:
{{- include "litellm.workload.selectorLabels" . | nindent 6 }}
template:
metadata:
annotations:
{{- if .Values.gateway.config.create }}
checksum/config: {{ include (print $.Template.BasePath "/gateway/configmap.yaml") . | sha256sum }}
{{- end }}
{{- with .Values.gateway.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "litellm.workload.selectorLabels" . | nindent 8 }}
{{- with .Values.gateway.podLabels }}
{{- include "litellm.podLabels" (dict "podLabels" . "componentName" "gateway") | nindent 8 }}
{{- end }}
spec:
serviceAccountName: {{ include "litellm.gateway.serviceAccountName" . }}
automountServiceAccountToken: {{ .Values.serviceAccounts.gateway.automount }}
{{- with .Values.gateway.podSecurityContext }}
securityContext:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.gateway.extraInitContainers }}
initContainers:
{{- tpl (toYaml .) $ | nindent 8 }}
{{- end }}
containers:
- name: {{ $component }}
image: {{ include "litellm.image" . | quote }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
{{- with .Values.gateway.securityContext }}
securityContext:
{{- toYaml . | nindent 12 }}
{{- end }}
args:
{{- if .Values.monolith.enabled }}
{{- include "litellm.proxy.args" . | nindent 12 }}
{{- else }}
- gateway
- --host
- 0.0.0.0
- --port
- "4000"
{{- end }}
ports:
- name: http
containerPort: 4000
protocol: TCP
env:
{{- include "litellm.lensConnectionEnv" . | nindent 12 }}
{{- if and (not .Values.monolith.enabled) .Values.ui.publicUrl }}
- name: LITELLM_CORS_ORIGINS
value: {{ include "litellm.ui.publicOrigin" . | quote }}
{{- end }}
{{- include "litellm.masterKeyEnv" $ | nindent 12 }}
{{- if .Values.monolith.enabled }}
{{- include "litellm.liteadminEnv" (dict "root" $ "component" .Values.gateway) | nindent 12 }}
{{- end }}
{{- include "litellm.serverEnv" (dict "root" $ "component" .Values.gateway) | nindent 12 }}
{{- if .Values.gateway.config.create }}
- name: CONFIG_FILE_PATH
value: /app/config/config.yaml
{{- end }}
{{- if .Values.gateway.numWorkers }}
- name: NUM_WORKERS
value: {{ .Values.gateway.numWorkers | quote }}
{{- end }}
{{- if .Values.database.connectionPool.enabled }}
{{- include "litellm.connectionPoolEnv" $ | nindent 12 }}
{{- end }}
{{- if .Values.billingMetrics.enabled }}
{{- include "litellm.billingMetricsEnv" . | nindent 12 }}
{{- end }}
{{- if .Values.gateway.metricsServer.enabled }}
{{- if eq (int .Values.gateway.metricsServer.port) 4000 }}
{{- fail (printf "gateway.metricsServer.port must differ from the %s port 4000" $component) }}
{{- end }}
- name: PROMETHEUS_MULTIPROC_DIR
value: {{ include "litellm.gateway.prometheusMultiprocDir" . }}
{{- end }}
{{- if .Values.gateway.collector.enabled }}
{{- include "litellm.gateway.collectorEnv" . | nindent 12 }}
{{- end }}
{{- include "litellm.envFrom" .Values.gateway | nindent 10 }}
{{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.gateway.metricsServer.enabled (include "litellm.gateway.collectorSocketDir" .) }}
volumeMounts:
{{- if .Values.gateway.config.create }}
- name: gateway-config
mountPath: /app/config/config.yaml
subPath: config.yaml
{{- end }}
{{- if .Values.gateway.metricsServer.enabled }}
- name: prometheus-multiproc
mountPath: {{ include "litellm.gateway.prometheusMultiprocDir" . }}
{{- end }}
{{- if include "litellm.gateway.collectorSocketDir" . }}
- name: collector-socket
mountPath: {{ include "litellm.gateway.collectorSocketDir" . }}
{{- end }}
{{- with .Values.gateway.volumeMounts }}
{{- toYaml . | nindent 12 }}
{{- end }}
{{- end }}
{{- with .Values.gateway.livenessProbe }}
livenessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.gateway.readinessProbe }}
readinessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.gateway.startupProbe }}
startupProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.gateway.lifecycle }}
lifecycle:
{{- toYaml . | nindent 12 }}
{{- end }}
resources:
{{- toYaml .Values.gateway.resources | nindent 12 }}
{{- if .Values.gateway.metricsServer.enabled }}
- name: metrics
image: {{ include "litellm.image" . | quote }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
{{- with .Values.gateway.securityContext }}
securityContext:
{{- toYaml . | nindent 12 }}
{{- end }}
args:
- metrics
- --port
- {{ .Values.gateway.metricsServer.port | quote }}
env:
- name: PROMETHEUS_MULTIPROC_DIR
value: {{ include "litellm.gateway.prometheusMultiprocDir" . }}
ports:
- name: metrics
containerPort: {{ .Values.gateway.metricsServer.port }}
protocol: TCP
volumeMounts:
- name: prometheus-multiproc
mountPath: {{ include "litellm.gateway.prometheusMultiprocDir" . }}
readinessProbe:
tcpSocket: { port: metrics }
periodSeconds: 10
livenessProbe:
tcpSocket: { port: metrics }
periodSeconds: 15
failureThreshold: 6
resources:
{{- toYaml .Values.gateway.metricsServer.resources | nindent 12 }}
{{- end }}
{{- if .Values.gateway.collector.enabled }}
- name: collector
image: {{ include "litellm.image" . | quote }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
{{- with .Values.gateway.securityContext }}
securityContext:
{{- toYaml . | nindent 12 }}
{{- end }}
args:
- collector
env:
{{- include "litellm.masterKeyEnv" $ | nindent 12 }}
{{- include "litellm.serverEnv" (dict "root" $ "component" .Values.gateway) | nindent 12 }}
{{- if .Values.gateway.config.create }}
- name: CONFIG_FILE_PATH
value: /app/config/config.yaml
{{- end }}
{{- if .Values.database.connectionPool.enabled }}
{{- include "litellm.connectionPoolEnv" $ | nindent 12 }}
{{- end }}
{{- include "litellm.gateway.collectorEnv" . | nindent 12 }}
- name: LITELLM_JOB_ROLE
value: collector
{{- include "litellm.envFrom" .Values.gateway | nindent 10 }}
{{- if or .Values.gateway.config.create .Values.gateway.volumeMounts (include "litellm.gateway.collectorSocketDir" .) }}
volumeMounts:
{{- if .Values.gateway.config.create }}
- name: gateway-config
mountPath: /app/config/config.yaml
subPath: config.yaml
{{- end }}
{{- if include "litellm.gateway.collectorSocketDir" . }}
- name: collector-socket
mountPath: {{ include "litellm.gateway.collectorSocketDir" . }}
{{- end }}
{{- with .Values.gateway.volumeMounts }}
{{- toYaml . | nindent 12 }}
{{- end }}
{{- end }}
resources:
{{- toYaml .Values.gateway.collector.resources | nindent 12 }}
{{- end }}
{{- with .Values.gateway.extraContainers }}
{{- tpl (toYaml .) $ | nindent 8 }}
{{- end }}
{{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.gateway.metricsServer.enabled (include "litellm.gateway.collectorSocketDir" .) }}
volumes:
{{- if .Values.gateway.config.create }}
- name: gateway-config
configMap:
name: {{ include "litellm.workload.fullname" . }}-config
{{- end }}
{{- if .Values.gateway.metricsServer.enabled }}
- name: prometheus-multiproc
emptyDir: {}
{{- end }}
{{- if include "litellm.gateway.collectorSocketDir" . }}
- name: collector-socket
emptyDir:
sizeLimit: 1Mi
{{- end }}
{{- with .Values.gateway.volumes }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}
{{- with .Values.gateway.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.gateway.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.gateway.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.gateway.topologySpreadConstraints }}
topologySpreadConstraints:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- $gracePeriod := .Values.gateway.terminationGracePeriodSeconds }}
{{- if not (or (kindIs "invalid" $gracePeriod) (eq (printf "%v" $gracePeriod) "")) }}
terminationGracePeriodSeconds: {{ $gracePeriod }}
{{- end }}
{{- end -}}
{{- define "litellm.workload.service" -}}
{{- $component := include "litellm.workload.componentName" . -}}
apiVersion: v1
kind: Service
metadata:
name: {{ include "litellm.workload.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: {{ $component }}
{{- with .Values.gateway.service.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
type: {{ .Values.gateway.service.type }}
{{- with include "litellm.service.extras" .Values.gateway.service }}
{{- . | nindent 2 }}
{{- end }}
ports:
- port: {{ .Values.gateway.service.port }}
targetPort: http
protocol: TCP
name: http
selector:
{{- include "litellm.workload.selectorLabels" . | nindent 4 }}
{{- end -}}
{{- define "litellm.workload.hpa" -}}
{{- $component := include "litellm.workload.componentName" . -}}
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: {{ include "litellm.workload.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: {{ $component }}
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: {{ include "litellm.workload.fullname" . }}
minReplicas: {{ .Values.gateway.hpa.minReplicas }}
maxReplicas: {{ .Values.gateway.hpa.maxReplicas }}
metrics:
{{- if .Values.gateway.hpa.targetCPUUtilizationPercentage }}
{{- if and .Values.gateway.collector.enabled .Values.gateway.collector.scaleOnGatewayContainerCpu }}
- type: ContainerResource
containerResource:
name: cpu
container: {{ $component }}
target:
type: Utilization
averageUtilization: {{ .Values.gateway.hpa.targetCPUUtilizationPercentage }}
{{- else }}
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: {{ .Values.gateway.hpa.targetCPUUtilizationPercentage }}
{{- end }}
{{- end }}
{{- if .Values.gateway.hpa.targetMemoryUtilizationPercentage }}
- type: Resource
resource:
name: memory
target:
type: Utilization
averageUtilization: {{ .Values.gateway.hpa.targetMemoryUtilizationPercentage }}
{{- end }}
{{- with .Values.gateway.hpa.targetRequestsPerSecond }}
- type: Pods
pods:
metric:
name: litellm_requests_per_second
target:
type: AverageValue
averageValue: {{ toJson . | trimAll "\"" | quote }}
{{- end }}
{{- with .Values.gateway.hpa.targetTokensPerSecond }}
- type: Pods
pods:
metric:
name: litellm_tokens_per_second
target:
type: AverageValue
averageValue: {{ toJson . | trimAll "\"" | quote }}
{{- end }}
{{- with .Values.gateway.hpa.behavior }}
behavior:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end -}}
{{/*
KEDA ScaledObject targeting the workload Deployment. The optional Prometheus
triggers query the same per pod counters the HPA workload metrics use,
scoped to this release's metrics Service through the `job` label the
ServiceMonitor gives every scrape.
*/}}
{{- define "litellm.workload.keda" -}}
{{- $component := include "litellm.workload.componentName" . -}}
{{- $keda := .Values.gateway.keda -}}
{{- $prom := $keda.prometheus -}}
{{- $wantsPrometheus := or $prom.targetRequestsPerSecond $prom.targetTokensPerSecond -}}
{{- if and $wantsPrometheus (not $prom.serverAddress) }}
{{- fail "gateway.keda.prometheus.serverAddress is required when gateway.keda.prometheus.targetRequestsPerSecond or targetTokensPerSecond is set" }}
{{- end }}
{{- if and $wantsPrometheus (not .Values.gateway.metricsServer.enabled) }}
{{- fail "gateway.metricsServer.enabled is required when gateway.keda.prometheus.targetRequestsPerSecond or targetTokensPerSecond is set" }}
{{- end }}
{{- $selector := printf "namespace=%q,job=%q" .Release.Namespace (printf "%s-metrics" (include "litellm.workload.fullname" .)) -}}
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
name: {{ include "litellm.workload.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: {{ $component }}
{{- with (dig "scaledObject" "annotations" (dict) $keda) }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: {{ include "litellm.workload.fullname" . }}
minReplicaCount: {{ $keda.minReplicaCount }}
maxReplicaCount: {{ $keda.maxReplicaCount }}
pollingInterval: {{ $keda.pollingInterval }}
cooldownPeriod: {{ $keda.cooldownPeriod }}
{{- with $keda.advanced }}
advanced:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with $keda.fallback }}
{{- $fallback := deepCopy . }}
{{- if not (hasKey $fallback "failureThreshold") }}
{{- $_ := set $fallback "failureThreshold" 3 }}
{{- end }}
{{- if not (hasKey $fallback "replicas") }}
{{- $_ := set $fallback "replicas" $keda.maxReplicaCount }}
{{- end }}
fallback:
{{- toYaml $fallback | nindent 4 }}
{{- end }}
triggers:
{{- with $keda.triggers }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with $prom.targetRequestsPerSecond }}
- type: prometheus
metricType: AverageValue
metadata:
serverAddress: {{ $prom.serverAddress | quote }}
query: {{ printf "sum(rate(litellm_proxy_total_requests_metric_total{%s}[1m]))" $selector | quote }}
threshold: {{ toJson . | trimAll "\"" | quote }}
{{- end }}
{{- with $prom.targetTokensPerSecond }}
- type: prometheus
metricType: AverageValue
metadata:
serverAddress: {{ $prom.serverAddress | quote }}
query: {{ printf "sum(rate(litellm_total_tokens_metric_total{%s}[1m]))" $selector | quote }}
threshold: {{ toJson . | trimAll "\"" | quote }}
{{- end }}
{{- end -}}
{{- define "litellm.workload.serviceMetrics" -}}
{{- $component := include "litellm.workload.componentName" . -}}
apiVersion: v1
kind: Service
metadata:
name: {{ include "litellm.workload.fullname" . }}-metrics
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: {{ $component }}
spec:
type: ClusterIP
ports:
- port: {{ .Values.gateway.metricsServer.port }}
targetPort: metrics
protocol: TCP
name: metrics
selector:
{{- include "litellm.workload.selectorLabels" . | nindent 4 }}
{{- end -}}
{{- define "litellm.workload.serviceMonitor" -}}
{{- $component := include "litellm.workload.componentName" . -}}
{{- if not .Values.gateway.metricsServer.enabled }}
{{- fail "gateway.serviceMonitor.enabled requires gateway.metricsServer.enabled: the http port serves /metrics/ behind virtual-key auth, so an unauthenticated scrape gets 401" }}
{{- end }}
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: {{ include "litellm.workload.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: {{ $component }}
{{- with .Values.gateway.serviceMonitor.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.gateway.serviceMonitor.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
selector:
matchLabels:
{{- include "litellm.workload.selectorLabels" . | nindent 6 }}
namespaceSelector:
matchNames:
{{- with .Values.gateway.serviceMonitor.namespaceSelector.matchNames }}
{{- toYaml . | nindent 6 }}
{{- else }}
- {{ .Release.Namespace | quote }}
{{- end }}
endpoints:
- port: metrics
path: /metrics/
interval: {{ .Values.gateway.serviceMonitor.interval }}
scrapeTimeout: {{ .Values.gateway.serviceMonitor.scrapeTimeout }}
scheme: http
{{- with .Values.gateway.serviceMonitor.relabelings }}
relabelings:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end -}}

31
helm/litellm/templates/backend/deployment.yaml
View file

@ -1,4 +1,4 @@
{{- if .Values.backend.enabled }}
{{- if include "litellm.backend.render" . }}
{{- $componentHosts := .Values.ingress.componentHosts | default dict -}}
{{- $hasComponentHosts := or $componentHosts.ui $componentHosts.backend $componentHosts.gateway -}}
apiVersion: apps/v1
@ -8,10 +8,21 @@ metadata:
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: backend
{{- with .Values.backend.deploymentLabels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.backend.deploymentAnnotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if and (not .Values.backend.hpa.enabled) (not (kindIs "invalid" .Values.backend.replicaCount)) }}
replicas: {{ .Values.backend.replicaCount }}
{{- end }}
{{- $minReady := .Values.backend.minReadySeconds }}
{{- if not (or (kindIs "invalid" $minReady) (eq (printf "%v" $minReady) "")) }}
minReadySeconds: {{ $minReady }}
{{- end }}
{{- with .Values.backend.strategy }}
strategy:
{{- toYaml . | nindent 4 }}
@ -46,14 +57,24 @@ spec:
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.backend.extraInitContainers }}
initContainers:
{{- tpl (toYaml .) $ | nindent 8 }}
{{- end }}
containers:
- name: backend
image: "{{ .Values.backend.image.repository }}:{{ .Values.backend.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.backend.image.pullPolicy }}
image: {{ include "litellm.image" . | quote }}
imagePullPolicy: {{ .Values.image.pullPolicy }}
{{- with .Values.backend.securityContext }}
securityContext:
{{- toYaml . | nindent 12 }}
{{- end }}
args:
- backend
- --host
- 0.0.0.0
- --port
- "4001"
ports:
- name: http
containerPort: 4001
@ -66,6 +87,8 @@ spec:
- name: LITELLM_CORS_ALLOW_CREDENTIALS
value: "true"
{{- end }}
{{- include "litellm.masterKeyEnv" $ | nindent 12 }}
{{- include "litellm.liteadminEnv" (dict "root" $ "component" .Values.backend) | nindent 12 }}
{{- include "litellm.serverEnv" (dict "root" $ "component" .Values.backend) | nindent 12 }}
{{- if .Values.backend.gatewayUrl }}
- name: LITELLM_GATEWAY_URL
@ -116,7 +139,7 @@ spec:
{{- if .Values.gateway.config.create }}
- name: gateway-config
configMap:
name: {{ include "litellm.gateway.fullname" . }}-config
name: {{ include "litellm.workload.fullname" . }}-config
{{- end }}
{{- with .Values.backend.volumes }}
{{- toYaml . | nindent 8 }}

2
helm/litellm/templates/backend/hpa.yaml
View file

@ -1,4 +1,4 @@
{{- if and .Values.backend.enabled .Values.backend.hpa.enabled }}
{{- if and (include "litellm.backend.render" .) .Values.backend.hpa.enabled }}
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:

1
helm/litellm/templates/backend/poddisruptionbudget.yaml
View file

@ -1,5 +1,6 @@
{{- include "litellm.pdb" (dict
"root" $
"enabled" (include "litellm.backend.render" .)
"component" .Values.backend
"componentName" "backend"
"fullname" (include "litellm.backend.fullname" .)

9
helm/litellm/templates/backend/service.yaml
View file

@ -1,4 +1,4 @@
{{- if .Values.backend.enabled }}
{{- if include "litellm.backend.render" . }}
apiVersion: v1
kind: Service
metadata:
@ -6,8 +6,15 @@ metadata:
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: backend
{{- with .Values.backend.service.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
type: {{ .Values.backend.service.type }}
{{- with include "litellm.service.extras" .Values.backend.service }}
{{- . | nindent 2 }}
{{- end }}
ports:
- port: {{ .Values.backend.service.port }}
targetPort: http

4
helm/litellm/templates/extra-resources.yaml Normal file
View file

@ -0,0 +1,4 @@
{{- range .Values.extraResources }}
---
{{ tpl (toYaml .) $ }}
{{- end }}

5
helm/litellm/templates/gateway/configmap.yaml
View file

@ -2,7 +2,10 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "litellm.gateway.fullname" . }}-config
name: {{ include "litellm.workload.fullname" . }}-config
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: {{ include "litellm.workload.componentName" . }}
data:
config.yaml: |
{{ .Values.gateway.config.proxy_config | toYaml | indent 6 }}

247
helm/litellm/templates/gateway/deployment.yaml
View file

@ -1,246 +1,3 @@
{{- if .Values.gateway.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "litellm.gateway.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: gateway
spec:
{{- if and (not .Values.gateway.hpa.enabled) (not (kindIs "invalid" .Values.gateway.replicaCount)) }}
replicas: {{ .Values.gateway.replicaCount }}
{{- end }}
{{- with .Values.gateway.strategy }}
strategy:
{{- toYaml . | nindent 4 }}
{{- end }}
selector:
matchLabels:
{{- include "litellm.gateway.selectorLabels" . | nindent 6 }}
template:
metadata:
annotations:
{{- if .Values.gateway.config.create }}
checksum/config: {{ include (print $.Template.BasePath "/gateway/configmap.yaml") . | sha256sum }}
{{- end }}
{{- with .Values.gateway.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "litellm.gateway.selectorLabels" . | nindent 8 }}
{{- with .Values.gateway.podLabels }}
{{- include "litellm.podLabels" (dict "podLabels" . "componentName" "gateway") | nindent 8 }}
{{- end }}
spec:
serviceAccountName: {{ include "litellm.gateway.serviceAccountName" . }}
automountServiceAccountToken: {{ .Values.serviceAccounts.gateway.automount }}
{{- with .Values.gateway.podSecurityContext }}
securityContext:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: gateway
image: "{{ .Values.gateway.image.repository }}:{{ .Values.gateway.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.gateway.image.pullPolicy }}
{{- with .Values.gateway.securityContext }}
securityContext:
{{- toYaml . | nindent 12 }}
{{- end }}
ports:
- name: http
containerPort: 4000
protocol: TCP
env:
{{- include "litellm.lensConnectionEnv" . | nindent 12 }}
{{- if .Values.ui.publicUrl }}
- name: LITELLM_CORS_ORIGINS
value: {{ include "litellm.ui.publicOrigin" . | quote }}
{{- end }}
{{- include "litellm.serverEnv" (dict "root" $ "component" .Values.gateway) | nindent 12 }}
{{- if .Values.gateway.config.create }}
- name: CONFIG_FILE_PATH
value: /app/config/config.yaml
{{- end }}
{{- if .Values.gateway.numWorkers }}
- name: NUM_WORKERS
value: {{ .Values.gateway.numWorkers | quote }}
{{- end }}
{{- if .Values.database.connectionPool.enabled }}
{{- include "litellm.connectionPoolEnv" $ | nindent 12 }}
{{- end }}
{{- if .Values.billingMetrics.enabled }}
{{- include "litellm.billingMetricsEnv" . | nindent 12 }}
{{- end }}
{{- if .Values.gateway.metricsServer.enabled }}
{{- if eq (int .Values.gateway.metricsServer.port) 4000 }}
{{- fail "gateway.metricsServer.port must differ from the gateway port 4000" }}
{{- end }}
- name: PROMETHEUS_MULTIPROC_DIR
value: {{ include "litellm.gateway.prometheusMultiprocDir" . }}
{{- end }}
{{- if .Values.gateway.collector.enabled }}
{{- include "litellm.gateway.collectorEnv" . | nindent 12 }}
{{- end }}
{{- include "litellm.envFrom" .Values.gateway | nindent 10 }}
{{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.gateway.metricsServer.enabled (include "litellm.gateway.collectorSocketDir" .) }}
volumeMounts:
{{- if .Values.gateway.config.create }}
- name: gateway-config
mountPath: /app/config/config.yaml
subPath: config.yaml
{{- end }}
{{- if .Values.gateway.metricsServer.enabled }}
- name: prometheus-multiproc
mountPath: {{ include "litellm.gateway.prometheusMultiprocDir" . }}
{{- end }}
{{- if include "litellm.gateway.collectorSocketDir" . }}
- name: collector-socket
mountPath: {{ include "litellm.gateway.collectorSocketDir" . }}
{{- end }}
{{- with .Values.gateway.volumeMounts }}
{{- toYaml . | nindent 12 }}
{{- end }}
{{- end }}
{{- with .Values.gateway.livenessProbe }}
livenessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.gateway.readinessProbe }}
readinessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.gateway.startupProbe }}
startupProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.gateway.lifecycle }}
lifecycle:
{{- toYaml . | nindent 12 }}
{{- end }}
resources:
{{- toYaml .Values.gateway.resources | nindent 12 }}
{{- if .Values.gateway.metricsServer.enabled }}
- name: metrics
image: "{{ .Values.gateway.image.repository }}:{{ .Values.gateway.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.gateway.image.pullPolicy }}
{{- with .Values.gateway.securityContext }}
securityContext:
{{- toYaml . | nindent 12 }}
{{- end }}
command:
- python
- -m
- litellm.proxy.prometheus_metrics_server
- --port
- {{ .Values.gateway.metricsServer.port | quote }}
env:
- name: PROMETHEUS_MULTIPROC_DIR
value: {{ include "litellm.gateway.prometheusMultiprocDir" . }}
ports:
- name: metrics
containerPort: {{ .Values.gateway.metricsServer.port }}
protocol: TCP
volumeMounts:
- name: prometheus-multiproc
mountPath: {{ include "litellm.gateway.prometheusMultiprocDir" . }}
readinessProbe:
tcpSocket: { port: metrics }
periodSeconds: 10
livenessProbe:
tcpSocket: { port: metrics }
periodSeconds: 15
failureThreshold: 6
resources:
{{- toYaml .Values.gateway.metricsServer.resources | nindent 12 }}
{{- end }}
{{- if .Values.gateway.collector.enabled }}
- name: collector
image: "{{ .Values.gateway.image.repository }}:{{ .Values.gateway.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.gateway.image.pullPolicy }}
{{- with .Values.gateway.securityContext }}
securityContext:
{{- toYaml . | nindent 12 }}
{{- end }}
command:
- python
- -m
- litellm.proxy.collector
env:
{{- include "litellm.serverEnv" (dict "root" $ "component" .Values.gateway) | nindent 12 }}
{{- if .Values.gateway.config.create }}
- name: CONFIG_FILE_PATH
value: /app/config/config.yaml
{{- end }}
{{- if .Values.database.connectionPool.enabled }}
{{- include "litellm.connectionPoolEnv" $ | nindent 12 }}
{{- end }}
{{- include "litellm.gateway.collectorEnv" . | nindent 12 }}
- name: LITELLM_JOB_ROLE
value: collector
{{- include "litellm.envFrom" .Values.gateway | nindent 10 }}
{{- if or .Values.gateway.config.create .Values.gateway.volumeMounts (include "litellm.gateway.collectorSocketDir" .) }}
volumeMounts:
{{- if .Values.gateway.config.create }}
- name: gateway-config
mountPath: /app/config/config.yaml
subPath: config.yaml
{{- end }}
{{- if include "litellm.gateway.collectorSocketDir" . }}
- name: collector-socket
mountPath: {{ include "litellm.gateway.collectorSocketDir" . }}
{{- end }}
{{- with .Values.gateway.volumeMounts }}
{{- toYaml . | nindent 12 }}
{{- end }}
{{- end }}
resources:
{{- toYaml .Values.gateway.collector.resources | nindent 12 }}
{{- end }}
{{- with .Values.gateway.extraContainers }}
{{- tpl (toYaml .) $ | nindent 8 }}
{{- end }}
{{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.gateway.metricsServer.enabled (include "litellm.gateway.collectorSocketDir" .) }}
volumes:
{{- if .Values.gateway.config.create }}
- name: gateway-config
configMap:
name: {{ include "litellm.gateway.fullname" . }}-config
{{- end }}
{{- if .Values.gateway.metricsServer.enabled }}
- name: prometheus-multiproc
emptyDir: {}
{{- end }}
{{- if include "litellm.gateway.collectorSocketDir" . }}
- name: collector-socket
emptyDir:
sizeLimit: 1Mi
{{- end }}
{{- with .Values.gateway.volumes }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}
{{- with .Values.gateway.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.gateway.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.gateway.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.gateway.topologySpreadConstraints }}
topologySpreadConstraints:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- $gracePeriod := .Values.gateway.terminationGracePeriodSeconds }}
{{- if not (or (kindIs "invalid" $gracePeriod) (eq (printf "%v" $gracePeriod) "")) }}
terminationGracePeriodSeconds: {{ $gracePeriod }}
{{- end }}
{{- if include "litellm.gateway.render" . }}
{{- include "litellm.workload.deployment" . }}
{{- end }}

66
helm/litellm/templates/gateway/hpa.yaml
View file

@ -1,65 +1,3 @@
{{- if and .Values.gateway.enabled .Values.gateway.hpa.enabled }}
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: {{ include "litellm.gateway.fullname" . }}
labels:
{{- include "litellm.commonLabels" . | nindent 4 }}
app.kubernetes.io/component: gateway
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: {{ include "litellm.gateway.fullname" . }}
minReplicas: {{ .Values.gateway.hpa.minReplicas }}
maxReplicas: {{ .Values.gateway.hpa.maxReplicas }}
metrics:
{{- if .Values.gateway.hpa.targetCPUUtilizationPercentage }}
{{- if and .Values.gateway.collector.enabled .Values.gateway.collector.scaleOnGatewayContainerCpu }}
- type: ContainerResource
containerResource:
name: cpu
container: gateway
target:
type: Utilization
averageUtilization: {{ .Values.gateway.hpa.targetCPUUtilizationPercentage }}
{{- else }}
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: {{ .Values.gateway.hpa.targetCPUUtilizationPercentage }}
{{- end }}
{{- end }}
{{- if .Values.gateway.hpa.targetMemoryUtilizationPercentage }}
- type: Resource
resource:
name: memory
target:
type: Utilization
averageUtilization: {{ .Values.gateway.hpa.targetMemoryUtilizationPercentage }}
{{- end }}
{{- with .Values.gateway.hpa.targetRequestsPerSecond }}
- type: Pods
pods:
metric:
name: litellm_requests_per_second
target:
type: AverageValue
averageValue: {{ toJson . | trimAll "\"" | quote }}
{{- end }}
{{- with .Values.gateway.hpa.targetTokensPerSecond }}
- type: Pods
pods:
metric:
name: litellm_tokens_per_second
target:
type: AverageValue
averageValue: {{ toJson . | trimAll "\"" | quote }}
{{- end }}
{{- with .Values.gateway.hpa.behavior }}
behavior:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- if and (include "litellm.gateway.render" .) .Values.gateway.hpa.enabled }}
{{- include "litellm.workload.hpa" . }}
{{- end }}

3
helm/litellm/templates/gateway/keda.yaml Normal file
View file

@ -0,0 +1,3 @@
{{- if and (include "litellm.gateway.render" .) .Values.gateway.keda.enabled }}
{{- include "litellm.workload.keda" . }}
{{- end }}

Some files were not shown because too many files have changed in this diff Show more